yanser 0.0.1 → 0.0.2

data/README

@@ -1 +1,21 @@
-new readme
+= YANSER
+
+* {RubyGems}[http://rubygems.org/gems/yanser]
+* Developers {Homepage}[http://www.uni-trier.de/index.php?id=24140]
+* {YANAPI Project Page}[http://yanser.rubyforge.org/]
+
+== DESCRIPTION
+
+YANSER (Yahoo! ANSwers harvestER) is a convinient search tool providing access to the Yahoo! Answers Q&A collection. Based on YANAPI it provides a simple CLI and helps to search for Questions and Answers which contain a set of key words, belong to a specific semantic domain or are posted by a certain user. Yanser is a research tool in the field of Computational Linguistics.
+
+== SYNOPSIS
+  $ yanser --help
+
+
+
+== LICENSE
+
+YANSER is a copyrighted software by Andrei Beliankou, 2011.
+You may use, redistribute and change it under the terms
+provided in the LICENSE file.
+

data/Rakefile

@@ -1,14 +1,41 @@
-require 'rubygems'
+# We need rake to user FileLists.
 require 'rake'
-# we can require 'rake/clean' to add 'clobber' and 'clean' tasks
+# We can require 'rake/clean' to add 'clobber' and 'clean' tasks.
 require 'rake/clean'
-require 'rake/testtask'
 
 
+# I am not sure how to use this constant.
 SRC = FileList['**/*.rb']
 
 CLOBBER.include('doc', '**/*.html', '**/*.gem')
 
+# testing
+require 'rake/testtask'
 Rake::TestTask.new do |t|
-  t.test_files = FileList.new('test/yanapi/*.rb')
+  t.test_files = FileList.new('test/**/*.rb').to_a
+  # rake starts an other ruby process with a new options set.
+  # ruby --some-option -S rake is not sufficient to propagate
+  # the option "--some-option".
+  t.ruby_opts = ['-rubygems']
+end
+
+# Build the gem package
+load 'yanser.gemspec'
+require 'rubygems/package_task'
+Gem::PackageTask.new(GEMSPEC).define
+
+# Generate documentation
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+  rdoc.rdoc_files.include('README', 'LICENSE', 'lib/**/*')
+end
+
+desc "Open an irb session preloaded with this library."
+task :console do
+  sh "irb -rubygems -I lib -r yanser/opt_parser -r yanser/yanser"
+end
+
+desc 'Run the main executable file of the project.'
+task :start do
+  sh "ruby -w -rubygems -I lib bin/yanser #{ENV['cmd']}"
 end

data/bin/yanser

@@ -1,19 +1,19 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8; mode: ruby -*-
 
 # это неверное решение, на самом деле я не должен напрямую указывать,
 # где находятся файлы, но пока пусть будет так
-lib_path = File.expand_path('../../lib', __FILE__)
-$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
+# it is required only for .tar.gz distribution
+#lib_path = File.expand_path('../../lib', __FILE__)
+#$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
 
 # подобный способ является правильным, только так классы Yanser и OptionParser будут независимы
-require 'yanser'
-require 'option_parser'
+require 'yanser/yanser'
+require 'yanser/opt_parser'
 
 # class method, why not?
-options = OptionParser.parse(ARGV)
+options = Yanser::OptParser.parse(ARGV)
 
 
-yanser = Yanser.new(options)
+yanser = Yanser::Yanser.new(options)
 
 yanser.start

data/lib/yanser/error.rb

@@ -0,0 +1,11 @@
+# -*- conding: utf-8 -*-
+
+module Yanser
+  class OptParserError < RuntimeError
+    # empty for now
+  end
+
+  class YanserError < RuntimeError
+    # empty for now
+  end
+end

data/lib/yanser/opt_parser.rb

@@ -0,0 +1,313 @@
+# -*- coding: utf-8 -*-
+require 'optparse'
+require 'fileutils'
+
+require 'yanser/version'
+require 'yanser/error'
+
+module Yanser
+  class OptParser
+    # OP expects cmd_args to be an array like ARGV
+    # dummy output for temporary usage
+    def self.parse(cmd_args)
+      @@options = {}
+      @@options[:query_params] = {}
+      
+      parser = create_parser
+      
+      # if no options provided print the help
+      if cmd_args.empty?
+        msg = "You have to provide some options. " +
+          "yanser --help provides more information."
+        raise OptParserError, msg
+#        $stderr.printf "You have to provide some options.\n\n"
+#        puts parser
+#        exit 1
+      end
+      
+      # Parse ARGV and provide the options hash.
+      # Check if everything is correct and handle exceptions
+      begin
+        parser.parse!(cmd_args)
+        # rescue all exceptions from OptionParser  
+      rescue => e
+        raise OptParserError, e.message
+#        $stderr.printf "#{e.message.capitalize}\n\n"
+#        puts parser
+#        exit 1
+      end
+      
+      # Check to see if we got the required arguments needed.
+      check_required_options(@@options)
+      
+      # Set the search method.
+      @@options[:query_type] = set_query_type(@@options[:query_params])
+      
+      # Set some defaults such as :region and :search_in
+      @@options = set_defaults(@@options)
+      
+      return @@options
+    end # parse
+    
+    private
+    
+    # Check if the value of given region is correct.
+    # Now 14 regions are supported by Yahoo! Answers.
+    def self.prove_region(region)
+      regions = ['de', 'us', 'uk', 'ca', 'au', 'in', 'es',
+                 'br', 'ar', 'mx', 'e1', 'it', 'fr', 'sg']
+      
+      unless regions.include?(region)
+        msg = "The provided search region #{region} is currently not supported by Yahoo!"
+        raise OptParserError, msg
+      end
+    end
+    
+    # define one of the following query types: TermQuery|CategoryQuery|
+    # QuestionQuery|UserQuery
+    def self.set_query_type(params)
+      case
+      when (params[:category_id] || params[:category_name]) && ! params[:query]
+        'CategoryQuery'
+      when params[:query]
+        'TermQuery'
+      when params[:user_id]
+        'UserQuery'
+      when params[:question_id]
+        'QuestionQuery'
+      end
+    end
+    
+    def self.check_required_options(options)
+      required_opts = [:appid]
+      
+      required_opts.each do |opt|
+        if options[:query_params].has_key?(opt)
+          next
+        else
+          msg = "A required option --#{opt} is missing."
+          raise OptParserError, msg
+        end
+      end
+    end
+    
+    def self.provide_dir(dir)
+      dir = File.expand_path(dir)
+      #check for existens
+      if File.directory?(dir)
+        if File.writable?(dir)
+          return dir
+        else
+          $stderr.puts 'The directory you have provided is not writable!'
+          exit 1
+        end
+      else
+        FileUtils.mkdir_p(dir)
+        return dir
+      end
+    end # provide_dir
+    
+    def self.set_defaults(opts)
+      # fake method for now
+      return opts
+    end # set_defaults
+
+    def self.create_parser
+      OptionParser.new do |opts|
+        opts.banner = 'Usage: yanser OPTIONS'
+        
+        opts.separator ''
+        opts.separator 'Program specific options:'
+        
+        opts.on('--appid APPID',
+                'Provide an ApplicationID given by Yahoo,',
+                'to test Yanser you can use <YahooDemo> as the APPID,',
+                'think in this case on limitations placed by Yahoo.',
+                'This option is required!'
+                ) do |appid|
+          @@options[:query_params][:appid] = appid
+        end
+        
+        
+        opts.separator ''
+        opts.separator '  Mandatory search arguments:'
+        
+        opts.on('-k', '--key-word KEYWORD',
+                'Provide a single keyword or a boolean expression.'
+                ) do |keyword|
+          @@options[:query_params][:query] = keyword
+          # not a solution!!!
+          @@options[:query_params][:search_in] = 'question'
+          
+        end
+        
+        opts.separator ''
+        
+        opts.on('-c', '--category CATEGORY',
+                'Provide a category name or ID.'
+                ) do |category|
+          
+          if category =~ /^[[:digit:]]+$/
+            @@options[:query_params][:category_id] = category
+          else
+            @@options[:query_params][:category_name] = category
+          end
+          
+        end
+        
+        opts.separator ''
+        
+        opts.on('--user-id ID',
+                'Provide an user ID of questions you search for.',
+                'This way you can get question by a specific user.'
+                ) do |user_id|
+          @@options[:query_params][:user_id] = user_id
+        end
+        
+        opts.separator ''
+        
+        opts.on('--question-id ID',
+                'Provide a question ID of the question you search for.',
+                'It returns a unique question.'
+                ) do |question_id|
+          @@options[:query_params][:question_id] = question_id
+        end
+        
+        opts.separator ''
+        opts.separator '  Optional search arguments:'
+        
+        opts.on('-r', '--region REGION',
+                'Provide a geographic region to search in for terms.',
+                'Possible values are: de, us, uk, ca, au, in, es, br,',
+                '                     ar, mx, e1, it, fr, sg.',
+                'This defaults to en.'
+                ) do |region|
+          # todo
+          prove_region(region)
+          @@options[:query_params][:region] = region
+        end
+        
+        opts.separator ''
+        
+        opts.on('-o', '--output-dir DIR',
+                'Provide an output folder.',
+                'This directory will be created if it does not exist yet.'
+                ) do |output_dir|
+          @@options[:output_dir] = provide_dir(output_dir)
+        end
+        
+        opts.separator ''
+        
+        opts.on('-l', '--limit NUMBER', Integer,
+                'Provide a number of answers you want to get from Yahoo.',
+                'This argument is not mandatory, if you want to get',
+                'all answers simply ommit this argument.'
+                ) do |limit|
+          @@options[:limit] = limit
+        end
+        
+        opts.separator ''
+        opts.on('-f', '--output-format FORMAT',
+                'Provide an output format: xml, json, rss, php.',
+                'It defaults to xml, and you can simply ommit this option.'
+                ) do |f|
+          raise NotImplementedError, 'Only default output format is implemented'
+        end
+        
+        opts.separator ''
+        opts.on('--prefix PREFIX',
+                'Provide a prefix for the output files. By default',
+                'the filename begins with the index of the retrieved',
+                'question. You can alter this by providing a prefix.',
+                'It can be useful if you want to put many query results',
+                'in the same output folder.'
+                ) do |pref|
+          raise NotImplementedError, 'No prefixes implemeted.'
+        end
+
+        opts.separator ''
+        opts.on('--key-word-list FILE',
+                'Provide a file with key words or phrases (boolean syntax',
+                'allowed) one search token per line.'
+                ) do |file_name|
+          raise NotImplementedError
+        end
+
+        opts.separator ''
+        opts.on('--exp-file FILE',
+                'Provide a formal description of the experiment.'
+                ) do |file_name|
+          raise NotImplementedError
+        end
+        opts.separator ""
+        opts.separator "Common options:"
+        
+        opts.on_tail('-h', '--help', 'Show the help message.') do
+          puts opts
+          exit
+        end
+        
+        opts.on_tail('-v', '--version', 'Show the program version.') do
+          puts VERSION
+          exit
+        end
+      end # parser
+    end # create_parser
+    
+  end # OptionParser
+end # module
+
+__END__
+# List of options planned for implementation.
+--appid
+-k, --key-word
+-c, --category-id # Term & CategorySearch
+--user-id
+--question-id # QuestionSearch
+-r, --region
+-f, --output-format
+-l, --limit
+--date-range
+--sort
+--filter
+
+
+--prefix #prefix for output files
+--key-word-list
+--exp-file
+-o, --output-dir
+-h, --help
+-v, --version
+-t, --time-interval
+
+
+
+
+
+
+instance interface
+
+{
+  :query_params => {
+    :appid => 'YahooDemo' | 'SomeStringWithYourID',
+    :callback => 'str',
+    :category_id => '123456',
+    :category_name => 'Wohnen',
+    :date_range => 'all'|'7'|'7-30'|'30-60'|'60-90'|'more90', # default: all
+    :filter => 'question'|'best_answer', # default: question
+    :output => 'xml'|'json'|'php'|'rss',
+    :query => 'Haus AND Grund',
+    :question_id => '123456',
+    :region => 'de'|'us'|'uk'|'ca'|'au'|'in'|'es'|'br'|'ar'|'mx'|'e1'|'it'|'fr'|'sg',  # default 'en'
+    :results => integer, # 0..50, default: 10 (0 returns the default value)
+    :search_in => "all" | "question" | "best_answer", # default 'all'
+    :sort => 'relevance' | 'date_desc'| 'date_asc', # default 'relevance'
+    :start => integer, # <= 1000, default: 0
+    :type => "all" | "resolved" | "open" | "undecided", # default 'all'
+    :user_id => '123456'
+  },
+  :query_type => 'TermQuery'|'QuestionQuery'|'UserQuery'|'CategoryQuery',
+  :output_dir => 'some path',
+  :prefix => 'some prefix', # prefix for output files
+  :limit => Integer
+}

data/lib/yanser/version.rb

@@ -0,0 +1,3 @@
+module Yanser
+  VERSION = '0.0.2'
+end

data/lib/yanser/yanser.rb

@@ -0,0 +1,135 @@
+# this solution is not ideal since I force people to use 'rubygems'
+# possible it would be a better solution to prompt at this point
+# to install the lib in the way the user is accustomed to
+=begin
+begin
+  require 'yanapi'
+rescue LoadError
+  require 'rubygems'
+  require 'yanapi'
+end
+=end
+require 'yanapi'
+
+# :title: YANSER, Yahoo! ANSwers harvestER
+# :main: Yanser
+# Main processing class.
+# Yanser encapsulates the main routine and instantiates
+# all other classes.
+#--
+# Yanser takes the users input and validates it.
+# It decides which search method to choose.
+# Then it collects all parameters and useful default values,
+# creates an XyzQuery with the starting point of 0 and gets the first results.
+# If more results were requested, Yanser creates a similar XyzQuery and gets
+# the next result set until the result limitation set by the user is met.
+# 
+module Yanser
+
+  class Yanser
+
+    # Yahoo! Answers returns starting not more than at the 1000st question.
+    # It makes no sense to step over.
+    START_LIMIT = 1000
+    
+    # We query the web service every two seconds.
+    QUERY_INTERVAL = 2
+    
+    # Yahoo! Answers returns maximum 50 results.
+    MAX_RESULTS = 50
+    
+    def initialize(opts)
+      
+      # the minimal output of an OptionParser
+      # {:query_type=>u|q|c|w, :query_params=>{appid, start, results, query}}
+      # opts come from the OptionParser
+      # they are supposed to be correct, no validation here
+      @options = opts
+      
+      @options[:limit] = @options[:limit] || START_LIMIT + MAX_RESULTS
+      
+      @query_params = @options[:query_params]
+      
+      @output_type = @query_params[:output] || 'xml'
+      
+      @query_params[:start] = @query_params[:start] || 0
+    end
+    
+    # TODO: implement some logging
+    def start
+      
+      if @options[:query_type] == 'QuestionQuery'
+        query(@query_params)
+      elsif @options[:limit] < MAX_RESULTS
+        @query_params[:results] = @options[:limit]
+        query(@query_params)
+      else
+        @query_params[:results] = MAX_RESULTS
+        
+        while query(@query_params) do
+          # we get the next start point here
+          @query_params[:start] += MAX_RESULTS
+          
+          results_left = @options[:limit] - @query_params[:start]
+          if results_left == 0
+            break
+          elsif results_left < MAX_RESULTS
+            @query_params[:results] = results_left
+            break
+          end
+        end # while
+      
+      end # if
+    
+    end # start
+    
+    private
+    
+    def query(params)
+      q = create_query(params)
+      tries = 0
+      begin
+        tries += 1
+        result = q.get
+        output(result)
+        sleep(2)
+      rescue YANAPI::EmptyResponse => e
+        $sderr.puts e
+        return false # do not iterate futher
+      rescue => e # some errors to retry
+        if (tries < 4)
+          sleep(QUERY_INTERVAL**tries)
+          retry
+        else
+        $stderr.puts e
+        return false # do not iterate futher
+      end
+    end
+    return true # we may iterate futher
+  end
+    
+    def create_query(params)
+      eval("YANAPI::#{@options[:query_type]}.new(params)")
+    end
+    
+    def output(result)
+      if @options[:output_dir]
+        save(result)
+      else
+        puts result
+      end
+    end
+    
+    # save results to a dir
+    # this dir exists since has been proved by OptionParser
+    # not a good implementation
+    # interface (filename, data)
+    def save(result)
+      filename = File.join(@options[:output_dir], "#{@query_params[:start]}.#{@output_type}")
+      file = File.new(filename, 'w')
+      file.puts result
+      file.close
+    end
+    
+  end # class Yanser
+end # module Yanser

data/test/test_opt_parser.rb

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+require 'test/unit'
+require 'yanser/opt_parser'
+include Yanser
+class TestOptionParser < Test::Unit::TestCase
+  def setup
+    @input = ['--appid', 'YahooDemo', '-r',
+              'de', '-l', '10', '-o', '/tmp',
+              '-k' 'Haus AND grün']
+    @output = {
+      :query_params => {
+        :appid => 'YahooDemo',
+        :query => 'Haus AND grün',
+        :region => 'de',
+        :search_in => 'question'
+      },
+      :query_type => 'TermQuery',
+      :output_dir => '/tmp',
+      :limit => 10
+    }
+
+    @cmd_args = ['--appid', 'YahooDemo']
+    @minimal_input = ['--appid', 'YahooDemo',
+                      '-k', 'Haus']
+  end
+
+  def teardown
+  end
+
+  # it should have a public class method 'parse'
+  def test_public_methods
+    assert_respond_to(Yanser::OptParser, :parse)
+  end
+
+  # it should return a non empty parameter hash
+  def test_parse_method
+    return_value = Yanser::OptParser.parse(@cmd_args)
+    assert(return_value.instance_of? Hash)
+    assert(! return_value.empty?)
+  end
+
+  # it should return a right complex output
+  def test_output
+    cmd_args = ['--appid', 'YahooDemo', '-r',
+                 'de', '-l', '10', '-o', '/tmp',
+                 '-k' 'Haus AND grün'
+                ]
+    assert_equal(@output, Yanser::OptParser.parse(cmd_args))
+  end
+
+
+  # it should set up the right method
+  def test_method
+    
+    cmd_args = ['-k', 'Haus'] | @cmd_args
+    assert_equal('TermQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+
+    cmd_args = ['-k', 'Haus', '-c', '09876543'] | @cmd_args
+    assert_equal('TermQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+    
+    cmd_args = ['-c', '09876543'] | @cmd_args
+    assert_equal('CategoryQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+
+    cmd_args = ['-c', 'Ausgehen'] | @cmd_args
+    assert_equal('CategoryQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+    
+    cmd_args = ['--user-id', '12345'] | @cmd_args
+    assert_equal('UserQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+
+    cmd_args = ['--question-id', '12345'] | @cmd_args
+    assert_equal('QuestionQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+  end
+
+  # it should set either :category_id or :category_name
+  def test_category_identifier
+    args = ['-k', 'Haus', '-c', '123'] | @cmd_args
+    opts = Yanser::OptParser.parse(args)
+    assert_equal('123', opts[:query_params][:category_id])
+    assert(! opts[:query_params].has_key?(:category_name))
+    
+    args = ['-k', 'Haus', '-c', 'Haushalt'] | @cmd_args
+    opts = Yanser::OptParser.parse(args)
+    assert_equal('Haushalt', opts[:query_params][:category_name])
+    assert(! opts[:query_params].has_key?(:category_id))
+    
+  end
+  
+  def test_output_dir
+  end
+
+  def test_query
+  end
+
+  # it should reject the empty input
+  def test_empty_input
+    assert_raises(OptParserError) {OptParser.parse []}
+  end
+
+  # it should accept only valid regions
+  def test_region_validity
+    regions = ['de', 'us', 'uk', 'ca', 'au', 'in', 'es',
+                'br', 'ar', 'mx', 'e1', 'it', 'fr', 'sg']
+    regions.each do |r|
+      input = @minimal_input | ['-r', r]
+      assert_nothing_raised(Yanser::OptParserError) {Yanser::OptParser.parse(input)}
+    end
+    input = @minimal_input | ['-r', 'abc']
+    assert_raises(Yanser::OptParserError) {Yanser::OptParser.parse(input)}
+  end
+
+  # it should set reasonable defaults
+  def test_default_values
+  end
+
+  # it should accept only valid query type combinations
+  def test_qtype_combinations
+  end
+
+  # it should require mandatory cmd arguments
+  def test_mandatory_args
+    args = ['-k', 'Haus']
+    assert_raises(OptParserError) {OptParser.parse args}
+  end
+
+  # it should successfully exit with some options
+  def test_successful_exit
+    orig_stderr = $stderr.clone
+    orig_stdout = $stdout.clone
+    $stderr.reopen(File.new('/dev/null', 'w'))
+    $stdout.reopen(File.new('/dev/null', 'w'))    
+    success_args = ['-h', '--help', '-v', '--version']
+    success_args.each do |arg|
+      assert_raises(SystemExit) {OptParser.parse(arg.to_a)}
+    end
+    $stderr.reopen(orig_stderr)
+    $stdout.reopen(orig_stdout) 
+  end
+end