yanser 0.0.3 → 0.1.0

data/.yardopts

@@ -0,0 +1,6 @@
+--private
+--protected
+--title 'YANSER - Yahoo! ANSwers harvestER'
+-
+CHANGELOG
+LICENSE

data/CHANGELOG

@@ -0,0 +1,24 @@
+=== COMPLETED
+==== 0.1.0
+Yanser supportst YANAPI 0.3.1 and further now.
+==== 0.0.3
+Yanser depends on the old interface of YANAPI. No functional improvements.
+==== 0.0.2
+Small changes in the documentation.
+==== 0.0.1
+Initial release of Yanser.
+
+
+=== PLANNED
+
+==== 0.0.2
+==== 0.0.3
+==== 0.4.0
+==== 0.5.0
+==== 0.6.0
+==== 0.7.0
+==== 0.8.0
+==== 0.9.0
+==== 1.0.0
+
+

data/README

@@ -2,20 +2,36 @@
 
 * {RubyGems}[http://rubygems.org/gems/yanser]
 * Developers {Homepage}[http://www.uni-trier.de/index.php?id=24140]
-* {YANAPI Project Page}[http://yanser.rubyforge.org/]
+* {YANSER Project Page}[http://yanser.rubyforge.org/]
 
 == DESCRIPTION
 
-YANSER (Yahoo! ANSwers harvestER) is a convinient search tool providing access to the Yahoo! Answers Q&A collection. Based on YANAPI it provides a simple CLI and helps to search for Questions and Answers which contain a set of key words, belong to a specific semantic domain or are posted by a certain user. Yanser is a research tool in the field of Computational Linguistics.
+YANSER (Yahoo! ANSwers harvestER) is a convinient search tool providing access
+to the Yahoo! Answers Q&A collection. Based on YANAPI it provides a simple CLI
+and helps to search for Questions and Answers which contain a set of key words,
+belong to a specific semantic domain or are posted by a certain user.
+
+Yanser is a research tool in the field of Computational Linguistics.
 
 == SYNOPSIS
   $ yanser --help
 
+== FEATURE LIST
+
+== INSTALLATION
+
+== USAGE
+
+== CHANGELOG
+
+See CHANGELOG.
+
+== CONTRIBUTORS
 
 
 == LICENSE
 
 YANSER is a copyrighted software by Andrei Beliankou, 2011.
+
 You may use, redistribute and change it under the terms
 provided in the LICENSE file.
-

data/bin/yanser

@@ -1,19 +1,36 @@
 # -*- coding: utf-8; mode: ruby -*-
-
-# это неверное решение, на самом деле я не должен напрямую указывать,
-# где находятся файлы, но пока пусть будет так
-# it is required only for .tar.gz distribution
-#lib_path = File.expand_path('../../lib', __FILE__)
-#$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
-
-# подобный способ является правильным, только так классы Yanser и OptionParser будут независимы
 require 'yanser/yanser'
 require 'yanser/opt_parser'
+require 'yanser/store'
 
-# class method, why not?
-options = Yanser::OptParser.parse(ARGV)
-
+begin
+  options = Yanser::OptParser.parse(ARGV)
+rescue
+  raise
+end
 
-yanser = Yanser::Yanser.new(options)
+# Handle here exceptions from YANAPI, use messages for it.
+# Add "See <yanser --help>." to it.
+begin
+  yanser = Yanser::Yanser.new(options)
+  result = yanser.start
+rescue YANAPI::UserError
+  # Give up with an error message.
+  # The user must change the input.
+  raise
+rescue YANAPI::ContentError
+  # Yahoo! says somethig.
+  raise
+rescue YANAPI::ExternalError
+  # Give up with an error message.
+  # It is a persistent external error.
+  # We tried three times in Yanser, but it seems to go kept.
+  raise
+end
 
-yanser.start
+begin
+  storage = Yanser::Store.new(options)
+  storage.store(result)
+rescue
+  raise
+end

data/lib/yanser/error.rb

@@ -1,11 +1,12 @@
-# -*- conding: utf-8 -*-
+# -*- coding: utf-8 -*-
 
 module Yanser
-  class OptParserError < RuntimeError
+  class Error < RuntimeError; end
+  class OptParserError < Error
     # empty for now
   end
 
-  class YanserError < RuntimeError
+  class YanserError < Error
     # empty for now
   end
 end

data/lib/yanser/opt_parser.rb

@@ -7,41 +7,54 @@ require 'yanser/error'
 
 module Yanser
   class OptParser
-    # OP expects cmd_args to be an array like ARGV
-    # dummy output for temporary usage
+
+    # Different possible values accepted by Yahoo!.
+    REGIONS = ['de', 'us', 'uk', 'ca', 'au', 'in', 'es',
+               'br', 'ar', 'mx', 'e1', 'it', 'fr', 'sg']
+    TYPES = ['all', 'resolved', 'open', 'undecided']
+    SORT_TYPES = ['relevance', 'date_desc', 'date_asc']
+    DATE_RANGES = ['all', '7', '7-30', '30-60', '60-90', 'more90']
+    SEARCH_LOCATIONS = ['all', 'question', 'best_answer']
+    OUTPUT_TYPES = ['xml', 'json', 'php', 'rss']
+    FILTERS = ['question', 'best_answer']
+    
+    # Main class method.
+    # OP expects cmd_args to be an array like ARGV.
     def self.parse(cmd_args)
       @@options = {}
       @@options[:query_params] = {}
       
       parser = create_parser
       
-      # if no options provided print the help
+      # If no options provided print the help.
       if cmd_args.empty?
-        msg = "You have to provide some options. " +
-          "yanser --help provides more information."
-        raise OptParserError, msg
-#        $stderr.printf "You have to provide some options.\n\n"
-#        puts parser
-#        exit 1
+        puts('You have to provide some options.',
+             'Please start with <yanser --help>.')
+        exit(1)
       end
       
       # Parse ARGV and provide the options hash.
       # Check if everything is correct and handle exceptions
       begin
-        parser.parse!(cmd_args)
-        # rescue all exceptions from OptionParser  
-      rescue => e
-        raise OptParserError, e.message
-#        $stderr.printf "#{e.message.capitalize}\n\n"
-#        puts parser
-#        exit 1
+        parser.parse(cmd_args)
+      rescue OptionParser::InvalidArgument => e
+        arg = e.message.split.last
+        puts "The provided argument #{arg} is currently not supported by Yahoo!"
+        puts 'Please colsult <yanser --help>.'
+        exit(1)
+      rescue OptionParser::InvalidOption => e
+        puts "You have provided an #{e.message}."
+        puts 'Please colsult <yanser --help>.'
+        exit(1)
+      rescue
+        raise
       end
       
       # Check to see if we got the required arguments needed.
-      check_required_options(@@options)
+#      check_required_options(@@options)
       
       # Set the search method.
-      @@options[:query_type] = set_query_type(@@options[:query_params])
+      @@options[:method] = set_query_type(@@options[:query_params])
       
       # Set some defaults such as :region and :search_in
       @@options = set_defaults(@@options)
@@ -50,34 +63,23 @@ module Yanser
     end # parse
     
     private
-    
-    # Check if the value of given region is correct.
-    # Now 14 regions are supported by Yahoo! Answers.
-    def self.prove_region(region)
-      regions = ['de', 'us', 'uk', 'ca', 'au', 'in', 'es',
-                 'br', 'ar', 'mx', 'e1', 'it', 'fr', 'sg']
-      
-      unless regions.include?(region)
-        msg = "The provided search region #{region} is currently not supported by Yahoo!"
-        raise OptParserError, msg
-      end
-    end
-    
+
     # define one of the following query types: TermQuery|CategoryQuery|
     # QuestionQuery|UserQuery
     def self.set_query_type(params)
       case
       when (params[:category_id] || params[:category_name]) && ! params[:query]
-        'CategoryQuery'
+        'getByCategory'
       when params[:query]
-        'TermQuery'
+        'questionSearch'
       when params[:user_id]
-        'UserQuery'
+        'getByUser'
       when params[:question_id]
-        'QuestionQuery'
+        'getQuestion'
       end
     end
-    
+
+=begin
     def self.check_required_options(options)
       required_opts = [:appid]
       
@@ -85,31 +87,19 @@ module Yanser
         if options[:query_params].has_key?(opt)
           next
         else
-          msg = "A required option --#{opt} is missing."
-          raise OptParserError, msg
+          puts "The required option --#{opt} is missing."
+          puts 'Please colsult <yanser --help>.'
+          exit(1)
         end
       end
     end
-    
-    def self.provide_dir(dir)
-      dir = File.expand_path(dir)
-      #check for existens
-      if File.directory?(dir)
-        if File.writable?(dir)
-          return dir
-        else
-          $stderr.puts 'The directory you have provided is not writable!'
-          exit 1
-        end
-      else
-        FileUtils.mkdir_p(dir)
-        return dir
-      end
-    end # provide_dir
+=end    
+
     
     def self.set_defaults(opts)
       # fake method for now
-      return opts
+      
+      opts
     end # set_defaults
 
     def self.create_parser
@@ -133,7 +123,9 @@ module Yanser
         opts.separator '  Mandatory search arguments:'
         
         opts.on('-k', '--key-word KEYWORD',
-                'Provide a single keyword or a boolean expression.'
+                'Provide a single keyword or a boolean expression.',
+                'You might combine this option with an explicit',
+                '<category> name or id to restrict your search.'
                 ) do |keyword|
           @@options[:query_params][:query] = keyword
           # not a solution!!!
@@ -141,10 +133,12 @@ module Yanser
           
         end
         
-        opts.separator ''
+        opts.separator '    and/or'
         
         opts.on('-c', '--category CATEGORY',
-                'Provide a category name or ID.'
+                'Provide a category name or ID. This is useful',
+                'while searching for categories or in the combination',
+                'with some key words.'
                 ) do |category|
           
           if category =~ /^[[:digit:]]+$/
@@ -155,7 +149,7 @@ module Yanser
           
         end
         
-        opts.separator ''
+        opts.separator '    or'
         
         opts.on('--user-id ID',
                 'Provide an user ID of questions you search for.',
@@ -164,7 +158,7 @@ module Yanser
           @@options[:query_params][:user_id] = user_id
         end
         
-        opts.separator ''
+        opts.separator '    or'
         
         opts.on('--question-id ID',
                 'Provide a question ID of the question you search for.',
@@ -172,32 +166,33 @@ module Yanser
                 ) do |question_id|
           @@options[:query_params][:question_id] = question_id
         end
-        
+
+        #################
         opts.separator ''
         opts.separator '  Optional search arguments:'
-        
-        opts.on('-r', '--region REGION',
-                'Provide a geographic region to search in for terms.',
-                'Possible values are: de, us, uk, ca, au, in, es, br,',
-                '                     ar, mx, e1, it, fr, sg.',
-                'This defaults to en.'
-                ) do |region|
-          # todo
-          prove_region(region)
-          @@options[:query_params][:region] = region
+        opts.on('--date-range DATE', DATE_RANGES,
+                'Provide some date range for the search. It defaults to <all>.',
+                "Possible values are: <#{DATE_RANGES.join('>, <')}>."
+                ) do |date|
+          @@options[:query_params][:date_range] = date
         end
-        
+
         opts.separator ''
-        
-        opts.on('-o', '--output-dir DIR',
-                'Provide an output folder.',
-                'This directory will be created if it does not exist yet.'
-                ) do |output_dir|
-          @@options[:output_dir] = provide_dir(output_dir)
+        opts.on('-f', '--output-format FORMAT', OUTPUT_TYPES,
+                "Provide an output format: <#{OUTPUT_TYPES.join('>, <')}>.",
+                'It defaults to <xml>, you may simply ommit this option.'
+                ) do |f|
+          @@options[:query_params][:output] = f
         end
         
         opts.separator ''
+        opts.on('--filter TYPE', FILTERS,
+                "Possible values are: <#{FILTERS.join('>, <')}>."
+                ) do |type|
+          @@options[:query_params][:filter] = type
+        end
         
+        opts.separator ''
         opts.on('-l', '--limit NUMBER', Integer,
                 'Provide a number of answers you want to get from Yahoo.',
                 'This argument is not mandatory, if you want to get',
@@ -206,12 +201,12 @@ module Yanser
           @@options[:limit] = limit
         end
         
-        opts.separator ''
-        opts.on('-f', '--output-format FORMAT',
-                'Provide an output format: xml, json, rss, php.',
-                'It defaults to xml, and you can simply ommit this option.'
-                ) do |f|
-          raise NotImplementedError, 'Only default output format is implemented'
+        opts.separator ''        
+        opts.on('-o', '--output-dir DIR',
+                'Provide an output folder.',
+                'This directory will be created if it does not exist yet.'
+                ) do |output_dir|
+          @@options[:output_dir] = output_dir
         end
         
         opts.separator ''
@@ -222,23 +217,92 @@ module Yanser
                 'It can be useful if you want to put many query results',
                 'in the same output folder.'
                 ) do |pref|
-          raise NotImplementedError, 'No prefixes implemeted.'
+          @@options[:prefix] = pref
         end
 
         opts.separator ''
-        opts.on('--key-word-list FILE',
-                'Provide a file with key words or phrases (boolean syntax',
-                'allowed) one search token per line.'
-                ) do |file_name|
-          raise NotImplementedError
+        opts.separator ''
+        opts.on('-r', '--region REGION', REGIONS,
+                'Provide a geographic region to search in for terms.',
+                'Possible values are: de, us, uk, ca, au, in, es, br,',
+                '                     ar, mx, e1, it, fr, sg.',
+                'This defaults to <en>.'
+                ) do |region|
+
+          @@options[:query_params][:region] = region
         end
 
         opts.separator ''
+        opts.on('--search-in PLACE', SEARCH_LOCATIONS,
+                "Possible values are: <#{SEARCH_LOCATIONS.join('>, <')}>."
+                ) do |place|
+          @@options[:query_params][:search_in] = place
+        end
+
+        opts.separator ''
+        opts.on('--start POSITION',
+                'Provide the position of the question in the search list',
+                'you want start with.'
+                ) do |pos|
+          @@options[:query_params][:start] = pos
+        end
+        
+        opts.separator ''
+        opts.on('--sort ORDER', SORT_TYPES,
+                "Possible values are: <#{SORT_TYPES.join('>, <')}>."
+                ) do |order|
+          @@options[:query_params][:sort] = order
+        end
+        
+        opts.separator ''
+        opts.on('--type TYPE', TYPES,
+                "Possible values are: <#{TYPES.join('>, <')}>."
+                ) do |type|
+          @@options[:query_params][:type] = type
+        end
+        
+
+        #################
+        opts.separator ""
+        opts.separator '  Experimental options. They may not be implemented yet.'
         opts.on('--exp-file FILE',
                 'Provide a formal description of the experiment.'
                 ) do |file_name|
-          raise NotImplementedError
+          warn 'Not yet implemented!'
+        end
+
+        opts.separator ''
+        opts.on('--key-word-list FILE',
+                'Provide a file with key words or phrases',
+                '(boolean syntax allowed), one search token per line.'
+                ) do |file_name|
+          warn 'Not yet implemented!'
+        end
+
+        opts.separator ''
+        opts.on('--log [LOGFILE]',
+                'SOME DESCRIPTION'
+                ) do |file|
+          warn 'Add description!'
+          warn 'Not yet implemented!'
+        end
+        
+        opts.separator ''
+        opts.on('--extract',
+                'SOME DESCRIPTION'
+                ) do
+        # some non xml representation in a human readable format
+          warn 'Add description!'
+          warn 'Not yet implemented!'
         end
+        opts.separator ''
+        opts.on('-t', '--time-interval SECONDS', Integer,
+                'SOME DESCRIPTION'
+                ) do |t|
+          @@options[:interval] = t
+          warn 'Add description!'
+        end
+
         opts.separator ""
         opts.separator "Common options:"
         
@@ -261,20 +325,24 @@ __END__
 # List of options planned for implementation.
 --appid
 -k, --key-word
--c, --category-id # Term & CategorySearch
+-c, --category # Term & CategorySearch
 --user-id
 --question-id # QuestionSearch
 -r, --region
 -f, --output-format
--l, --limit
 --date-range
 --sort
+--start
 --filter
+--callback # bad idea
+--type
+--search-in
 
 
+-l, --limit
 --prefix #prefix for output files
 --key-word-list
---exp-file
+--exp-file # experiment description, yaml
 -o, --output-dir
 -h, --help
 -v, --version
@@ -306,8 +374,9 @@ instance interface
     :type => "all" | "resolved" | "open" | "undecided", # default 'all'
     :user_id => '123456'
   },
-  :query_type => 'TermQuery'|'QuestionQuery'|'UserQuery'|'CategoryQuery',
+  :interval => Integer, # in seconds
+  :limit => Integer,
+  :method => 'TermQuery'|'QuestionQuery'|'UserQuery'|'CategoryQuery',
   :output_dir => 'some path',
-  :prefix => 'some prefix', # prefix for output files
-  :limit => Integer
+  :prefix => 'some prefix' # prefix for output files
 }

data/lib/yanser/store.rb

@@ -0,0 +1,48 @@
+require 'fileutils'
+
+module Yanser
+  class Store
+    def initialize(params)
+      @params = params
+    end
+    
+    def store(result)
+      if dir = @params[:output_dir]
+        provide_dir(dir)
+        save(result)
+      else
+        puts result
+      end
+    end
+
+    private
+
+    # rework!
+    def provide_dir(dir)
+      dir = File.expand_path(dir)
+      #check for existens
+      if File.directory?(dir)
+        unless File.writable?(dir)
+          warn 'The directory you have provided is not writable!'
+          exit(1)
+        end
+      else
+        FileUtils.mkdir_p(dir)
+      end
+    end
+
+    # save results to a dir
+    # not a good implementation
+    # interface (filename, data)
+    def save(result)
+      filename = File.join(@params[:output_dir],
+                           "yanser_output.xml")
+      File.open(filename, 'w') do |file|
+        file.puts result
+      end
+    end
+  end # Store
+end
+
+    
+

data/lib/yanser/version.rb

@@ -1,3 +1,3 @@
 module Yanser
-  VERSION = '0.0.3'
+  VERSION = '0.1.0'
 end

data/lib/yanser/yanser.rb

@@ -1,29 +1,10 @@
-# this solution is not ideal since I force people to use 'rubygems'
-# possible it would be a better solution to prompt at this point
-# to install the lib in the way the user is accustomed to
-=begin
-begin
-  require 'yanapi'
-rescue LoadError
-  require 'rubygems'
-  require 'yanapi'
-end
-=end
 require 'yanapi'
 
 # :title: YANSER, Yahoo! ANSwers harvestER
-# :main: Yanser
+# :main: README
+
 # Main processing class.
-# Yanser encapsulates the main routine and instantiates
-# all other classes.
-#--
-# Yanser takes the users input and validates it.
-# It decides which search method to choose.
-# Then it collects all parameters and useful default values,
-# creates an XyzQuery with the starting point of 0 and gets the first results.
-# If more results were requested, Yanser creates a similar XyzQuery and gets
-# the next result set until the result limitation set by the user is met.
-# 
+# Yanser encapsulates the main routine and instantiates all other classes.
 module Yanser
 
   class Yanser
@@ -38,98 +19,85 @@ module Yanser
     # Yahoo! Answers returns maximum 50 results.
     MAX_RESULTS = 50
     
-    def initialize(opts)
-      
-      # the minimal output of an OptionParser
-      # {:query_type=>u|q|c|w, :query_params=>{appid, start, results, query}}
-      # opts come from the OptionParser
-      # they are supposed to be correct, no validation here
-      @options = opts
-      
-      @options[:limit] = @options[:limit] || START_LIMIT + MAX_RESULTS
+    def initialize(params)
+      # Libs are not allowed to mutate the input.
+      @params = params.clone
       
-      @query_params = @options[:query_params]
+      @params[:limit] = @params[:limit] || START_LIMIT + MAX_RESULTS
       
-      @output_type = @query_params[:output] || 'xml'
+      @query_params = @params[:query_params]
       
+      @output = @query_params[:output] || 'xml'
+
+      # Yahoo counts results beginning with 0.
+      # <:results=50> means all questins from 0 to 49.
       @query_params[:start] = @query_params[:start] || 0
     end
     
-    # TODO: implement some logging
+    # It returns the accumulated string or <nil>, if nothing found.
     def start
-      
-      if @options[:query_type] == 'QuestionQuery'
-        query(@query_params)
-      elsif @options[:limit] < MAX_RESULTS
-        @query_params[:results] = @options[:limit]
-        query(@query_params)
+      if @params[:method] == 'getQuestion'
+        response = get_response(@params)
       else
-        @query_params[:results] = MAX_RESULTS
+        response = ''
         
-        while query(@query_params) do
-          # we get the next start point here
-          @query_params[:start] += MAX_RESULTS
+        while @query_params[:start] < @params[:limit]
+          results_left = @params[:limit] - @query_params[:start]
           
-          results_left = @options[:limit] - @query_params[:start]
-          if results_left == 0
-            break
-          elsif results_left < MAX_RESULTS
+          if results_left < MAX_RESULTS
             @query_params[:results] = results_left
+          else
+            @query_params[:results] = MAX_RESULTS
+          end
+          
+          r = get_response(@params)
+          if r
+            response << r
+          else
+            if response.empty?
+              response = nil
+            end
             break
           end
-        end # while
-      
-      end # if
-    
+          @query_params[:start] += @query_params[:results]
+          sleep(QUERY_INTERVAL)
+        end
+      end
+
+      response
     end # start
     
     private
-    
-    def query(params)
-      q = create_query(params)
-      tries = 0
+    # It returns a result string or <nil>.
+    # It tries three times in case of YANAPI::ExternalError,
+    # it gives up if an YANAPI::ContentError occures (returns <nil>).
+    def get_response(params)
+      query = create_query(params)
+      t = 0
       begin
-        tries += 1
-        result = q.get
-        output(result)
-        sleep(2)
-      rescue YANAPI::EmptyResponse => e
-        $sderr.puts e
-        return false # do not iterate futher
-      rescue => e # some errors to retry
-        if (tries < 4)
-          sleep(QUERY_INTERVAL**tries)
+        t += 1
+        response = query.get
+      rescue YANAPI::ExternalError => e
+        warn "External error: #{e}!"
+        if t <= 3
+          sleep(QUERY_INTERVAL**t)
           retry
         else
-        $stderr.puts e
-        return false # do not iterate futher
+          raise
+        end
       end
+      
+      response
     end
-    return true # we may iterate futher
-  end
     
+    # It creates a new api object or raises an exception.
+    # YANAPI::UserError should not be rescued, it indicates that the user
+    # must correct the input hash.
     def create_query(params)
-      eval("YANAPI::#{@options[:query_type]}.new(params)")
-    end
-    
-    def output(result)
-      if @options[:output_dir]
-        save(result)
-      else
-        puts result
-      end
-    end
-    
-    # save results to a dir
-    # this dir exists since has been proved by OptionParser
-    # not a good implementation
-    # interface (filename, data)
-    def save(result)
-      filename = File.join(@options[:output_dir], "#{@query_params[:start]}.#{@output_type}")
-      file = File.new(filename, 'w')
-      file.puts result
-      file.close
+      YANAPI::API.new(params)
     end
     
   end # class Yanser
 end # module Yanser
+
+__END__

data/test/test_opt_parser.rb

@@ -1,138 +1,264 @@
 # -*- coding: utf-8 -*-
 require 'test/unit'
 require 'yanser/opt_parser'
+require 'stringio' # for helper methods
+
 include Yanser
+
 class TestOptionParser < Test::Unit::TestCase
-  def setup
-    @input = ['--appid', 'YahooDemo', '-r',
-              'de', '-l', '10', '-o', '/tmp',
-              '-k' 'Haus AND grün']
-    @output = {
-      :query_params => {
-        :appid => 'YahooDemo',
-        :query => 'Haus AND grün',
-        :region => 'de',
-        :search_in => 'question'
-      },
-      :query_type => 'TermQuery',
-      :output_dir => '/tmp',
-      :limit => 10
-    }
 
+  def setup
     @cmd_args = ['--appid', 'YahooDemo']
-    @minimal_input = ['--appid', 'YahooDemo',
-                      '-k', 'Haus']
+    @minimal_input = ['--appid', 'YahooDemo', '-k', 'Haus']
+    @valid_opts = ['--appid', 'MyID',
+                  '--category', '12345',
+                  '--date-range', 'all',
+                  '--exp-file', 'exp_file.yml',
+                  '--filter', 'question',
+                  '--help',
+                  '--key-word', 'Haus',
+                  '--key-word-list', 'some_file.txt',
+                  '--limit', '1',
+                  '--output-dir', '/tmp',
+                  '--output-format', 'xml',
+                  '--prefix', 'file1',
+                  '--question-id', '12345',
+                  '--region', 'de',
+                  '--search-in', 'question',
+                  '--sort', 'relevance',
+                  '--start', '10',
+                  '--time-interval', '10',
+                  '--type', 'resolved',
+                  '--user-id', '12345',
+                  '--version'
+                 ]
   end
 
   def teardown
   end
 
-  # it should have a public class method 'parse'
+  # It should have a public class method 'parse'.
   def test_public_methods
     assert_respond_to(Yanser::OptParser, :parse)
   end
 
-  # it should return a non empty parameter hash
+  # It should return a non empty parameter hash.
   def test_parse_method
-    return_value = Yanser::OptParser.parse(@cmd_args)
-    assert(return_value.instance_of? Hash)
-    assert(! return_value.empty?)
+    return_value = Yanser::OptParser.parse(@minimal_input)
+    assert(return_value.instance_of?(Hash))
+    assert_equal(false, return_value.empty?)
   end
 
-  # it should return a right complex output
-  def test_output
-    cmd_args = ['--appid', 'YahooDemo', '-r',
-                 'de', '-l', '10', '-o', '/tmp',
-                 '-k' 'Haus AND grün'
-                ]
-    assert_equal(@output, Yanser::OptParser.parse(cmd_args))
+  # It should reject the empty input and exit.
+  def test_empty_input
+    out, err = intercept_output do
+      assert_raises(SystemExit) { OptParser.parse([]) }
+    end
+    assert_match(/You have to provide some options./, out)
   end
 
+  # It should accept correct options.
+  # Invalid options is the matter of OptionParser itself,
+  # do not test it here.
+  # We test only, that OP exits and does not raise an exception.
+  def test_accept_correct_options
+    # this options we should treat separately
+    @valid_opts.delete('--help')
+    @valid_opts.delete('--version')
+    assert_nothing_raised { OptParser.parse(@valid_opts) }
+
+    stdout, stderr = intercept_output do 
+      assert_raises(SystemExit) { OptParser.parse(['--invalid-option']) }
+    end
+
+    assert_match(/You have provided an invalid option:/, stdout)
+  end
+
+  # It should successfully exit with some options.
+  def test_successful_exit
+    quietly do
+      success_args = ['-h', '--help', '-v', '--version']
+      success_args.each do |arg|
+        assert_raises(SystemExit) { OptParser.parse(arg.split) }
+      end
+    end
+  end
+
+=begin
+  # It is done in YANAPI.   
+  # It should require mandatory cmd arguments.
+  def test_mandatory_args
+    flunk('Rewrite!')
+    args = ['-k', 'Haus']
+    assert_raises(OptParserError) {OptParser.parse args}
+  end
+=end
+
+  # It should accept only valid date arguments.
+  def test_date_arguments
+    validate(OptParser::DATE_RANGES, '--date-range', :date_range)
+  end
+
+  # It should accept a file and parse it.
+  def test_exp_file
+    flunk('Not implemented! It should accept a file and parse it.')
+  end
+
+  # It should accept only valid filter arguments.
+  def test_filter_arguments
+    validate(OptParser::FILTERS, '--filter', :filter)
+  end
+
+  # It should accept only numeric parameters for limits.
+  def test_limit_arguments
+    flunk('Not implemented! It should accept only numeric paramters for limits.')
+  end
 
-  # it should set up the right method
-  def test_method
+  # It should accept only valid output formats.
+  def test_output_arguments
+    validate(OptParser::OUTPUT_TYPES, '--output-format', :output)
+  end
+
+  # It should accept only valid regions.
+  def test_region_arguments
+    validate(OptParser::REGIONS, '-r', :region)
+  end
+  
+  # It should accept only valid search locations.
+  def test_search_locations
+    validate(OptParser::SEARCH_LOCATIONS, '--search-in', :search_in)
+  end
+
+  # It should accept only valid sort types.
+  def test_sort_types
+    validate(OptParser::SORT_TYPES, '--sort', :sort)
+  end
+
+  # It should accept only valid types.
+  def test_types
+    validate(OptParser::TYPES, '--type', :type)
+  end
+
+  # It should set up the right search method.
+  def test_correct_setting_of_the_method
     
     cmd_args = ['-k', 'Haus'] | @cmd_args
-    assert_equal('TermQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+    assert_equal('questionSearch', OptParser.parse(cmd_args)[:method])
 
     cmd_args = ['-k', 'Haus', '-c', '09876543'] | @cmd_args
-    assert_equal('TermQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+    assert_equal('questionSearch', OptParser.parse(cmd_args)[:method])
     
     cmd_args = ['-c', '09876543'] | @cmd_args
-    assert_equal('CategoryQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+    assert_equal('getByCategory', OptParser.parse(cmd_args)[:method])
 
     cmd_args = ['-c', 'Ausgehen'] | @cmd_args
-    assert_equal('CategoryQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+    assert_equal('getByCategory', OptParser.parse(cmd_args)[:method])
     
     cmd_args = ['--user-id', '12345'] | @cmd_args
-    assert_equal('UserQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+    assert_equal('getByUser', OptParser.parse(cmd_args)[:method])
 
     cmd_args = ['--question-id', '12345'] | @cmd_args
-    assert_equal('QuestionQuery', Yanser::OptParser.parse(cmd_args)[:query_type])
+    assert_equal('getQuestion', OptParser.parse(cmd_args)[:method])
   end
 
-  # it should set either :category_id or :category_name
+  # It should set either :category_id or :category_name.
   def test_category_identifier
     args = ['-k', 'Haus', '-c', '123'] | @cmd_args
     opts = Yanser::OptParser.parse(args)
     assert_equal('123', opts[:query_params][:category_id])
-    assert(! opts[:query_params].has_key?(:category_name))
+    assert_equal(false, opts[:query_params].has_key?(:category_name))
     
     args = ['-k', 'Haus', '-c', 'Haushalt'] | @cmd_args
     opts = Yanser::OptParser.parse(args)
     assert_equal('Haushalt', opts[:query_params][:category_name])
-    assert(! opts[:query_params].has_key?(:category_id))
-    
+    assert_equal(false, opts[:query_params].has_key?(:category_id))
   end
   
-  def test_output_dir
+  # It should set reasonable defaults.
+  # Yanser may set defaults, Yanapi provides minimal output, i.e. if Yahoo
+  # changes defaults, every user can be affected. That's whe we offer some
+  # reasonable defaults for our experiments.
+  # :region => 'de'
+  # :data_range => 'all'
+  def test_default_values
+    flunk('Not implemented! It should set reasonable defaults.')
+    
   end
 
-  def test_query
+  # It should return a right complex output.
+  def test_output
+    cmd_args = ['--appid', 'YahooDemo', '-r',
+                 'de', '-l', '10', '-o', '/tmp',
+                 '-k' 'Haus AND grün'
+                ]
+    output = {
+      :query_params => {
+        :appid => 'YahooDemo',
+        :query => 'Haus AND grün',
+        :region => 'de',
+        :search_in => 'question'
+      },
+      :method => 'questionSearch',
+      :output_dir => '/tmp',
+      :limit => 10
+    }
+    assert_equal(output, OptParser.parse(cmd_args))
   end
 
-  # it should reject the empty input
-  def test_empty_input
-    assert_raises(OptParserError) {OptParser.parse []}
+  # It should set correct parameters for logging.
+  def test_log_arguments
+    flunk('Not implemented. It should set correct parameters for logging.')
   end
 
-  # it should accept only valid regions
-  def test_region_validity
-    regions = ['de', 'us', 'uk', 'ca', 'au', 'in', 'es',
-                'br', 'ar', 'mx', 'e1', 'it', 'fr', 'sg']
-    regions.each do |r|
-      input = @minimal_input | ['-r', r]
-      assert_nothing_raised(Yanser::OptParserError) {Yanser::OptParser.parse(input)}
+  # It should set correct parameters for data extraction.
+  def test_extraction_arguments
+    flunk('Not implemented. It should set correct parameters for data extraction.')
+  end
+################################################################################
+  # It is a helper method, many testable units provide some verbose output
+  # to stderr and/or stdout. It is usefull to suppress any kind of verbosity.
+  def quietly(&b)
+    begin
+      orig_stderr = $stderr.clone
+      orig_stdout = $stdout.clone
+      $stderr.reopen(File.new('/dev/null', 'w'))
+      $stdout.reopen(File.new('/dev/null', 'w'))
+      b.call
+    ensure
+      $stderr.reopen(orig_stderr)
+      $stdout.reopen(orig_stdout)
     end
-    input = @minimal_input | ['-r', 'abc']
-    assert_raises(Yanser::OptParserError) {Yanser::OptParser.parse(input)}
   end
 
-  # it should set reasonable defaults
-  def test_default_values
+  # It is a helper method for validation of used parameter values.
+  def validate(params, option, key)
+    params.each do |p|
+      input = @minimal_input | [option, p]
+      params = {}
+      assert_nothing_raised(SystemExit) { params = OptParser.parse(input) }
+      assert_equal(p, params[:query_params][key])
+    end
+    
+    input = @minimal_input | [option, 'invalid']
+    stdout, stderr = intercept_output do
+      assert_raises(SystemExit) { OptParser.parse(input) }
+    end
+    assert_match(/The provided .+ is currently not supported by Yahoo!/,
+                 stdout)
   end
 
-  # it should accept only valid query type combinations
-  def test_qtype_combinations
-  end
+  # It is a helper method for handling stdout and stderr as strings.
+  def intercept_output
+    orig_stdout = $stdout
+    orig_stderr = $stderr
+    $stdout = StringIO.new
+    $stderr = StringIO.new
 
-  # it should require mandatory cmd arguments
-  def test_mandatory_args
-    args = ['-k', 'Haus']
-    assert_raises(OptParserError) {OptParser.parse args}
-  end
+    yield
 
-  # it should successfully exit with some options
-  def test_successful_exit
-    orig_stderr = $stderr.clone
-    orig_stdout = $stdout.clone
-    $stderr.reopen(File.new('/dev/null', 'w'))
-    $stdout.reopen(File.new('/dev/null', 'w'))    
-    success_args = ['-h', '--help', '-v', '--version']
-    success_args.each do |arg|
-      assert_raises(SystemExit) {OptParser.parse(arg.to_a)}
-    end
-    $stderr.reopen(orig_stderr)
-    $stdout.reopen(orig_stdout) 
+    return $stdout.string, $stderr.string
+  ensure
+    $stdout = orig_stdout
+    $stderr = orig_stderr
   end
 end

data/test/test_yanser.rb

@@ -10,6 +10,7 @@ class TestYanser < Test::Unit::TestCase
   def teardown
   end
   
+  # It should have a public method <start>.
   def test_public_methods
     yanser = Yanser::Yanser.new(
                         :query_params => {
@@ -26,10 +27,15 @@ class TestYanser < Test::Unit::TestCase
     assert_respond_to(yanser, :start)
   end
 
+  # It should have a constant <VERSION>.
   def test_constants
     assert(Yanser::VERSION.instance_of?(String))
   end
-  def test_mandatory_options
+
+  # It should accept input according to the defined interface.
+  def test_input
+    flunk('Not implemented. It should accept input according to the defined interface.')
   end
+
   
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: yanser
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 27
   prerelease: 
   segments: 
   - 0
+  - 1
   - 0
-  - 3
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors: 
 - Andrei Beliankou
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-06 00:00:00 Z
+date: 2011-08-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: yanapi
@@ -23,17 +23,17 @@ dependencies:
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - "="
+    - - ">="
       - !ruby/object:Gem::Version 
-        hash: 25
+        hash: 17
         segments: 
         - 0
+        - 3
         - 1
-        - 1
-        version: 0.1.1
+        version: 0.3.1
   type: :runtime
   version_requirements: *id001
-description: YANSER (Yahoo! ANSwers harvestER) is a convinient search tool providing access to the Yahoo! Answers Q&A collection. Based on YANAPI it provides a simple CLI and helps to search for Questions and Answers which contain a set of key words, belong to a specific semantic domain or are posted by a certain user. Yanser is a research tool in the field of Computational Linguistics.
+description: YANSER (Yahoo! ANSwers harvestER) is a convinient search tool providing access to the Yahoo! Answers Q&A collection. Based on YANAPI it provides a simple CLI and helps to search for Questions and Answers which contain a set of key words, belong to a specific semantic domain or are posted by a certain user. Yanser is a research tool in the field of Computational Linguistics e.g for Question Answering.
 email: a.belenkow@uni-trier.de
 executables: 
 - yanser
@@ -42,25 +42,27 @@ extensions: []
 extra_rdoc_files: 
 - README
 - LICENSE
+- CHANGELOG
 files: 
-- lib/tester.rb
 - lib/yanser/error.rb
+- lib/yanser/store.rb
 - lib/yanser/yanser.rb
 - lib/yanser/opt_parser.rb
 - lib/yanser/version.rb
 - bin/yanser
-- LICENSE
-- Rakefile
 - README
+- LICENSE
+- CHANGELOG
 - test/test_opt_parser.rb
 - test/test_yanser.rb
+- .yardopts
 homepage: http://www.uni-trier.de/index.php?id=34451
 licenses: []
 
 post_install_message: 
 rdoc_options: 
 - -m
-- README.rdoc
+- README
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -86,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: yanser
-rubygems_version: 1.7.2
+rubygems_version: 1.8.7
 signing_key: 
 specification_version: 3
 summary: Yanser is a convinient search tool providing access to the Yahoo! Answers Q/A collection.

data/Rakefile

@@ -1,41 +0,0 @@
-# We need rake to user FileLists.
-require 'rake'
-# We can require 'rake/clean' to add 'clobber' and 'clean' tasks.
-require 'rake/clean'
-
-
-# I am not sure how to use this constant.
-SRC = FileList['**/*.rb']
-
-CLOBBER.include('doc', '**/*.html', '**/*.gem')
-
-# testing
-require 'rake/testtask'
-Rake::TestTask.new do |t|
-  t.test_files = FileList.new('test/**/*.rb').to_a
-  # rake starts an other ruby process with a new options set.
-  # ruby --some-option -S rake is not sufficient to propagate
-  # the option "--some-option".
-  t.ruby_opts = ['-rubygems']
-end
-
-# Build the gem package
-load 'yanser.gemspec'
-require 'rubygems/package_task'
-Gem::PackageTask.new(GEMSPEC).define
-
-# Generate documentation
-require 'rdoc/task'
-RDoc::Task.new do |rdoc|
-  rdoc.rdoc_files.include('README', 'LICENSE', 'lib/**/*')
-end
-
-desc "Open an irb session preloaded with this library."
-task :console do
-  sh "irb -rubygems -I lib -r yanser/opt_parser -r yanser/yanser"
-end
-
-desc 'Run the main executable file of the project.'
-task :start do
-  sh "ruby -w -rubygems -I lib bin/yanser #{ENV['cmd']}"
-end

data/lib/tester.rb

@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-
-require 'yanser'
-
-params = {
-  :query_method => 'TermQuery',
-  :query_params => {
-    :appid => 'YahooDemo',
-    :query => 'Köln',
-    :region => 'de',
-    :results => 5,
-    :start => 0
-  }
-}
-
-y = Yanser.new(params)
-
-y.start