yanser 0.0.1

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2011 Andrei Beliankou, University of Trier, Germany
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README

@@ -0,0 +1 @@
+new readme

data/README.rdoc

@@ -0,0 +1,21 @@
+= YANSER
+
+* {RubyGems}[http://rubygems.org/gems/yanser]
+* Developers {Homepage}[http://www.uni-trier.de/index.php?id=24140]
+* {YANAPI Project Page}[http://yanser.rubyforge.org/]
+
+== DESCRIPTION
+
+YANSER (Yahoo! ANSwers harvestER) is a convinient search tool providing access to the Yahoo! Answers Q&A collection. Based on YANAPI it provides a simple CLI and helps to search for Questions and Answers which contain a set of key words, belong to a specific semantic domain or are posted by a certain user. Yanser is a research tool in the field of Computational Linguistics.
+
+== SYNOPSIS
+  $ yanser --help
+
+
+
+== LICENSE
+
+YANSER is a copyrighted software by Andrei Beliankou, 2011.
+You may use, redistribute and change it under the terms
+provided in the LICENSE file.
+

data/Rakefile

@@ -0,0 +1,14 @@
+require 'rubygems'
+require 'rake'
+# we can require 'rake/clean' to add 'clobber' and 'clean' tasks
+require 'rake/clean'
+require 'rake/testtask'
+
+
+SRC = FileList['**/*.rb']
+
+CLOBBER.include('doc', '**/*.html', '**/*.gem')
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList.new('test/yanapi/*.rb')
+end

data/bin/yanser

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+# это неверное решение, на самом деле я не должен напрямую указывать,
+# где находятся файлы, но пока пусть будет так
+lib_path = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
+
+# подобный способ является правильным, только так классы Yanser и OptionParser будут независимы
+require 'yanser'
+require 'option_parser'
+
+# class method, why not?
+options = OptionParser.parse(ARGV)
+
+
+yanser = Yanser.new(options)
+
+yanser.start

data/lib/option_parser.rb

@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+require 'optparse'
+require 'fileutils'
+
+require 'version'
+
+class OptionParser
+  # OP expects cmd_args to be an array like ARGV
+  # dummy output for temporary usage
+  def self.parse(cmd_args)
+    options = {}
+    options[:query_params] = {}
+    
+    parser = OptionParser.new do |opts|
+      opts.banner = 'Usage: yanser OPTIONS'
+
+      opts.separator ''
+      opts.separator 'Program specific options:'
+      
+      opts.on('--appid APPID',
+              'Provide an ApplicationID given by Yahoo,',
+              'to test Yanser you can use <YahooDemo> as the APPID,',
+              'think in this case on limitations placed by Yahoo.',
+              'This option is required!'
+              ) do |appid|
+        options[:query_params][:appid] = appid
+      end
+
+
+      opts.separator ''
+      opts.separator '  Mandatory search arguments:'
+      
+      opts.on('-k', '--key-word KEYWORD',
+              'Provide a single keyword or a boolean expression.'
+              ) do |keyword|
+        options[:query_params][:query] = keyword
+        # not a solution!!!
+        options[:query_params][:search_in] = 'question'
+
+      end
+
+      opts.separator ''
+
+      opts.on('-c', '--category CATEGORY',
+              'Provide a category name or ID.'
+              ) do |category|
+
+        if category =~ /^[[:digit:]]+$/
+          options[:query_params][:category_id] = category
+        else
+          options[:query_params][:category_name] = category
+        end
+
+      end
+      
+      opts.separator ''
+
+      opts.on('--user-id ID',
+              'Provide an user ID of questions you search for.',
+              'This way you can get question by a specific user.'
+              ) do |user_id|
+        options[:query_params][:user_id] = user_id
+      end
+      
+      opts.separator ''
+
+      opts.on('--question-id ID',
+              'Provide a question ID of the question you search for.',
+              'It returns a unique question.'
+              ) do |question_id|
+        options[:query_params][:question_id] = question_id
+      end
+
+      opts.separator ''
+      opts.separator '  Optional search arguments:'
+      
+      opts.on('-r', '--region REGION',
+              'Provide a geographic region to search in for terms.',
+              'Possible values are: de, us, uk, ca, au, in, es, br,',
+              '                     ar, mx, e1, it, fr, sg.',
+              'This defaults to en.'
+              ) do |region|
+        # todo
+        prove_region(region)
+        options[:query_params][:region] = region
+      end
+
+      opts.separator ''
+      
+      opts.on('-o', '--output-dir DIR',
+              'Provide an output folder.',
+              'This directory will be created if it does not exist yet.'
+              ) do |output_dir|
+        options[:output_dir] = provide_dir(output_dir)
+      end
+      
+      opts.separator ''
+      
+      opts.on('-l', '--limit NUMBER', Integer,
+              'Provide a number of answers you want to get from Yahoo.',
+              'This argument is not mandatory, if you want to get',
+              'all answers simply ommit this argument.'
+              ) do |limit|
+        options[:limit] = limit
+      end
+
+      opts.separator ''
+      opts.on('-f', '--output-format FORMAT',
+              'Provide an output format: xml, json, rss, php.',
+              'It defaults to xml, and you can simply ommit this option.'
+              ) do |f|
+        raise NotImplementedError, 'Only default output format is implemented'
+      end
+      
+      opts.separator ''
+      opts.on('--prefix PREFIX',
+              'Provide a prefix for the output files. By default',
+              'the filename begins with the index of the retrieved',
+              'question. You can alter this by providing a prefix.',
+              'It can be useful if you want to put many query results',
+              'in the same output folder.'
+              ) do |pref|
+        raise NotImplementedError, 'No prefixes implemeted.'
+      end
+
+      opts.separator ""
+      opts.separator "Common options:"
+
+      opts.on_tail('-h', '--help', 'Show the help message.') do
+        puts opts
+        exit
+      end
+
+      opts.on_tail('-v', '--version', 'Show the program version.') do
+        puts YANSER::VERSION
+        exit
+      end
+    end
+    
+    # if no options provided print the help
+    if cmd_args.empty?
+      $stderr.printf "You have to provide some options.\n\n"
+      puts parser
+      exit 1
+    end
+
+    # Parse ARGV and provide the options hash.
+    # Check if everything is correct and handle exceptions
+    begin
+      parser.parse!(cmd_args)
+    # rescue all exceptions from OptionParser  
+    rescue => e
+      $stderr.printf "#{e.message.capitalize}\n\n"
+      puts parser
+      exit 1
+    end
+
+    # Check to see if we got the required arguments needed.
+    check_required_options(options)
+
+    # Set the search method.
+    options[:query_type] = set_query_type(options[:query_params])
+
+    return options
+  end # parse
+
+  private
+
+  # Check if the value of given region is correct.
+  # Now 14 regions are supported by Yahoo! Answers.
+  def self.prove_region(region)
+    regions = ['de', 'us', 'uk', 'ca', 'au', 'in', 'es',
+	       'br', 'ar', 'mx', 'e1', 'it', 'fr', 'sg']
+    
+    unless regions.include?(region)
+      $stderr.puts "The provided search region #{region} is currently not supported by Yahoo!"
+      exit 1
+    end
+  end
+  
+  # define one of the following query types: TermQuery|CategoryQuery|
+  # QuestionQuery|UserQuery
+  def self.set_query_type(params)
+    case
+    when (params[:category_id] || params[:category_name]) && ! params[:query]
+      return 'CategoryQuery'
+    when params[:query]
+      return 'TermQuery'
+    when params[:user_id]
+      return 'UserQuery'
+    when params[:question_id]
+      return 'QuestionQuery'
+    end
+  end
+
+  def self.check_required_options(options)
+    required_opts = [:appid]
+    
+    required_opts.each do |opt|
+      if options[:query_params].has_key?(opt)
+        next
+      else
+        $stderr.puts "A required option --#{opt} is missing."
+        exit 1
+      end
+    end
+  end
+  
+  def self.provide_dir(dir)
+    dir = File.expand_path(dir)
+    #check for existens
+    if File.directory?(dir)
+      if File.writable?(dir)
+	return dir
+      else
+	$stderr.puts 'The directory you have provided is not writable!'
+	exit 1
+      end
+    else
+      FileUtils.mkdir_p(dir)
+      return dir
+    end
+  end # provide_dir
+  
+end # OptionParser
+
+__END__
+
+-c, --category-id # Term & CategorySearch
+-q, --question-id # QuestionSearch
+
+-t, --time-interval
+-f, --output-format
+-p, --prefix #prefix for output files
+
+
+
+
+instance interface
+
+{
+  :query_params => {
+    :appid => 'YahooDemo' | 'SomeStringWithYourID',
+    :callback => '',
+    :category_id => '',
+    :category_name => '',
+    :date_range => '',
+    :filter => '',
+    :output => '',
+    :query => '',
+    :question_id => '',
+    :region => 'de'|'us'|'uk'|'ca'|'au'|'in'|'es'|'br'|'ar'|'mx'|'e1'|'it'|'fr'|'sg',  # default 'en'
+    :results => Integer, # 0..50
+    :search_in => "all" | "question" | "best_answer", # default 'all'
+    :sort => 'relevance' | 'date_desc'| 'date_asc', # default 'relevance'
+    :start => Integer, # <= 1000
+    :type => "all" | "resolved" | "open" | "undecided", # default 'all'
+    :user_id => ''
+  },
+  :query_type => 'TermQuery'|'QuestionQuery'|'UserQuery'|'CategoryQuery',
+  :output_dir => 'some path',
+  :prefix => 'some prefix' # prefix for output files 
+}

data/lib/tester.rb

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+require 'yanser'
+
+params = {
+  :query_method => 'TermQuery',
+  :query_params => {
+    :appid => 'YahooDemo',
+    :query => 'Köln',
+    :region => 'de',
+    :results => 5,
+    :start => 0
+  }
+}
+
+y = Yanser.new(params)
+
+y.start

data/lib/version.rb

@@ -0,0 +1,3 @@
+module YANSER
+  VERSION = '0.0.1'
+end

data/lib/yanser.rb

@@ -0,0 +1,121 @@
+require 'yanapi'
+
+# :title: YANSER, Yahoo! ANSwers harvestER
+# :main: Yanser
+# Main processing class.
+# Yanser encapsulates the main routine and instantiates
+# all other classes.
+#--
+# Yanser takes the users input and validates it.
+# It decides which search method to choose.
+# Then it collects all parameters and useful default values,
+# creates an XyzQuery with the starting point of 0 and gets the first results.
+# If more results were requested, Yanser creates a similar XyzQuery and gets
+# the next result set until the result limitation set by the user is met.
+# 
+class Yanser
+
+  # Yahoo! Answers returns starting not more than at the 1000st question.
+  # It makes no sense to step over.
+  START_LIMIT = 1000
+  
+  # We query the web service every two seconds.
+  QUERY_INTERVAL = 2
+
+  # Yahoo! Answers returns maximum 50 results.
+  MAX_RESULTS = 50
+  
+  def initialize(opts)
+
+    # the minimal output of an OptionParser
+    # {:query_type=>u|q|c|w, :query_params=>{appid, start, results, query}}
+    # opts come from the OptionParser
+    # they are supposed to be correct, no validation here
+    @options = opts
+
+    @options[:limit] = @options[:limit] || START_LIMIT + MAX_RESULTS
+
+    @query_params = @options[:query_params]
+
+    @output_type = @query_params[:output] || 'xml'
+
+    @query_params[:start] = @query_params[:start] || 0
+  end
+
+  # TODO: implement some logging
+  def start
+
+    if @options[:query_type] == 'QuestionQuery'
+      query(@query_params)
+    elsif @options[:limit] < MAX_RESULTS
+      @query_params[:results] = @options[:limit]
+      query(@query_params)
+    else
+      @query_params[:results] = MAX_RESULTS
+
+      while query(@query_params) do
+        # we get the next start point here
+        @query_params[:start] += MAX_RESULTS
+
+        results_left = @options[:limit] - @query_params[:start]
+        if results_left == 0
+          break
+        elsif results_left < MAX_RESULTS
+          @query_params[:results] = results_left
+          break
+        end
+      end # while
+      
+    end # if
+    
+  end # start
+
+  private
+
+  def query(params)
+    q = create_query(params)
+    tries = 0
+    begin
+      tries += 1
+      result = q.get
+      output(result)
+      sleep(2)
+    rescue YANAPI::EmptyResponse => e
+      $sderr.puts e
+      return false # do not iterate futher
+    rescue => e # some errors to retry
+      if (tries < 4)
+        sleep(QUERY_INTERVAL**tries)
+        retry
+      else
+        $stderr.puts e
+        return false # do not iterate futher
+      end
+    end
+    return true # we may iterate futher
+  end
+
+  def create_query(params)
+    eval("YANAPI::#{@options[:query_type]}.new(params)")
+  end
+  
+  def output(result)
+    if @options[:output_dir]
+      save(result)
+    else
+      puts result
+    end
+  end
+
+  # save results to a dir
+  # this dir exists since has been proved by OptionParser
+  # not a good implementation
+  # interface (filename, data)
+  def save(result)
+    filename = File.join(@options[:output_dir], "#{@query_params[:start]}.#{@output_type}")
+    file = File.new(filename, 'w')
+    file.puts result
+    file.close
+  end
+
+end # Yanser