apohllo-poliqarpr 0.0.3 → 0.0.4

data/README.txt

@@ -11,8 +11,16 @@ Poliqarpr is Ruby client for Poliqarp server.
 
 * built-in pagination of query results
 * support for lemmatization 
-* asynchronous communication is implemented in synchronous manner 
-* only partial implementation of server protocol
+* asynchronous communication might be not stable (100% CPU ussage)
+* only partial implementation of server protocol:
+** PING, VERSION
+** MAKE-SESSION, CLOSE-SESSION
+** OPEN (corpus), CLOSE (corpus)
+** CORPUS-STATS, GET-TAGSET
+** STATUS, CANCEL (used internally)
+** MAKE-QUERY, RUN-QUERY, BUFFER-STATE (used internally)
+** GET-RESULTS, GET-CONTEXT, METADATA
+** SET: lemmata, tags 
 
 == SYNOPSIS:
 

data/changelog.txt

@@ -1,3 +1,10 @@
+0.0.4
+- ping/pong diagnostics 
+- server version
+- corpus statistics 
+- implementation of asynchronous protocol (not stable)
+
+
 0.0.3
 - the license of the corpus included
 - client rdoc documentation 

data/lib/poliqarpr.rb

@@ -1,11 +1,9 @@
-path = File.join(File.dirname(__FILE__), 'poliqarpr')
-require File.join(path, 'client')
-require File.join(path, 'query_result')
-require File.join(path, 'excerpt')
-require File.join(path, 'segment')
-require File.join(path, 'lemmata')
 begin
   require 'poliqarpr-corpus' 
 rescue LoadError
   # Do nothig, since the default corpus is optional
 end
+
+$LOAD_PATH.unshift File.dirname(__FILE__)
+Dir.glob(File.join(File.dirname(__FILE__), 'poliqarpr/**.rb')).each { |f| require f }
+

data/lib/poliqarpr/client.rb

@@ -1,29 +1,9 @@
-require 'socket'
 module Poliqarp
   # Author:: Aleksander Pohl (mailto:apohllo@o2.pl)
   # License:: MIT License
   #
   # This class is the implementation of the Poliqarp server client. 
   class Client
-    ERRORS = {
-      1 =>   "Incorrect number of arguments",
-      3 =>   "No session opened",
-      4 =>   "Cannot create a session for a connection that",
-      5 =>   "Not enough memory",
-      6 =>   "Invalid session ID",
-      7 =>   "Session with this ID is already bound",
-      8 =>   "Session user ID does not match the argument",
-      10 =>   "Session already has an open corpus",
-      12 =>   "System error while opening the corpus",
-      13 =>   "No corpus opened",
-      14 =>   "Invalid job ID",
-      15 =>   "A job is already in progress",
-      16 =>   "Incorrect query",
-      17 =>   "Invalid result range",
-      18 =>   "Incorrect session option",
-      19 =>   "Invalid session option value",
-      20 =>   "Invalid sorting criteria"
-    }
     GROUPS = [:left_context, :left_match, :right_match, :right_context]
 
     # If debug is turned on, the communication between server and client 
@@ -46,10 +26,12 @@ module Poliqarp
       @right_context = 5
       @debug = debug
       @buffer_size = 500000
+      @connector = Connector.new(debug)
+      @answer_queue = Queue.new
       new_session
     end
 
-    # A hint about uninstalled default corpus gem
+    # A hint about installation of default corpus gem
     def self.const_missing(const)
       if const.to_s =~ /DEFAULT_CORPUS/ 
         raise "You need to install 'apohllo-poliqarpr-corpus' to use the default corpus"
@@ -64,26 +46,25 @@ module Poliqarp
     # * +port+ - the port on which the poliqarpd server is accepting connections (defaults to 4567)
     def new_session(port=4567)
       close if @session
-      @socket = TCPSocket.new("localhost",port)
-      talk "MAKE-SESSION #{@session_name}"
-      rcv_sync
+      @connector.open("localhost",port)
+      talk("MAKE-SESSION #{@session_name}")
       talk("BUFFER-RESIZE #{@buffer_size}")
-      rcv_sync
       @session = true
       self.tags = {}
       self.lemmata = {}
     end
 
-    # Closes the opened connection to the poliqarpd server.
+    # Closes the opened session.
     def close
-      #talk "CLOSE"
-      #rcv_sync
       talk "CLOSE-SESSION" 
-      rcv_sync
-      #@socket.close
       @session = false
     end
 
+    # Closes the opened corpus.
+    def close_corpus
+      talk "CLOSE"
+    end
+
     # Sets the size of the left short context. It must be > 0
     #
     # The size of the left short context is the number 
@@ -91,8 +72,7 @@ module Poliqarp
     # matched segment(s).
     def left_context=(value)
       if correct_context_value?(value) 
-        talk "SET left-context-width #{value}" 
-        result = rcv_sync 
+        result = talk("SET left-context-width #{value}")
         @left_context = value if result =~ /^R OK/
       else
         raise "Invalid argument: #{value}. It must be fixnum greater than 0."
@@ -106,8 +86,7 @@ module Poliqarp
     # matched segment(s).
     def right_context=(value)
       if correct_context_value?(value)
-        talk "SET right-context-width #{value}"     
-        result = rcv_sync 
+        result = talk("SET right-context-width #{value}")
         @right_context = value if result =~ /^R OK/
       else
         raise "Invalid argument: #{value}. It must be fixnum greater than 0."
@@ -135,8 +114,7 @@ module Poliqarp
       GROUPS.each do |flag|
         flags << (options[flag] ? "1" : "0")
         end
-      talk "SET retrieve-tags #{flags}"
-      rcv_sync
+      talk("SET retrieve-tags #{flags}")
     end
 
     # Sets the lemmatas' flags. There are four groups of segments 
@@ -160,20 +138,87 @@ module Poliqarp
       GROUPS.each do |flag|
         flags << (options[flag] ? "1" : "0")
         end
-      talk "SET retrieve-lemmata #{flags}"
-      rcv_sync
+      talk("SET retrieve-lemmata #{flags}")
     end
 
-    # Opens the corpus given as +path+. To open the default
+    # *Asynchronous* Opens the corpus given as +path+. To open the default
     # corpus pass +:default+ as the argument. 
-    def open_corpus(path)
+    # 
+    # If you don't want to wait until the call is finished, you
+    # have to provide +handler+ for the asynchronous answer.
+    def open_corpus(path, &handler)
       if path == :default
-        open_corpus(DEFAULT_CORPUS)
+        open_corpus(DEFAULT_CORPUS, &handler)
       else
-        talk("OPEN #{path}")
-        rcv_sync
-        rcv_async
+        real_handler = handler || lambda{|msg| @answer_queue.push msg }
+        talk("OPEN #{path}", :async, &real_handler)
+        do_wait if handler.nil?
+      end
+    end
+
+    # Server diagnostics -- the result should be :pong
+    def ping 
+      :pong if talk("PING") =~ /PONG/
+    end
+
+    # Returns server version
+    def version 
+      talk("VERSION")
+    end
+
+    # Returns corpus statistics:
+    # * +:segment_tokens+ the number of segments in the corpus 
+    #   (two segments which look exactly the same are counted separately)
+    # * +:segment_types+ the number of segment types in the corpus
+    #   (two segments which look exactly the same are counted as one type)
+    # * +:lemmata+ the number of lemmata (lexemes) types
+    #   (all forms of inflected word, e.g. 'kot', 'kotu', ... 
+    #   are treated as one "word" -- lemmata)
+    # * +:tags+ the number of different grammar tags (each combination
+    #   of atomic tags is treated as different "tag")
+    def stats
+      stats = {}
+      talk("CORPUS-STATS").split.each_with_index do |value, index|
+        case index
+        when 1 
+          stats[:segment_tokens] = value.to_i
+        when 2
+          stats[:segment_types] = value.to_i
+        when 3
+          stats[:lemmata] = value.to_i
+        when 4
+          stats[:tags] = value.to_i
+        end
+      end
+      stats
+    end
+
+    # TODO
+    def metadata_types
+      raise "Not implemented"
+    end
+
+    # Returns the tag-set used in the corpus.
+    # It is divided into two groups:
+    # * +:categories+ enlists tags belonging to grammatical categories
+    #   (each category has a list of its tags, eg. gender: m1 m2 m3 f n,
+    #   means that there are 5 genders: masculine(1,2,3), feminine and neuter)
+    # * +:classes+ enlists grammatical tags used to describe it
+    #   (each class has a list of tags used to describe it, eg. adj: degree 
+    #   gender case number, means that adjectives are described in terms
+    #   of degree, gender, case and number)
+    def tagset
+      answer = talk("GET-TAGSET")
+      counters = answer.split
+      result = {}
+      [:categories, :classes].each_with_index do |type, type_index|
+        result[type] = {}
+        counters[type_index+1].to_i.times do |index|
+          values = read_word.split
+          result[type][values[0].to_sym] = values[1..-1].map{|v| v.to_sym}
+        end
       end
+      result
     end
 
     # Send the query to the opened corpus.
@@ -210,8 +255,6 @@ module Poliqarp
       make_query(query)
       result = []
       talk "GET-CONTEXT #{index}"
-      # R OK
-      rcv_sync
       # 1st part
       result << read_word 
       # 2nd part
@@ -228,11 +271,11 @@ module Poliqarp
     def metadata(query, index)
       make_query(query)
       result = {}
-      talk "METADATA #{index}"
-      count = rcv_sync.split(" ")[2].to_i
+      answer = talk("METADATA #{index}")
+      count = answer.split(" ")[1].to_i
       count.times do |index|
         type = read_word.gsub(/[^a-zA-Z]/,"").to_sym
-        value = rcv_sync[4..-2]
+        value = read_word[2..-1]
         unless value.nil?
           result[type] ||= []
           result[type] << value
@@ -244,36 +287,40 @@ module Poliqarp
 protected
     # Sends a message directly to the server
     # * +msg+ the message to send
-    def talk(msg)
+    # * +mode+ if set to :sync, the method block untli the message
+    #   is received. If :async the method returns immediately.
+    #   Default: :sync
+    # * +handler+ the handler of the assynchronous message. 
+    #   It is ignored when the mode is set to :sync.
+    def talk(msg, mode = :sync, &handler)
       puts msg if @debug
-      @socket.puts(msg)
+      @connector.send(msg, mode, &handler)
     end
 
+    # Make query and retrieve many results. 
+    # * +query+ the query to be sent to the server.
+    # * +options+ see find
     def find_many(query, options)
       page_size = (options[:page_size] || 0)
       page_index = (options[:page_index] || 1)
-      answers = make_query(query)
-      #talk("GET-COLUMN-TYPES")
-      #rcv_sync
-      result_count = count_results(answers)
+
       answer_offset = page_size * (page_index - 1)
       if page_size > 0
+        result_count = make_async_query(query,answer_offset)
         answers_limit = answer_offset + page_size > result_count ?  
           result_count - answer_offset : page_size
       else
+        # all answers needed -- the call must be synchronous
+        result_count = count_results(make_query(query))
         answers_limit = result_count
       end
-      page_count = if page_size > 0
-                     result_count / page_size + (result_count % page_size > 0 ? 1 : 0)
-                   else
-                     1
-                   end
+
+      page_count = page_size <= 0 ? 1 :
+        result_count / page_size + (result_count % page_size > 0 ? 1 : 0)
+
       result = QueryResult.new(page_index, page_count,page_size,self,query)
       if answers_limit > 0
         talk("GET-RESULTS #{answer_offset} #{answer_offset + answers_limit - 1}") 
-        # R OK              1
-        rcv_sync
-
         answers_limit.times do |answer_index|
           result << fetch_result(answer_offset + answer_index, query)
         end
@@ -281,17 +328,18 @@ protected
       result 
     end
 
+    # Make query and retrieve only one result
+    # * +query+ the query to be sent to the server
+    # * +index+ the index of the answer to be retrieved
     def find_one(query,index)
-      make_query(query)
+      make_async_query(query,index)
       talk("GET-RESULTS #{index} #{index}") 
-      # R OK              1
-      rcv_sync
       fetch_result(index,query) 
     end
 
     # Fetches one result of the query
-    ##
-    # MAKE-QUERY and GET-RESULTS must be called on server before 
+    #
+    # MAKE-QUERY and GET-RESULTS must be sent to the server before 
     # this method is called
     def fetch_result(index, query)
       result = Excerpt.new(index, self, query)
@@ -300,18 +348,17 @@ protected
       # XXX
       #result << read_segments(:right_match)
       result << read_segments(:right_context)
-
       result
     end
 
     def read_segments(group)
-      size = get_number(rcv_sync)
+      size = read_number()
       segments = []
       size.times do |segment_index|
         segment = Segment.new(read_word)
         segments << segment 
         if @lemmata_flags[group] || @tag_flags[group]
-          lemmata_size = get_number(rcv_sync)
+          lemmata_size = read_number()
           lemmata_size.times do |lemmata_index| 
             lemmata = Lemmata.new()
             if @lemmata_flags[group]
@@ -327,62 +374,56 @@ protected
       segments
     end
 
-    def get_number(str)
-      str.match(/\d+/)[0].to_i
+    # Reads number stored in the message received from the server.
+    def read_number
+      @connector.read_message.match(/\d+/)[0].to_i
     end
 
+    # Counts number of results for given answer
     def count_results(answer)
-      answer.split(" ")[2].to_i
+      answer.split(" ")[1].to_i
     end
 
-    def make_query(query)
+    # *Asynchronous* Sends the query to the server
+    # * +query+ query to send
+    # * +handler+ if given, the method returns immediately, 
+    #   and the answer is sent to the handler. In this case
+    #   the result returned by make_query should be IGNORED!
+    def make_query(query, &handler)
       if @last_query != query
         @last_query = query
-        talk("MAKE-QUERY #{query}")
-        rcv_sync
-        talk("RUN-QUERY #{@buffer_size}")
-        @last_query_result = rcv_async
+        if handler.nil?
+          real_handler = lambda { |msg| @answer_queue.push msg }
+        else
+          real_handler = handler
+        end
+        begin
+          talk("MAKE-QUERY #{query}")
+        rescue JobInProgress
+          talk("CANCEL") rescue nil
+          talk("MAKE-QUERY #{query}")
+        end
+        talk("RUN-QUERY #{@buffer_size}", :async, &real_handler) 
+        @last_result = do_wait if handler.nil?
       end
-      @last_query_result 
+      @last_result
     end
 
+    # Reads string stored in the last message received from server
     def read_word
-      rcv_sync[2..-2]
-    end
-
-    def read_line
-      line = ""
-      begin
-        chars = @socket.recvfrom(1)
-        line << chars[0]
-      end while chars[0] != "\n"
-      line
+      @connector.read_message
     end
 
-    def error_message(line)
-      RuntimeError.new("Poliqarp Error: "+ERRORS[line.match(/\d+/)[0].to_i])
-    end
-
-    # XXX
-    def rcv_sync
-      result = read_line
-      puts result if @debug
-      raise error_message(result) if result =~ /^R ERR/
-      result
-      #    @socket.recvfrom(1024)
-    end
-
-    # XXX
-    def rcv_async
-      begin
-        line = read_line
-        raise error_message(line) if line =~ /^. ERR/
-        puts line if @debug
-      end until line =~ /^M/
-      line
+private 
+    def do_wait
+      loop {
+        status = talk("STATUS") rescue break
+        puts "STATUS: #{status}" if @debug
+        sleep 0.3
+      }
+      @answer_queue.shift
     end
 
-private 
     def set_all_flags
       options = {}
       GROUPS.each{|g| options[g] = true}
@@ -392,5 +433,19 @@ private
     def correct_context_value?(value)
       value.is_a?(Fixnum) && value > 0
     end
+
+    def make_async_query(query,answer_offset) 
+      # the handler is empty, since we access the result count through 
+      # BUFFER-STATE call
+      make_query(query){|msg| }
+      result_count = 0 
+      begin 
+        # the result count might be not exact!
+        result_count = talk("BUFFER-STATE").split(" ")[2].to_i
+        talk("STATUS") rescue break
+      end while result_count < answer_offset
+      @last_result = "OK #{result_count}"
+      result_count
+    end
   end 
 end

data/lib/poliqarpr/connector.rb

@@ -0,0 +1,128 @@
+require 'socket'
+
+module Poliqarp
+  # Author:: Aleksander Pohl (mailto:apohllo@o2.pl)
+  # License:: MIT License
+  #
+  # This class hold the TCP connection to the server and is responsible
+  # for dispatching synchronous and asynchronous queries and answers.
+  class Connector
+
+    # Error messages assigned to error codes
+    # (taken from poliqarpd implementation)
+    ERRORS = {
+      1 =>   "Incorrect number of arguments",
+      3 =>   "No session opened",
+      4 =>   "Cannot create a session for a connection that",
+      5 =>   "Not enough memory",
+      6 =>   "Invalid session ID",
+      7 =>   "Session with this ID is already bound",
+      8 =>   "Session user ID does not match the argument",
+      10 =>   "Session already has an open corpus",
+      12 =>   "System error while opening the corpus",
+      13 =>   "No corpus opened",
+      14 =>   "Invalid job ID",
+      15 =>   "A job is already in progress",
+      16 =>   "Incorrect query",
+      17 =>   "Invalid result range",
+      18 =>   "Incorrect session option",
+      19 =>   "Invalid session option value",
+      20 =>   "Invalid sorting criteria"
+    }
+
+    # Creates new connector
+    def initialize(debug)
+      @message_queue = Queue.new
+      @socket_mutex = Mutex.new
+      @loop_mutex = Mutex.new
+      @debug = debug
+    end
+
+    # Opens connection with poliqarp server which runs 
+    # on given +host+ and +port+.
+    def open(host,port)
+      @socket_mutex.synchronize {
+        @socket = TCPSocket.new(host,port) if @socket.nil?
+      }
+      running = nil
+      @loop_mutex.synchronize {
+        running = @loop_running
+      }
+      main_loop unless running
+      @loop_mutex.synchronize {
+        @loop_running = true
+      }
+    end
+
+    # Sends message to the poliqarp server. Returns the first synchronous 
+    # answer of the server.
+    # * +message+ the message to send
+    # * +mode+ synchronous (+:sync:) or asynchronous (+:async+)
+    # * +handler+ the handler of the asynchronous message
+    def send(message, mode, &handler)
+      puts "send #{mode} #{message}" if @debug
+      @socket.puts(message)
+      if mode == :async
+        @handler = handler
+      end
+      read_message
+    end
+
+    # Retrives one message from the server.
+    # If the message indicates an error, new runtime error 
+    # containing the error description is returned.
+    def read_message
+      message = @message_queue.shift
+      if message =~ /^ERR/
+        code = message.match(/\d+/)[0].to_i
+        raise JobInProgress.new() if code == 15
+        raise RuntimeError.new("Poliqarp Error: "+ERRORS[code])
+      else
+        message
+      end
+    end
+
+private
+    def main_loop
+      @loop = Thread.new { 
+        loop {
+          receive
+          # XXX ??? needed
+          #sleep 0.001
+        }
+      }
+    end
+
+    def receive
+      result = read_line
+      msg = result[2..-2]
+      if result =~ /^M/
+        receive_async(msg)
+      elsif result
+        receive_sync(msg)
+      end
+      # if nil, nothing was received
+    end
+
+    def receive_sync(message)
+      puts "receive sync: #{message}" if @debug
+      @message_queue << message
+    end
+
+    def receive_async(message)
+      puts "receive async: #{message}" if @debug
+      Thread.new{ 
+        @handler.call(message) 
+      }
+    end
+
+    def read_line
+      line = ""
+      begin
+        chars = @socket.recvfrom(1)
+        line << chars[0]
+      end while chars[0] != "\n"
+      line
+    end
+  end
+end

data/lib/poliqarpr/exceptions.rb

@@ -0,0 +1,9 @@
+module Poliqarp
+  # Author:: Aleksander Pohl (mailto:apohllo@o2.pl)
+  # License:: MIT License
+
+  # The JobInProgress exception is raised if there was asynchronous call 
+  # to the server which haven't finished, which is interrupted by another
+  # asynchronous call.
+  class JobInProgress < Exception; end
+end

data/poliqarpr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "poliqarpr"
-  s.version = "0.0.3"
+  s.version = "0.0.4"
   s.date = "2008-12-20"
   s.summary = "Ruby client for Poliqarp"
   s.email = "apohllo@o2.pl"
@@ -14,6 +14,8 @@ Gem::Specification.new do |s|
     "lib/poliqarpr/excerpt.rb", 
     "lib/poliqarpr/lemmata.rb", 
     "lib/poliqarpr/segment.rb", 
+    "lib/poliqarpr/connector.rb", 
+    "lib/poliqarpr/exceptions.rb", 
     "README.txt",
   ]
   s.test_files = [

data/spec/client.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), '..','lib','poliqarpr')
+require 'lib/poliqarpr'
 
 describe Poliqarp::Client do
   describe "(general test)" do
@@ -17,6 +17,15 @@ describe Poliqarp::Client do
     it "should allow to open :default corpus" do
       @client.open_corpus(:default)
     end
+
+    it "should respond to :ping" do
+      @client.ping.should == :pong
+    end
+
+    it "should return server version" do
+      @client.version.should_not == nil
+    end
+
   end
 
   describe "(with 'sample' corpus)" do
@@ -61,6 +70,21 @@ describe Poliqarp::Client do
       end).should raise_error(RuntimeError)
     end
 
+    it "should return corpus statistics" do
+      stats = @client.stats
+      stats.size.should == 4
+      [:segment_tokens, :segment_types, :lemmata, :tags].each do |type|
+        stats[type].should_not == nil
+        stats[type].should > 0
+      end
+    end
+
+    it "should return the corpus tagset" do
+      tagset = @client.tagset
+      tagset[:categories].should_not == nil
+      tagset[:classes].should_not == nil
+    end
+
     it "should allow to find 'kot'" do 
       @client.find("kot").size.should_not == 0
     end
@@ -111,6 +135,10 @@ describe Poliqarp::Client do
         @result.should_not respond_to(:[])
       end
 
+      it "should not be nil" do
+        @result.should_not == nil
+      end
+
       it "should fetch the same excerpt as in find without index " do
         @result.to_s.should == @client.find("nachalny")[0].to_s
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: apohllo-poliqarpr
 version: !ruby/object:Gem::Version 
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors: 
 - Aleksander Pohl
@@ -31,6 +31,8 @@ files:
 - lib/poliqarpr/excerpt.rb
 - lib/poliqarpr/lemmata.rb
 - lib/poliqarpr/segment.rb
+- lib/poliqarpr/connector.rb
+- lib/poliqarpr/exceptions.rb
 - README.txt
 has_rdoc: true
 homepage: http://www.apohllo.pl/projekty/poliqarpr