thinkingtank 0.0.2 → 0.0.3

data/lib/indextank_client.rb

@@ -0,0 +1,255 @@
+require 'net/http'
+require 'uri'
+require 'rubygems'
+require 'json'
+
+module IndexTank
+
+    private
+    
+    class RestClient
+        def GET(path, params={})
+            path = "#{path}?#{to_query(params)}" if params
+            request = Net::HTTP::Get.new "#{@uri}#{path}"
+            authorize request
+            return execute(request)
+        end
+        
+        def PUT(path, body={})
+            request = Net::HTTP::Put.new "#{@uri}#{path}"
+            authorize request
+            request.body = body.to_json if body
+            return execute(request)
+        end
+        
+        def DELETE(path, params={})
+            path = "#{path}?#{to_query(params)}" if params
+            request = Net::HTTP::Delete.new "#{@uri}#{path}"
+            authorize request
+            return execute(request)
+        end
+        
+        private
+        
+        def to_query(params)
+            require 'cgi' unless defined?(CGI) && defined?(CGI::escape)
+            r = ''
+            params.each do |k,v|
+                r << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}&"
+            end
+            return r            
+        end
+        
+        def authorize(req)
+            req.basic_auth(@uri.user, @uri.password)
+        end
+        
+        def execute(req)
+            res = Net::HTTP.new(@uri.host).start { |http| http.request(req) }
+            if res.is_a? Net::HTTPSuccess
+                if res.body.nil? or res.body.empty? 
+                    return res.code, nil
+                else
+                    begin
+                        return res.code, JSON.parse(res.body)
+                    rescue
+                        raise "Invalid JSON response: #{res.body}"
+                    end
+                end
+            elsif res.is_a? Net::HTTPUnauthorized
+                raise SecurityError, "Authorization required"
+            elsif res.is_a? Net::HTTPBadRequest
+                raise ArgumentError, res.body
+            else
+                raise HttpCodeException.new(res.code, res.body)
+            end
+        end
+        
+    end
+    
+    public    
+    
+    class ApiClient < RestClient
+        def initialize(api_url)
+            @uri = URI.parse(api_url)
+        end
+        
+        def get_index(name)
+            return IndexClient.new("#{@uri}/v1/indexes/#{name}")
+        end
+    
+        def create_index(name)
+            index = get_index(name)
+            index.create_index()
+            return index
+        end
+    
+        def delete_index(name)
+            return get_index(name).delete_index()
+        end
+    
+        def list_indexes()
+            code, indexes = GET "/v1/indexes"
+            return indexes.map do |name,metadata| IndexClient.new "#{@uri}/v1/indexes/#{name}", metadata end 
+        end
+    end
+    
+    class IndexClient < RestClient
+        def initialize(index_url, metadata=nil)
+            @uri = URI.parse(index_url)
+            @metadata = metadata
+        end
+        
+        def code
+            return metadata['code']
+        end
+    
+        def running?
+            return metadata!['started']
+        end
+        
+        def creation_time
+            return metadata['creation_time']
+        end
+        
+        def size
+            return metadata['size']
+        end
+    
+        def exists?
+            begin
+                metadata!
+                return true
+            rescue HttpCodeException
+                if $!.code == "404"
+                    return false
+                end
+                raise
+            end
+        end
+    
+        # the options argument may contain a :variables key
+        # with a Hash from variable numbers to their float values
+        # this variables can be used in the scoring functions
+        # when sorting a search
+        def add_document(docid, fields, options={})
+            options.merge!( :docid => docid, :fields => fields )
+            code, r = PUT "/docs", options
+            return r
+        end
+    
+        def update_variables(docid, variables, options={})
+            options.merge!( :docid => docid, :variables => variables )
+            code, r = PUT "/docs/variables", options
+            return r
+        end
+    
+        def delete_document(docid, options={})
+            options.merge!( :docid => docid )
+            code, r = DELETE "/docs", options
+            return r
+        end
+    
+        # the options argument may contain an :index_code definition to override 
+        # this instance's default index_code  
+        def promote(docid, query, options={})
+            options.merge!( :docid => docid, :query => query )
+            code, r = PUT "/promote", options
+            return r
+        end
+    
+    
+        # the options argument may contain an :index_code definition to override 
+        # this instance's default index_code
+        # it can also contain any of the following:
+        #   :start => an int with the number of results to skip
+        #   :len => an int with the number of results to return
+        #   :snippet => a comma separated list of field names for which a snippet
+        #               should be returned. (requires an index that supports snippets)
+        #   :fetch => a comma separated list of field names for which its content
+        #             should be returned. (requires an index that supports storage)
+        #   :function => an int with the index of the scoring function to be used
+        #                for this query 
+        def search(query, options={})
+            options = { :start => 0, :len => 10 }.merge(options)
+            options.merge!( :q => query )
+            begin
+                code, r = GET "/search", options
+                return r
+            rescue HttpCodeException
+                raise
+            end
+        end
+    
+        def add_function(function_index, definition, options={})
+            options.merge!( :definition => definition )
+            code, r = PUT "/functions/#{function_index}", options
+            return r
+        end
+        
+        def del_function(function_index, options={})
+            code, r = DELETE "/functions/#{function_index}", options
+            return r
+        end
+        
+        def list_functions(options={})
+            code, r = GET "/functions"
+            return r
+        end
+    
+        def create_index()
+            begin
+                code, r = PUT ""
+                raise IndexAlreadyExists if code == "204"
+                return r
+            rescue HttpCodeException
+                if $!.code == "409"
+                    puts $!.code
+                    raise TooManyIndexes
+                end
+                raise
+            end
+        end
+        
+        def delete_index()
+            code, r = DELETE ""
+            return r
+        end
+        
+        def metadata
+            metadata! if @metadata.nil?
+            return @metadata
+        end    
+        
+        def metadata!
+            code, @metadata = GET ""
+            return @metadata
+        end
+        
+    end
+    
+    class IndexAlreadyExists < StandardError
+    end
+    class TooManyIndexes < StandardError
+    end
+
+    class HttpCodeException < StandardError
+        def initialize(code, message)
+            @code = code
+            @message = message
+            super("#{code}: #{message}")
+        end
+        
+        attr_accessor :code
+        attr_accessor :message
+    end
+    
+    class HerokuClient < ApiClient
+        def initialize()
+            super(ENV['HEROKUTANK_API_URL'])
+        end
+    end
+
+    
+end
+

data/lib/thinkingtank/init.rb

@@ -1,4 +1,4 @@
-require 'indextank'
+require 'indextank_client'
 
 module ThinkingTank
     class Builder
@@ -31,7 +31,7 @@ module ThinkingTank
             return unless File.exists?(path)
 
             conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
-            self.client = IndexTank.new(conf['api_key'], :index_code => conf['index_code'], :index_name => conf['index_name'])
+            self.client = IndexTank::ApiClient.new(conf['api_url']).get_index(conf['index_name'])
         end
         def environment
             if defined?(Merb)
@@ -51,11 +51,11 @@ module ThinkingTank
             data = {}
             self.class.thinkingtank_builder.index_fields.each do |field|
                 val = self.instance_eval(field.to_s)
-                data["_" + field.to_s] = val.to_s unless val.nil?
+                data[field.to_s] = val.to_s unless val.nil?
             end
-            data[:text] = data.values.join " . "
-            data[:type] = self.class.name
-            it.add(docid, data)
+            data[:__any] = data.values.join " . "
+            data[:__type] = self.class.name
+            it.add_document(docid, data)
         end
     end
 
@@ -64,26 +64,10 @@ end
 class << ActiveRecord::Base
     @indexable = false
     def search(*args)
-        options = args.extract_options!
-        query = args.join(' ')
-        if options.has_key? :conditions
-            options[:conditions].each do |field,value|
-                field = "_#{field}" # underscore prepended to ActiveRecord fields
-                query += " #{field}:(#{value})"
-            end
-        end
-        # TODO : add relevance functions
-
-        it = ThinkingTank::Configuration.instance.client
-        models = []
-        ok, res = it.search("#{query.to_s} type:#{self.name}")
-        if ok
-            res['docs'].each do |doc|
-                type, docid = doc['docid'].split(" ", 2)
-                models << self.find(id=docid)
-            end
-        end
-        return models
+        return indextank_search(true, *args)
+    end
+    def search_raw(*args)
+        return indextank_search(false, *args)
     end
 
     def define_index(name = nil, &block)
@@ -100,6 +84,41 @@ class << ActiveRecord::Base
     def thinkingtank_builder
         return @thinkingtank_builder
     end
+    
+    private
+    
+    def indextank_search(models, *args)
+        options = args.extract_options!
+        query = args.join(' ')
+        
+        # transform fields in query
+        
+        if options.has_key? :conditions
+            options[:conditions].each do |field,value|
+                query += " #{field}:(#{value})"
+            end
+        end
+        
+        options.slice!(:snippet, :fetch, :function)
+
+        it = ThinkingTank::Configuration.instance.client
+        models = []
+        res = it.search("__any:(#{query.to_s}) __type:#{self.name}", options)
+        if models
+            res['results'].each do |doc|
+                type, docid = doc['docid'].split(" ", 2)
+                models << self.find(id=docid)
+            end
+            return models
+        else
+            res['results'].each do |doc|
+                type, docid = doc['docid'].split(" ", 2)
+                doc['model'] = self.find(id=docid)
+            end
+            return res
+        end
+    end
+
 end
 
 

data/lib/thinkingtank/tasks.rb

@@ -1,5 +1,6 @@
 require 'erb'
 require 'active_record'
+require 'indextank_client'
 
 def load_models
     app_root = ThinkingTank::Configuration.instance.app_root
@@ -28,6 +29,18 @@ def load_models
 end
 
 def reindex_models
+    it = ThinkingTank::Configuration.instance.client
+    if it.exists?
+        puts "Deleting existing index"
+        it.delete_index()
+    end
+    puts "Creating a new empty index"
+    it.create_index()
+    puts "Waiting for the index to be ready"
+    while not it.running?
+        sleep 0.5
+    end
+
     Object.subclasses_of(ActiveRecord::Base).each do |klass|
         reindex klass if klass.is_indexable?
     end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - Flaptor
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-19 00:00:00 -03:00
+date: 2010-07-26 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -30,7 +30,7 @@ extra_rdoc_files: []
 
 files: 
 - rails/init.rb
-- lib/indextank.rb
+- lib/indextank_client.rb
 - lib/thinkingtank/init.rb
 - README
 - tasks/rails.rake

data/lib/indextank.rb

@@ -1,150 +0,0 @@
-require 'net/http'
-require 'uri'
-require 'rubygems'
-require 'json'
-
-BASE_URL = 'http://api.indextank.com/api/v0'
-class IndexTank
-    def initialize(api_key, options={})
-        @api_key = api_key
-        @def_index_code = options[:index_code]
-        @def_index_name = options[:index_name]
-    end
-
-    def create_index(index_name)
-        return api_call("admin/create", "index_code", :index_name => index_name)
-    end
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def delete_index(options={})
-        return api_call("admin/delete", nil, options)
-    end
-
-
-    def list_indexes()
-        return api_call("admin/list", "results")
-    end
-
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def add(doc_id, content, options={})
-        options.merge!(:document_id => doc_id, :document => content.to_json)
-        options[:boosts] = options[:boosts].to_json if options.key?(:boosts)             
-        return api_call("index/add", nil, options)
-    end
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def boost(doc_id, boosts, options={})
-        options.merge!(:document_id => doc_id, :boosts => boosts.to_json)
-        return api_call("index/boost", nil, options)
-    end
-
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def promote(doc_id, query, options={})
-        options.merge!(:document_id => doc_id, :query => query)
-        return api_call("index/promote", nil, options)
-    end
-
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def update(doc_id, content, options={})
-        options.merge!(:document_id => doc_id, :document => content.to_json)
-        return api_call("index/update", nil, options)
-    end
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def delete(doc_id, options={})
-        options.merge!(:document_id => doc_id)
-        return api_call("index/delete", nil, options)
-    end
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code
-    # it can also contain any of the following:
-    #   :start => an int with the number of results to skip
-    #   :len => an int with the number of results to return
-    #   :snippet_fields => a comma separated list of field names for which a snippet
-    #                      should be returned. (requires an index that supports snippets)
-    #   :fetch_fields => a comma separated list of field names for which its content
-    #                    should be returned. (requires an index that supports storage)
-    #   :relevance_function => an int with the index of the relevance function to be used
-    #                          for this query 
-    def search(query, options={})
-        options = { :start => 0, :len => 10 }.merge(options)
-        options.merge!(:query => query)
-        return api_call("search/query", "results", options)
-    end
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def add_function(function_index, definition, options={})
-        options.merge!( :function_id => function_index, :definition => definition )
-        return api_call("index/add_function", nil, options)
-    end
-    
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def del_function(function_index, options={})
-        options.merge!( :function_id => function_index )
-        return api_call("index/remove_function", nil, options)
-    end
-    
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def list_functions(options={})
-        return api_call("index/list_functions", "results", options)
-    end
-
-       
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def index_stats(options={})
-        return api_call("index/stats", "results", options)
-    end
-
-
-    # the options argument may contain an :index_code definition to override 
-    # this instance's default index_code  
-    def search_stats(options={})
-        return api_call("search/stats", "results", options)
-    end
-
-    private
-
-    def base_url
-        return ENV['INDEXTANK_BASE_URL'] || BASE_URL
-    end
-
-    def api_call(method, return_key, params={})
-        params = { "api_key" => @api_key }.merge(params)
-        params = { :index_code => @def_index_code }.merge(params) unless @def_index_code.nil?
-        params = { :index_name => @def_index_name }.merge(params) unless @def_index_name.nil?
-        url = base_url + '/' + method
-        req = Net::HTTP::Post.new(url)
-        req.set_form_data(params, ';')
-        res = Net::HTTP.new(URI.parse(base_url).host).start {|http| http.request(req) }
-        if res.is_a? Net::HTTPOK
-            result = JSON.parse(res.body)
-            ok = result['status'] == 'OK'
-            return ok ? [ok, (return_key.nil? ? nil : result[return_key])] : [ok, result['message']]
-        elsif res.is_a? Net::HTTPForbidden
-            return false, "Access forbidden"
-        elsif res.is_a? Net::HTTPClientError
-            return false, "Unknown client error"
-        elsif res.is_a? Net::HTTPServerError
-            puts res.body
-            return false, "Unknown server error"
-        else
-            puts res.body
-            return false, "Unexpected response"
-        end
-    end
-
-end