tire 0.3.12 → 0.4.0.pre

data/README.markdown

@@ -161,8 +161,8 @@ from the database:
 
       sort { by :title, 'desc' }
 
-      facet 'global-tags' do
-        terms :tags, :global => true
+      facet 'global-tags', :global => true do
+        terms :tags
       end
 
       facet 'current-tags' do
@@ -282,6 +282,15 @@ Note, that you can pass options for configuring queries, facets, etc. by passing
     end
 ```
 
+You don't have to define the search criteria in one monolithic _Ruby_ block -- you can build the search step by step,
+until you call the `results` method:
+
+```ruby
+    s = Tire.search('articles') { query { string 'title:T*' } }
+    s.filter :terms, :tags => ['ruby']
+    p s.results
+```
+
 If configuring the search payload with blocks feels somehow too weak for you, you can pass
 a plain old Ruby `Hash` (or JSON string) with the query declaration to the `search` method:
 
@@ -293,7 +302,7 @@ If this sounds like a great idea to you, you are probably able to write your app
 using just `curl`, `sed` and `awk`.
 
 Do note again, however, that you're not tied to the declarative block-style DSL _Tire_ offers to you.
-If it makes more sense in your context, you can use its classes directly, in a more imperative style:
+If it makes more sense in your context, you can use the API directly, in a more imperative style:
 
 ```ruby
     search = Tire::Search::Search.new('articles')
@@ -302,7 +311,7 @@ If it makes more sense in your context, you can use its classes directly, in a m
     search.sort   { by :title, 'desc' }
     search.facet('global-tags') { terms :tags, :global => true }
     # ...
-    p search.perform.results
+    p search.results
 ```
 
 To debug the query we have laboriously set up like this,
@@ -413,11 +422,12 @@ For serious usage, though, you'll definitely want to define a custom _mapping_ f
       include Tire::Model::Callbacks
 
       mapping do
-        indexes :id,           :type => 'string',  :index    => :not_analyzed
-        indexes :title,        :type => 'string',  :analyzer => 'snowball', :boost => 100
-        indexes :content,      :type => 'string',  :analyzer => 'snowball'
-        indexes :author,       :type => 'string',  :analyzer => 'keyword'
-        indexes :published_on, :type => 'date',    :include_in_all => false
+        indexes :id,           :index    => :not_analyzed
+        indexes :title,        :analyzer => 'snowball', :boost => 100
+        indexes :content,      :analyzer => 'snowball'
+        indexes :content_size, :as       => 'content.size'
+        indexes :author,       :analyzer => 'keyword'
+        indexes :published_on, :type => 'date', :include_in_all => false
       end
     end
 ```
@@ -425,6 +435,13 @@ For serious usage, though, you'll definitely want to define a custom _mapping_ f
 In this case, _only_ the defined model attributes are indexed. The `mapping` declaration creates the
 index when the class is loaded or when the importing features are used, and _only_ when it does not yet exist.
 
+You can define different [_analyzers_](http://www.elasticsearch.org/guide/reference/index-modules/analysis/index.html),
+[_boost_](http://www.elasticsearch.org/guide/reference/mapping/boost-field.html) levels for different properties,
+or any other configuration for _elasticsearch_.
+
+You're not limited to 1:1 mapping between your model properties and the serialized document. With the `:as` option,
+you can pass a string or a _Proc_ object which is evaluated in the instance context (see the `content_size` property).
+
 Chances are, you want to declare also a custom _settings_ for the index, such as set the number of shards,
 replicas, or create elaborate analyzer chains, such as the hipster's choice: [_ngrams_](https://gist.github.com/1160430).
 In this case, just wrap the `mapping` method in a `settings` one, passing it the settings as a Hash:
@@ -671,12 +688,11 @@ Well, things stay mostly the same:
       include Tire::Model::Search
       include Tire::Model::Callbacks
 
-      # Let's use a different index name so stuff doesn't get mixed up
+      # Let's use a different index name so stuff doesn't get mixed up.
       #
       index_name 'mongo-articles'
 
-      # These Mongo guys sure do some funky stuff with their IDs
-      # in +serializable_hash+, let's fix it.
+      # These Mongo guys sure do get funky with their IDs in +serializable_hash+, let's fix it.
       #
       def to_indexed_json
         self.to_json
@@ -696,36 +712,31 @@ _Tire_ implements not only _searchable_ features, but also _persistence_ feature
 
 Well, because you're tired of database migrations and lots of hand-holding with your
 database to store stuff like `{ :name => 'Tire', :tags => [ 'ruby', 'search' ] }`.
-Because what you need is to just dump a JSON-representation of your data into a database and
-load it back when needed.
+Because all you need, really, is to just dump a JSON-representation of your data into a database and load it back again.
 Because you've noticed that _searching_ your data is a much more effective way of retrieval
 then constructing elaborate database query conditions.
-Because you have _lots_ of data and want to use _ElasticSearch's_
-advanced distributed features.
+Because you have _lots_ of data and want to use _ElasticSearch's_ advanced distributed features.
 
-To use the persistence features, just include the `Tire::Persistence` module in your class and define the properties (like with _CouchDB_- or _MongoDB_-based models):
+All good reasons to use _ElasticSearch_ as a schema-free and highly-scalable storage and retrieval/aggregation engine for your data.
+
+To use the persistence mode, we'll include the `Tire::Persistence` module in our class and define its properties;
+we can add the standard mapping declarations, set default values, or define casting for the property to create
+lightweight associations between the models.
 
 ```ruby
     class Article
       include Tire::Model::Persistence
-      include Tire::Model::Search
-      include Tire::Model::Callbacks
 
       validates_presence_of :title, :author
 
-      property :title
-      property :author
-      property :content
-      property :published_on
+      property :title,        :analyzer => 'snowball'
+      property :published_on, :type => 'date'
+      property :tags,         :default => [], :analyzer => 'keyword'
+      property :author,       :class => Author
+      property :comments,     :class => [Comment]
     end
 ```
 
-Of course, not all validations or `ActionPack` helpers will be available to your models,
-but if you can live with that, you've just got a schema-free, highly-scalable storage
-and retrieval engine for your data.
-
-This will result in Article instances being stored in an index called 'test_articles' when used in tests but in the index 'development_articles' when used in the development environment.
-
 Please be sure to peruse the [integration test suite](https://github.com/karmi/tire/tree/master/test/integration)
 for examples of the API and _ActiveModel_ integration usage.
 
@@ -734,22 +745,7 @@ Extensions and Additions
 ------------------------
 
 The [_tire-contrib_](http://github.com/karmi/tire-contrib/) project contains additions
-and extensions to the _Tire_ functionality.
-
-
-Todo, Plans & Ideas
--------------------
-
-_Tire_ is already used in production by its authors. Nevertheless, it's not considered finished yet.
-
-There are todos, plans and ideas, some of which are listed below, in the order of importance:
-
-* Proper RDoc annotations for the source code
-* [Statistical](http://www.elasticsearch.org/guide/reference/api/search/facets/statistical-facet.html) facets
-* [Geo Distance](http://www.elasticsearch.org/guide/reference/api/search/facets/geo-distance-facet.html) facets
-* [Index aliases](http://www.elasticsearch.org/guide/reference/api/admin-indices-aliases.html) management
-* [Analyze](http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze.html) API support
-* Embedded webserver to display statistics and to allow easy searches
+and extensions to the core _Tire_ functionality — be sure to check them out.
 
 
 Other Clients

data/examples/tire-dsl.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+#
 # **Tire** provides rich and comfortable Ruby API for the
 # [_ElasticSearch_](http://www.elasticsearch.org/) search engine/database.
 #
@@ -474,6 +476,7 @@ end
 # Eventually, _Tire_ will support all of them. So far, only these are supported:
 #
 # * [string](http://www.elasticsearch.org/guide/reference/query-dsl/query-string-query.html)
+# * [text](http://www.elasticsearch.org/guide/reference/query-dsl/text-query.html)
 # * [term](http://elasticsearch.org/guide/reference/query-dsl/term-query.html)
 # * [terms](http://elasticsearch.org/guide/reference/query-dsl/terms-query.html)
 # * [bool](http://www.elasticsearch.org/guide/reference/query-dsl/bool-query.html)
@@ -532,11 +535,11 @@ s = Tire.search 'articles' do
   #
   query { string 'title:T*' }
 
-  facet 'global-tags' do
+  facet 'global-tags', :global => true do
 
     # ...but set the `global` scope for the facet in this case.
     #
-    terms :tags, :global => true
+    terms :tags
   end
 
   # We can even _combine_ facets scoped to the current query
@@ -583,6 +586,8 @@ end
 # * [date](http://www.elasticsearch.org/guide/reference/api/search/facets/date-histogram-facet.html)
 # * [range](http://www.elasticsearch.org/guide/reference/api/search/facets/range-facet.html)
 # * [histogram](http://www.elasticsearch.org/guide/reference/api/search/facets/histogram-facet.html)
+# * [statistical](http://www.elasticsearch.org/guide/reference/api/search/facets/statistical-facet.html)
+# * [terms_stats](http://www.elasticsearch.org/guide/reference/api/search/facets/terms-stats-facet.html)
 # * [query](http://www.elasticsearch.org/guide/reference/api/search/facets/query-facet.html)
 
 # We have seen that _ElasticSearch_ facets enable us to fetch complex aggregations from our data.

data/lib/tire/configuration.rb

@@ -3,7 +3,7 @@ module Tire
   class Configuration
 
     def self.url(value=nil)
-      @url    = (value ? value.to_s.gsub(%r|/*$|, '') : nil) || @url || "http://localhost:9200"
+      @url = (value ? value.to_s.gsub(%r|/*$|, '') : nil) || @url || ENV['ELASTICSEARCH_URL'] || "http://localhost:9200"
     end
 
     def self.client(klass=nil)

data/lib/tire/dsl.rb

@@ -7,7 +7,7 @@ module Tire
 
     def search(indices=nil, options={}, &block)
       if block_given?
-        Search::Search.new(indices, options, &block).perform
+        Search::Search.new(indices, options, &block)
       else
         payload = case options
           when Hash    then options.to_json

data/lib/tire/http/client.rb

@@ -5,33 +5,44 @@ module Tire
     module Client
 
       class RestClient
+        ConnectionExceptions = [::RestClient::ServerBrokeConnection, ::RestClient::RequestTimeout]
 
         def self.get(url, data=nil)
           perform ::RestClient::Request.new(:method => :get, :url => url, :payload => data).execute
+        rescue *ConnectionExceptions
+          raise
         rescue ::RestClient::Exception => e
           Response.new e.http_body, e.http_code
         end
 
         def self.post(url, data)
           perform ::RestClient.post(url, data)
+        rescue *ConnectionExceptions
+          raise
         rescue ::RestClient::Exception => e
           Response.new e.http_body, e.http_code
         end
 
         def self.put(url, data)
           perform ::RestClient.put(url, data)
+        rescue *ConnectionExceptions
+          raise
         rescue ::RestClient::Exception => e
           Response.new e.http_body, e.http_code
         end
 
         def self.delete(url)
           perform ::RestClient.delete(url)
+        rescue *ConnectionExceptions
+          raise
         rescue ::RestClient::Exception => e
           Response.new e.http_body, e.http_code
         end
 
         def self.head(url)
           perform ::RestClient.head(url)
+        rescue *ConnectionExceptions
+          raise
         rescue ::RestClient::Exception => e
           Response.new e.http_body, e.http_code
         end

data/lib/tire/http/clients/curb.rb

@@ -8,15 +8,22 @@ module Tire
 
       class Curb
         @client = ::Curl::Easy.new
+        @client.resolve_mode = :ipv4
+
         # @client.verbose = true
 
         def self.get(url, data=nil)
           @client.url = url
-          @client.post_body = data
+
           # FIXME: Curb cannot post bodies with GET requests?
           #        Roy Fielding seems to approve:
           #        <http://tech.groups.yahoo.com/group/rest-discuss/message/9962>
-          @client.http_post
+          if data
+            @client.post_body = data
+            @client.http_post
+          else
+            @client.http_get
+          end
           Response.new @client.body_str, @client.response_code
         end
 

data/lib/tire/index.rb

@@ -85,7 +85,7 @@ module Tire
         response = Configuration.client.post("#{Configuration.url}/_bulk", payload.join("\n"))
         raise RuntimeError, "#{response.code} > #{response.body}" if response.failure?
         response
-      rescue Exception => error
+      rescue StandardError => error
         if count < tries
           count += 1
           STDERR.puts "[ERROR] #{error.message}, retrying (#{count})..."
@@ -152,8 +152,8 @@ module Tire
       h = MultiJson.decode(@response.body)
       if Configuration.wrapper == Hash then h
       else
-        document = {}
-        document = h['_source'] ? document.update( h['_source'] ) : document.update( h['fields'] )
+        return nil if h['exists'] == false
+        document = h['_source'] || h['fields'] || {}
         document.update('id' => h['_id'], '_type' => h['_type'], '_index' => h['_index'], '_version' => h['_version'])
         Configuration.wrapper.new(document)
       end
@@ -177,7 +177,7 @@ module Tire
       MultiJson.decode(@response.body)['ok']
 
     ensure
-      curl = %Q|curl -X POST "#{Configuration.url}/#{@name}/open"|
+      curl = %Q|curl -X POST "#{Configuration.url}/#{@name}/_open"|
       logged('_open', curl)
     end
 

data/lib/tire/model/indexing.rb

@@ -38,8 +38,9 @@ module Tire
         #     class Article
         #       # ...
         #       mapping :_source => { :compress => true } do
-        #         indexes :id,    :type => 'string',  :index    => :not_analyzed
-        #         indexes :title, :type => 'string',  :analyzer => 'snowball',   :boost => 100
+        #         indexes :id,    :index    => :not_analyzed
+        #         indexes :title, :analyzer => 'snowball', :boost => 100
+        #         indexes :words, :as       => 'content.split(/\W/).length'
         #         # ...
         #       end
         #     end
@@ -66,6 +67,10 @@ module Tire
         #
         # * Use different analyzer for indexing a property: `indexes :title, :analyzer => 'snowball'`
         #
+        # * Use the `:as` option to dynamically define the serialized property value, eg:
+        #
+        #       :as => 'content.split(/\W/).length'
+        #
         # Please refer to the
         # [_mapping_ documentation](http://www.elasticsearch.org/guide/reference/mapping/index.html)
         # for more information.

data/lib/tire/model/persistence/attributes.rb

@@ -9,11 +9,50 @@ module Tire
 
         module ClassMethods
 
+          # Define property of the model:
+          #
+          #    class Article
+          #      include Tire::Model::Persistence
+          #
+          #      property :title,     :analyzer => 'snowball'
+          #      property :published, :type => 'date'
+          #      property :tags,      :analyzer => 'keywords', :default => []
+          #    end
+          #
+          # You can pass mapping definition for ElasticSearch in the options Hash.
+          #
+          # You can define default property values.
+          #
           def property(name, options = {})
-            attr_accessor name.to_sym
+
+            # Define attribute reader:
+            define_method("#{name}") do
+              instance_variable_get(:"@#{name}")
+            end
+
+            # Define attribute writer:
+            define_method("#{name}=") do |value|
+              instance_variable_set(:"@#{name}", value)
+            end
+
+            # Save the property in properties array:
             properties << name.to_s unless properties.include?(name.to_s)
+
+            # Define convenience <NAME>? method:
             define_query_method      name.to_sym
+
+            # ActiveModel compatibility. NEEDED?
             define_attribute_methods [name.to_sym]
+
+            # Save property default value (when relevant):
+            unless (default_value = options.delete(:default)).nil?
+              property_defaults[name.to_sym] = default_value
+            end
+
+            # Save property casting (when relevant):
+            property_types[name.to_sym] = options[:class] if options[:class]
+
+            # Store mapping for the property:
             mapping[name] = options
             self
           end
@@ -22,6 +61,14 @@ module Tire
             @properties ||= []
           end
 
+          def property_defaults
+            @property_defaults ||= {}
+          end
+
+          def property_types
+            @property_types ||= {}
+          end
+
           private
 
           def define_query_method name
@@ -35,7 +82,14 @@ module Tire
           attr_accessor :id
 
           def initialize(attributes={})
-            attributes.each { |name, value| send("#{name}=", value) }
+            # Make a copy of objects in the property defaults hash, so default values such as `[]` or `{ foo: [] }` are left intact
+            property_defaults = self.class.property_defaults.inject({}) do |hash, item|
+              key, value = item
+              hash[key.to_s] = value.class.respond_to?(:new) ? value.clone : value
+              hash
+            end
+
+            __update_attributes(property_defaults.merge(attributes))
           end
 
           def attributes
@@ -52,6 +106,34 @@ module Tire
           end
           alias :has_property? :has_attribute?
 
+          def __update_attributes(attributes)
+            attributes.each { |name, value| send "#{name}=", __cast_value(name, value) }
+          end
+
+          # Casts the values according to the <tt>:class</tt> option set when
+          # defining the property, cast Hashes as Hashr[http://rubygems.org/gems/hashr]
+          # instances and automatically convert UTC formatted strings to Time.
+          #
+          def __cast_value(name, value)
+            case
+
+              when klass = self.class.property_types[name.to_sym]
+                if klass.is_a?(Array) && value.is_a?(Array)
+                  value.map { |v| klass.first.new(v) }
+                else
+                  klass.new(value)
+                end
+
+              when value.is_a?(Hash)
+                Hashr.new(value)
+
+              else
+                # Strings formatted as <http://en.wikipedia.org/wiki/ISO8601> are automatically converted to Time
+                value = Time.parse(value) if value.is_a?(String) && value =~ /^\d{4}[\/\-]\d{2}[\/\-]\d{2}T\d{2}\:\d{2}\:\d{2}Z$/
+                value
+            end
+          end
+
         end
 
       end

data/lib/tire/model/persistence/finders.rb

@@ -16,9 +16,12 @@ module Tire
             options = args.pop if args.last.is_a?(Hash)
             args.flatten!
             if args.size > 1
-              Tire::Search::Search.new(index.name).query do |query|
-                query.ids(args, document_type)
-              end.perform.results
+              Tire::Search::Search.new(index.name) do |search|
+                search.query do |query|
+                  query.ids(args, document_type)
+                end
+                search.size args.size
+              end.results
             else
               case args = args.pop
                 when Fixnum, String
@@ -38,7 +41,7 @@ module Tire
             old_wrapper = Tire::Configuration.wrapper
             Tire::Configuration.wrapper self
             s = Tire::Search::Search.new(index.name).query { all }
-            s.perform.results
+            s.results
           ensure
             Tire::Configuration.wrapper old_wrapper
           end
@@ -48,7 +51,7 @@ module Tire
             old_wrapper = Tire::Configuration.wrapper
             Tire::Configuration.wrapper self
             s = Tire::Search::Search.new(index.name).query { all }.size(1)
-            s.perform.results.first
+            s.results.first
           ensure
             Tire::Configuration.wrapper old_wrapper
           end

data/lib/tire/model/persistence/storage.rb

@@ -30,14 +30,12 @@ module Tire
         module InstanceMethods
 
           def update_attribute(name, value)
-            send("#{name}=", value)
+            __update_attributes name => value
             save
           end
 
           def update_attributes(attributes={})
-            attributes.each do |name, value|
-              send("#{name}=", value)
-            end
+            __update_attributes attributes
             save
           end
 

data/lib/tire/model/persistence.rb

@@ -50,6 +50,18 @@ module Tire
             define_method("#{attr}")  { @attributes[attr] }
           end
 
+          def self.search(*args, &block)
+            # Update options Hash with the wrapper definition
+            args.last.update(:wrapper => self) if args.last.is_a? Hash
+            args << { :wrapper => self }       unless args.any? { |a| a.is_a? Hash }
+
+            self.__search_without_persistence(*args, &block)
+          end
+
+          def self.__search_without_persistence(*args, &block)
+            self.tire.search(*args, &block)
+          end
+
         end
 
       end

data/lib/tire/model/search.rb

@@ -94,7 +94,7 @@ module Tire
             s.fields Array(options[:fields]) if options[:fields]
           end
 
-          s.perform.results
+          s.results
         end
 
         # Returns a Tire::Index instance for this model.
@@ -151,13 +151,29 @@ module Tire
         # If you do define the mapping for _ElasticSearch_, only attributes
         # declared in the mapping are serialized.
         #
+        # For properties declared with the `:as` option, the passed String or Proc
+        # is evaluated in the instance context.
+        #
         def to_indexed_json
           if instance.class.tire.mapping.empty?
+            # Reject the id and type keys
             instance.to_hash.reject {|key,_| key.to_s == 'id' || key.to_s == 'type' }.to_json
           else
-            instance.to_hash.
-            reject { |key, value| ! instance.class.tire.mapping.keys.map(&:to_s).include?(key.to_s) }.
-            to_json
+            mapping = instance.class.tire.mapping
+            # Reject keys not declared in mapping
+            hash = instance.to_hash.reject { |key, value| ! mapping.keys.map(&:to_s).include?(key.to_s) }
+
+            # Evalute the `:as` options
+            mapping.each do |key, options|
+              case options[:as]
+                when String
+                  hash[key] = instance.instance_eval(options[:as])
+                when Proc
+                  hash[key] = instance.instance_eval(&options[:as])
+              end
+            end
+
+            hash.to_json
           end
         end
 

data/lib/tire/search/facet.rb

@@ -11,7 +11,7 @@ module Tire
       def initialize(name, options={}, &block)
         @name    = name
         @options = options
-        self.instance_eval(&block) if block_given?
+        block.arity < 1 ? self.instance_eval(&block) : block.call(self) if block_given?
       end
 
       def terms(field, options={})
@@ -37,6 +37,16 @@ module Tire
         self
       end
 
+      def statistical(field, options={})
+        @value = { :statistical => (options.delete(:statistical) || {:field => field}.update(options)) }
+        self
+      end
+
+      def terms_stats(key_field, value_field, options={})
+        @value = { :terms_stats => {:key_field => key_field, :value_field => value_field}.update(options) }
+        self
+      end
+
       def query(&block)
         @value = { :query => Query.new(&block).to_hash }
       end

data/lib/tire/search/query.rb

@@ -21,6 +21,12 @@ module Tire
         @value = { :range => { field => value } }
       end
 
+      def text(field, value, options={})
+        query_options = { :query => value }.update(options)
+        @value = { :text => { field => query_options } }
+        @value
+      end
+
       def string(value, options={})
         @value = { :query_string => { :query => value } }
         @value[:query_string].update(options)
@@ -121,8 +127,9 @@ module Tire
       end
 
       def filter(type, *options)
-        @value[:filter] ||= []
-        @value[:filter] << Filter.new(type, *options).to_hash
+        @value[:filter] ||= {}
+        @value[:filter][:and] ||= []
+        @value[:filter][:and] << Filter.new(type, *options).to_hash
         @value
       end