repertoire-faceting 0.5.5 → 0.6.0

data/lib/repertoire-faceting/facets/abstract_facet.rb

@@ -3,54 +3,70 @@ require 'active_support/core_ext/object/blank'
 module Repertoire
   module Faceting
     module Facets #:nodoc:
-      
+
       # Abstract interface all facet implementations must fulfil.  At minimum, implementors should
       # define self.claim?(), signature(), and drill() to create a new kind of facet.
       #
-      # For indexing support, implement create_index to create your index table, detect its presence
-      # in signature() and drill(), and act accordingly.
+      # For indexing support, implement create_index(), refresh_index(), and drop_index(). Detect the
+      # index's presence in signature() and drill(), and act accordingly.
+      #
+      # N.B. Facet instances can assume they have been mixed in with an ActiveRecord::Relation and
+      #      Repertoire::Faceting::Model (see the mixin() method below). Think of them as scoped
+      #      relations that identify an attribute of the model dataset.
       #
       # See BasicFacet and NestedFacet for examples of facet implementations.
+      #
       module AbstractFacet
-        
+
         attr_accessor :facet_name
-      
+
         # Return true if this facet implementation can index this ActiveRecord relation.  If multiple
-        # implementations claim a facet, the one that is laoded last wins.  
+        # implementations claim a facet, the one that is laoded last wins.
         def self.claim?(relation)
-          raise 'Please implement claim? for your facet'
+          raise "Please implement claim? for your facet"
         end
 
         # Return an arel expression for the signature of all models matching the given current refinement state for
-        # this facet (and ignoring all others).
+        # this facet (and ignoring all others). Signatures should be constructed from the column returned by
+        # faceting_id().
         def signature(state)
-          raise 'Please implement signature for your facet'
+          raise "Please implement signature for your facet"
         end
-        
-        # Return an arel expression for state, signature pairs from which one can refine the current state of this facet
-        # (ignoring all others)
+
+        # Return an arel expression for (state, signature) pairs from which one can refine the current state of this facet
+        # (ignoring all others). Signatures should be constructed from the column returned by faceting_id().
         def drill(state)
-          raise 'Please implement drill for your facet'
+          raise "Please implement drill for your facet"
         end
 
-        # Return an arel expression describing this facet's index table, or raise an exception if indexing not supported
-        #
-        # Implementations should build signatures using the faceting_id column passed in, which may not exist yet on the
-        # model table itself.
-        def create_index(faceting_id)
-          raise 'Facet #{facet_name} does not support indexing'
+        # Create this facet's index table, or raise an exception if indexing not supported. Signatures should be
+        # constructed from the column returned by faceting_id().
+        def create_index
+          raise "Facet #{facet_name} does not support indexing"
         end
         
-        protected
+        # Refresh this facet's index, or raise an exception if it is not indexed
+        def refresh_index
+          raise "Facet #{facet_name} is not indexed" unless facet_indexed?
+          connection.refresh_materialized_view(facet_index_name)
+        end
         
-        # Return the facet's index table name
-        def facet_index_table
-          connection.facet_table_name(table_name, facet_name).to_sym
+        # Drop this facet's index, or raise an exception if it is not indexed
+        def drop_index
+          raise "Facet #{facet_name} is not indexed" unless facet_indexed?
+          connection.drop_materialized_view(facet_index_name)
         end
 
         # Returns true if the facet's index table exists
-        def indexed?
-          connection.indexed_facets(table_name).map(&:to_sym).include?(facet_name)
+        def facet_indexed?
+          indexed_facets.map(&:to_sym).include?(facet_name)
+        end
+
+        protected
+
+        # Return a facet's index table name
+        def facet_index_name
+          connection.facet_table_name(table_name, facet_name)
         end
 
         def self.implementations
@@ -70,10 +86,10 @@ module Repertoire
           # mix in facet implementation and configure
           relation.singleton_class.send(:include, impls.last)
           relation.facet_name = name
-          
+
           relation
         end
-      
+
       end
     end
   end

data/lib/repertoire-faceting/facets/basic_facet.rb

@@ -1,8 +1,8 @@
 module Repertoire #:nodoc:
   module Faceting #:nodoc:
     module Facets #:nodoc:
-      
-      # Implementation of AbstractFacet for non-nested, single-valued facets.  By default, 
+
+      # Implementation of AbstractFacet for non-nested, single-valued facets.  By default,
       # all facets that have a single group column will follow this behavior.
       #
       # See Repertoire::Faceting::Model::ClassMethods for usage.
@@ -10,56 +10,57 @@ module Repertoire #:nodoc:
       module BasicFacet
         include AbstractFacet
         include Arel
-      
+
         def self.claim?(relation)
           relation.group_values.size == 1
         end
-        
+
         def signature(state)
-          return read_index(state, true)             if indexed?
+          return read_index(state, true)             if facet_indexed?
           col = group_values.first
           rel = only(:where, :joins)
           rel = rel.where(in_clause(col, state))     unless state.empty?
-          rel.select("signature(#{table_name}.#{faceting_id})").arel
+          rel.select("facet.signature(#{table_name}.#{faceting_id})").arel
         end
-      
+
         def drill(state)
-          return read_index(state, false)            if indexed?
-          col = group_values.first          
+          return read_index(state, false)            if facet_indexed?
+          col = group_values.first
           rel = only(:where, :joins, :group)
           rel = rel.where(in_clause(col, state))     unless state.empty?
-          rel.select(["#{col} AS #{facet_name}", "signature(#{table_name}.#{faceting_id})"]).arel
+          rel.select(["#{col} AS #{facet_name}", "facet.signature(#{table_name}.#{faceting_id})"]).arel
         end
-      
-        def create_index(faceting_id)
+
+        def create_index
           col = group_values.first
-          sql = only(:where, :joins, :group).select(["#{col} AS #{facet_name}", "signature(#{table_name}.#{faceting_id})"]).to_sql
-          
-          connection.recreate_table(facet_index_table, sql)
+          rel = only(:where, :joins, :group)
+          sql = rel.select(["#{col} AS #{facet_name}", "facet.signature(#{table_name}.#{faceting_id})"]).to_sql
+
+          connection.create_materialized_view(facet_index_name, sql)
         end
 
         private
-        
+
         def in_clause(col, values)
           # ActiveRecord unhelpfully scatters wrong table names in predicates...
           values = values.map { |v| connection.quote(v) }
           "#{col} IN (#{values.join(', ')})"
         end
-      
+
         def read_index(state, aggregate)
-          index = Arel::Table.new(facet_index_table)
+          index = Arel::Table.new(facet_index_name)
           rel   = SelectManager.new Table.engine
-          
+
           rel.from index
           rel.where(index[facet_name].in(state)) unless state.empty?
-        
+
           if aggregate
-            rel.project('collect(signature) AS signature')
+            rel.project('facet.collect(signature) AS signature')
           else
             rel.project(index[facet_name], index[:signature])
           end
         end
-      
+
       end
     end
   end

data/lib/repertoire-faceting/facets/nested_facet.rb

@@ -1,7 +1,7 @@
 module Repertoire
   module Faceting
     module Facets
-      
+
       # Implementation of AbstractFacet for facets whose values fall into a nested taxonomy.
       # By default, all facets that group several columns will follow this behavior.
       #
@@ -10,66 +10,90 @@ module Repertoire
       module NestedFacet
         include AbstractFacet
         include Arel
-      
+
         def self.claim?(relation)
           relation.group_values.size > 1
         end
-      
+
         def signature(state)
-          return read_index(state, true) if indexed?
+          return read_index(state, true) if facet_indexed?
           rel = only(:where, :joins)
           bind_nest(group_values, state) do |expr, val|
             rel = rel.where("#{expr} = #{connection.quote(val)}")
           end
-          rel.select("signature(#{table_name}.#{faceting_id})").arel
+          rel.select("facet.signature(#{table_name}.#{faceting_id})").arel
         end
-        
+
         def drill(state)
-          return read_index(state, false) if indexed?
+          return read_index(state, false) if facet_indexed?
           rel = only(:where, :joins)
           grp = bind_nest(group_values, state) do |expr, val|
             rel = rel.where("#{expr} = #{connection.quote(val)}")
           end
-          rel.group(grp).select(["#{grp.last} AS #{facet_name}", "signature(#{table_name}.#{faceting_id})"]).arel
+          rel.group(grp).select(["#{grp.last} AS #{facet_name}", "facet.signature(#{table_name}.#{faceting_id})"]).arel
         end
-      
-        def create_index(faceting_id)
-          rel = only(:where, :joins, :group)
-          group_values.zip(columns).each do |expr, col|
-            rel = rel.select("#{expr} AS #{col}")
+
+        def create_index
+          levels = group_values.length
+
+          # Construct expressions at each grouping level, right pad with nil
+          expr_drills = (0..levels-1).map do |i|
+            group_values[0..i] + (i+1..levels-1).collect { "NULL" }
+          end
+
+          # Construct indexes for each drill level
+          queries = []
+          expr_drills.each_with_index do |exprs, level|
+            rel = only(:where, :joins)
+            exprs.zip(columns).each do |expr, col|
+              rel = rel.select("#{expr} AS #{col}")
+            end
+            rel = rel.group(columns[0..level])
+
+            queries << rel.select(["#{level+1} AS level", "facet.signature(#{table_name}.#{faceting_id})"]).to_sql
           end
-          sql = rel.select(["#{group_values.size} AS level", "signature(#{table_name}.#{faceting_id})"]).to_sql
-          
-          connection.recreate_table(facet_index_table, sql)
-          connection.expand_nesting(facet_index_table)
+
+          # Root of tree
+          empty_cols = columns.map { |col| "NULL AS #{col}"}
+          queries << only(:where).select(empty_cols + ["0 AS level", "facet.signature(#{table_name}.#{faceting_id})"]).to_sql
+
+          # Give the fullest index first (i.e. leaves of the tree), so the database
+          # can resolve types before encountering any NULL values (i.e. values of
+          # indeterminate type)
+          queries = queries.reverse
+
+          # The full index table is union of indices at each drill level
+          sql = queries.join(" UNION ")
+
+          connection.create_materialized_view(facet_index_name, sql)
         end
-        
+
         private
 
         def columns
-          (1..group_values.size).map { |i| "#{facet_name}#{i}"}
+          (1..group_values.size).map { |i| "#{facet_name}_#{i}"}
         end
-        
+
         # Iterates over all the columns that have state in turn, and returns
         # a grouping of the columns one level further nested
         def bind_nest(cols, state, &block)
           level = state.size
           grp   = cols[0..level]                      # advance one nest step
-                    
+
           if level > 0
-            cols[0..level-1].zip(state).each do |col, val| 
+            cols[0..level-1].zip(state).each do |col, val|
               yield(col, val)
             end
           end
           grp << "NULL::TEXT" if level >= cols.size
-          
+
           grp
         end
-      
+
         def read_index(state, aggregate)
-          index = Table.new(facet_index_table)
+          index = Table.new(facet_index_name)
           rel   = SelectManager.new Table.engine
-          
+
           rel.from index
           grp = bind_nest(columns, state) do |col, val|
             rel.where(index[col].eq(val))
@@ -84,7 +108,6 @@ module Repertoire
             rel.project("#{grp.last} AS #{facet_name}", index[:signature])
           end
         end
-      
       end
     end
   end

data/lib/repertoire-faceting/model.rb

@@ -2,15 +2,18 @@ module Repertoire
   module Faceting
     module Model #:nodoc:
       extend ActiveSupport::Concern
-      
+
       SIGNATURE_WASTAGE_THRESHOLD = 0.15
       DEFAULT_SIGNATURE_COLUMN    = 'id'
       PACKED_SIGNATURE_COLUMN     = '_packed_id'
-      
+
       included do |base|
-        base.singleton_class.delegate :refine, :minimum, :nils, :reorder, :to_sql, :to => :scoped
+        base.singleton_class.delegate :refine, :minimum, :nils, :reorder, :to_sql, :to => :scoped_all
+
+        base.class_attribute(:facets)
+        base.facets = {}
       end
-      
+
       #
       # == Facet declarations
       #
@@ -28,12 +31,12 @@ module Repertoire
       #   end
       #
       # Implicitly, any facet declaration is an SQL aggregate that divides the attribute values into discrete
-      # groups.  When no relation is provided, /model/.group(/facet name/) is assumed by default.  So the 
+      # groups.  When no relation is provided, /model/.group(/facet name/) is assumed by default.  So the
       # discipline facet declaration above is equivalent to
       #
       #   facet :discipline, group(:discipline)
       #
-      # and the grouping on degree could be left out.  You can use this behavior to construct a facet 
+      # and the grouping on degree could be left out.  You can use this behavior to construct a facet
       # from differently-named columns:
       #
       #   facet :balloon_color, group(:color)
@@ -80,13 +83,13 @@ module Repertoire
       # To incorporate refinements on other facets on this model, use refine:
       #
       #   Nobelist.refine(:nobel_year => 2001, :degree => 'Ph.D.').count(:discipline)
-      # 
+      #
       # If you provide multiple values for a simple facet refinement, they are interpreted as a logical "or":
       #
       #   Nobelist.refine(:nobel_year => [2000, 2001])     # => 'WHERE name IN (2000, 2001)'
       #
       # In the case of a nested facet, multiple values identify levels in the taxonomy:
-      # 
+      #
       #   Nobelist.refine(:birth_place => [ 'Ukraine', 'Kiev' ]).count(:nobel_year)
       #
       # Refinements are integrated into result queries automatically:
@@ -96,9 +99,9 @@ module Repertoire
       # == Index access
       #
       # As you will have noted already, facet counts and queries are quite similar to their ActiveRecord/SQL
-      # counterparts.  Behind the scenes, the Repertoire faceting code re-writes your query.  
+      # counterparts.  Behind the scenes, the Repertoire faceting code re-writes your query.
       #
-      # Facets defined on associations are joined and limited automatically, and facet indices in the database 
+      # Facets defined on associations are joined and limited automatically, and facet indices in the database
       # are used wherever possible rather than querying the model table.
       #
       # == Facet registration
@@ -112,114 +115,147 @@ module Repertoire
       # See AbstractFacet for more details.
       #
       module ClassMethods
-        
+
         # Declare a facet by name
         def facet(name, rel=nil)
           name = name.to_sym
 
           # default: group by column with facet name, order by count descending
-          rel ||= scoped
+          rel ||= scoped_all
           rel = rel.group(name)                           if rel.group_values.empty?
           rel = rel.order(["count DESC", "#{name} ASC"])  if rel.order_values.empty?
 
           # locate facet implementation that can handle relation
           facets[name] = Facets::AbstractFacet.mixin(name, rel)
         end
-        
-        # Accessor for the facet definitions
-        def facets
-          read_inheritable_attribute(:facets) || write_inheritable_attribute(:facets, {})
-        end
-        
+
         # Is there a facet by this name?
         def facet?(name)
           facets.key?(name.to_sym)
         end
-        
+
         # All defined facets by name
         def facet_names
           facets.keys
         end
-        
-        # Drops any unused facet indices, updates its packed ids, then recreates 
-        # indices for the facets with the provided names.  If no names are provided, 
+
+        # Returns a list of the facets that currently have indices declared
+        def indexed_facets
+          connection.indexed_facets(table_name)
+        end
+
+        # Drops any unused facet indices, updates its packed ids, then recreates
+        # indices for the facets with the provided names.  If no names are provided,
         # then the existing facet indices are refreshed.
         #
-        # If a signature id column name is provided, it will be used to build the 
-        # bitset indices. Otherwise the indexer will add or remove a new packed 
+        # If a signature id column name is provided, it will be used to build the
+        # bitset indices. Otherwise the indexer will add or remove a new packed
         # id column as appropriate.
         #
         # Examples:
         #
         # === Refresh existing facet indices
         #
-        #   Nobelist.update_indexed_facets
+        #   Nobelist.index_facets
         #
         # === Adjust which facets are indexed
         #
-        #   Nobelist.update_indexed_facets([:degree, :nobel_year])
+        #   Nobelist.index_facets([:degree, :nobel_year])
         #
         # === Drop all facet indices, but add/remove packed id as necessary
         #
-        #   Nobelist.update_indexed_facets([])
+        #   Nobelist.index_facets([])
         #
-        # === Drop absolutely everything, force manual faceting using 'id' 
+        # === Drop absolutely everything, force manual faceting using 'id'
         #     column
         #
-        #   Nobelist.udpate_indexed_facets([], 'id')
+        #   Nobelist.index_facets([], 'id')
         #
-        def update_indexed_facets(facet_names=nil, signature_column=nil)
+        def index_facets(next_indexes=nil, next_faceting_id=nil)
           # default: update existing facets
-          indexed_facets = connection.indexed_facets(table_name)
-          facet_names ||= indexed_facets
-          
+          current_indexes = indexed_facets
+          next_indexes ||= current_indexes
+
+          # sanity checks
+          current_indexes = current_indexes.map { |name| name.to_sym }
+          next_indexes    = next_indexes.map    { |name| name.to_sym }
+          (current_indexes | next_indexes).each do
+            |name| raise QueryError, "Unknown facet #{name}" unless facet?(name)
+          end
+
           # determine best column for signature bitsets, unless set manually
-          signature_column ||= if signature_wastage('id') < SIGNATURE_WASTAGE_THRESHOLD
+          next_faceting_id ||= if signature_wastage(DEFAULT_SIGNATURE_COLUMN) < SIGNATURE_WASTAGE_THRESHOLD
             DEFAULT_SIGNATURE_COLUMN
           else
             PACKED_SIGNATURE_COLUMN
           end
-          
+
+          # default behavior: no changes to packed id column
+          drop_packed_id = create_packed_id = false
+
+          # default behavior: adjust facet indexes
+          drop_list    = current_indexes - next_indexes
+          refresh_list = next_indexes & current_indexes
+          create_list  = next_indexes - current_indexes
+
+          # adding or removing a packed id column
+          if next_faceting_id != faceting_id
+            drop_packed_id   = (next_faceting_id == DEFAULT_SIGNATURE_COLUMN)
+            create_packed_id = (next_faceting_id != DEFAULT_SIGNATURE_COLUMN)
+          end
+
+          # special case: repacking an existing packed id column
+          if next_faceting_id == faceting_id && next_faceting_id != DEFAULT_SIGNATURE_COLUMN
+            drop_packed_id = create_packed_id = (signature_wastage > SIGNATURE_WASTAGE_THRESHOLD)
+          end
+
+          # changing item ids invalidates all existing facet indices
+          if drop_packed_id || create_packed_id
+            drop_list, refresh_list, create_list = [ current_indexes, [], next_indexes ]
+          end
+
           connection.transaction do
-            # drop old facet indices
-            indexed_facets.each do |name|
-              table = connection.facet_table_name(table_name, name)
-              connection.drop_table(table)
-            end
-            
-            # create or remove packed signature column as necessary
-            ensure_numbering(signature_column)
-
-            # re-create the facet indices
-            facet_names.each do |name|
-              name = name.to_sym
-              raise "Unknown facet #{name}" unless facet?(name)
-              facets[name].create_index(signature_column)
-            end
+            # adjust faceting id column
+            connection.remove_column(table_name, PACKED_SIGNATURE_COLUMN)           if drop_packed_id
+            connection.add_column(table_name, PACKED_SIGNATURE_COLUMN, "SERIAL")    if create_packed_id
+            @faceting_id = next_faceting_id
+
+            # adjust facet indices
+            drop_list.each    { |name| facets[name].drop_index }
+            refresh_list.each { |name| facets[name].refresh_index }
+            create_list.each  { |name| facets[name].create_index }
           end
-          
+
+          # TODO. in a nested transaction, this would need to fire after the final commit...
           reset_column_information
+
         end
-        
+
+        # Over-rides reset_column_information in ActiveRecord::ModelSchema
+        def reset_column_information
+          @faceting_id = nil
+          super
+        end
+
         # Returns the name of the id column to use for constructing bitset signatures
         # over this model.
         def faceting_id
-          [PACKED_SIGNATURE_COLUMN, DEFAULT_SIGNATURE_COLUMN].detect { |c| column_names.include?(c) }
+          @faceting_id ||= [PACKED_SIGNATURE_COLUMN, DEFAULT_SIGNATURE_COLUMN].detect { |c| column_names.include?(c) }
         end
-        
-        def signature_wastage(col=nil)
-          col ||= faceting_id
-          connection.signature_wastage(table_name, col)
+
+        # Returns the proportion of wasted slots in 0..max(id)
+        def signature_wastage(signature_column = nil)
+          signature_column ||= faceting_id
+          connection.signature_wastage(table_name, signature_column)
         end
-        
-        def ensure_numbering(signature_column)
-          if signature_column == DEFAULT_SIGNATURE_COLUMN
-            connection.remove_column(table_name, PACKED_SIGNATURE_COLUMN) if column_names.include?(PACKED_SIGNATURE_COLUMN)
-          else
-            connection.renumber_table(table_name, PACKED_SIGNATURE_COLUMN, SIGNATURE_WASTAGE_THRESHOLD)
-          end
+
+        # Once clients have migrated to Rails 4, delete and replace with 'all' where this is called
+        #
+        # c.f. http://stackoverflow.com/questions/18198963/with-rails-4-model-scoped-is-deprecated-but-model-all-cant-replace-it
+        def scoped_all
+          where(nil)
         end
-        
+
       end
     end
   end

data/lib/repertoire-faceting/rails/engine.rb

@@ -0,0 +1,8 @@
+require 'rails'
+
+module Repertoire
+  module Faceting
+    class Engine < ::Rails::Engine
+    end
+  end
+end

data/lib/repertoire-faceting/rails/postgresql_adapter.rb

@@ -7,7 +7,6 @@ module ActiveRecord #:nodoc: all
     end
     
     class PostgreSQLAdapter
-      include Repertoire::Faceting::AbstractAdapter
       include Repertoire::Faceting::PostgreSQLAdapter
     end
   end

data/lib/repertoire-faceting/rails/relation.rb

@@ -4,7 +4,6 @@ require 'active_record'
 module ActiveRecord #:nodoc: all
   class Relation
     include Repertoire::Faceting::Relation::QueryMethods
-    include Repertoire::Faceting::Relation::SpawnMethods
     include Repertoire::Faceting::Relation::Calculations
   end
 end

data/lib/repertoire-faceting/railtie.rb

@@ -1,4 +1,3 @@
-require 'repertoire-faceting'
 require 'rails'
 
 module Repertoire

data/lib/repertoire-faceting/relation/calculations.rb

@@ -9,9 +9,14 @@ module Repertoire
         def count(name = nil, options = {})
           if name.present? && @klass.facet?(name)
             name       = name.to_sym
-            facet      = @klass.facets[name].merge(self)
+            parent     = @klass.facets[name]
+            facet      = parent.merge(self)
             state      = refine_value[name] || []
             signatures = facet.drill(state)
+
+            facet.minimum_value = self.minimum_value || parent.minimum_value
+            facet.nils_value    = self.nils_value    || parent.nils_value
+
             connection.population(facet, masks, signatures)
           else
             super
@@ -39,7 +44,7 @@ module Repertoire
           base = except(:order, :limit, :offset)
           masks = []
           
-          masks << base.only(:where, :join).select("signature(#{table_name}.#{faceting_id})").arel  if base.where_values.present?
+          masks << base.only(:where, :join).select("facet.signature(#{table_name}.#{faceting_id})").arel  if base.where_values.present?
           refine_value.each do |name, values|
             raise QueryError, "Unkown facet #{name} on #{klass.name}"                 unless @klass.facet?(name)
             masks << @klass.facets[name].signature(values)