spontaneous 0.1.0.alpha1 → 0.2.0.alpha1

data/lib/spontaneous/schema/uid_map.rb

@@ -0,0 +1,120 @@
+# encoding: UTF-8
+
+
+require 'base58'
+
+module Spontaneous::Schema
+  class UIDMap
+
+    def self.uid_lock
+      @uid_lock ||= Mutex.new
+    end
+
+    def self.uid_index
+      @uid_index ||= 0
+    end
+
+    def self.increment_uid_index
+      @uid_index = (uid_index + 1) % 0xFFFF
+    end
+
+    def self.get_inc
+      uid_lock.synchronize do
+        increment_uid_index
+      end
+    end
+
+    def self.generate(ref = nil)
+      generate58(ref)
+    end
+
+    def self.generate58(ref)
+      # reverse the time so that sequential ids are more obviously different
+      oid =  Base58.encode((Time.now.to_f * 1000).to_i).reverse
+      oid << Base58.encode(get_inc).rjust(3, '0')
+    end
+
+    def self.generate16(ref)
+      oid = ''
+      # 4 bytes current time
+      oid = (Time.now.to_f * 1000).to_i.to_s(16)
+      # 2 bytes inc
+      oid << get_inc.to_s(16).rjust(4, '0')
+    end
+
+    include Enumerable
+
+    def initialize
+      @instance_lock  = Mutex.new
+      @instances = {}
+    end
+
+    def load(id, reference)
+      @instance_lock.synchronize do
+        unless instance = @instances[id]
+          instance = UID.new(self, id, reference)
+          # check to make sure that the uid is valid first
+          # this ensures that UIDs are not created before their owners
+          # have uids
+          if instance.valid?
+            @instances[id] = instance
+          else
+            instance = nil
+          end
+        end
+        instance
+      end
+    end
+
+    def create(reference)
+      unless instance = @instances.values.detect { |i| i.reference == reference }
+        instance = load(self.class.generate(reference), reference)
+      end
+      instance
+    end
+
+    def existing_references
+      @instances.values.map { |i| i.reference }
+    end
+
+    def create_for(obj)
+      id = create(obj.schema_name)
+      obj.update_schema_id(id) if obj.respond_to?(:update_schema_id)
+    end
+
+    def destroy(uid)
+      @instance_lock.synchronize do
+        @instances.delete(uid.to_s)
+        uid.after_destroy
+      end
+    end
+
+    def clear!
+      @instances = {}
+    end
+
+    def [](id)
+      return id if UID === id
+      return nil if id.blank?
+      @instances[id]
+    end
+
+    def get_id(reference)
+      self.find { |uid| uid.reference == reference }
+    end
+
+    def each
+      uids = @instances.map { |id, instance| instance }
+      if block_given?
+        uids.each { |instance| yield(instance) }
+      else
+        uids.each
+      end
+    end
+
+
+    def export
+      Hash[ @instances.map { |id, ref| [id, ref.reference] } ]
+    end
+  end # UIDMap
+end

data/lib/spontaneous/search.rb

@@ -0,0 +1,11 @@
+# encoding: UTF-8
+
+module Spontaneous
+  module Search
+    autoload :Index,    'spontaneous/search/index'
+    autoload :Field,    'spontaneous/search/field'
+    autoload :Database, 'spontaneous/search/database'
+    autoload :Results,  'spontaneous/search/results'
+    autoload :CompoundIndexer, 'spontaneous/search/compound_indexer'
+  end
+end

data/lib/spontaneous/search/compound_indexer.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+
+module Spontaneous::Search
+  class CompoundIndexer
+    def initialize(revision, indexes)
+      @revision, @indexes, @dbs = revision, indexes, indexes.map { |index| index.create_db(revision) }
+    end
+
+    def length
+      @indexes.length
+    end
+    alias_method :count, :length
+    alias_method :size,  :length
+
+    def add(page)
+      @dbs.each { |db| db << page }
+    end
+
+    alias_method :<<, :add
+
+    def close
+      @dbs.each { |db| db.close }
+    end
+  end # CompoundIndexer
+end

data/lib/spontaneous/search/database.rb

@@ -0,0 +1,72 @@
+# encoding: UTF-8
+
+require 'xapian-fu'
+
+module Spontaneous::Search
+  class Database
+    def self.create(index, revision)
+      db = self.new(index, revision)
+      db.create
+      db
+    end
+
+    def initialize(index, revision)
+      @index, @revision, @database = index, revision, nil
+    end
+
+    def create
+      FileUtils.mkdir_p(directory)
+      @database = XapianFu::XapianDb.new(xapian_options.merge({
+        :create => true,
+        :overwrite => true
+      }))
+    end
+
+    def xapian_options
+      options = { :dir => directory,
+        :language => @index.language,
+        :fields => @index.fields,
+        :spelling => true }
+      options[:stemmer] = @index.stemmer unless @index.stemmer.nil?
+      options[:stopper] = @index.stopper unless @index.stopper.nil?
+      options
+    end
+
+    def database
+      @database ||= open_database
+    end
+
+    def open_database
+      XapianFu::XapianDb.new(xapian_options)
+    end
+
+    def add(page)
+      if @index.include?(page)
+        database << @index.indexable_content(page)
+      end
+    end
+
+    def close
+      database.flush
+    end
+
+    def search(query, options = {})
+      xapian_results = database.search(query, options)
+      autocorrect = options.delete(:autocorrect)
+      corrected_query = xapian_results.corrected_query
+      # don't know if this is the right way to deal with the corrected query suggestion
+      # but I would prefer that any searches act more like google and return a set of
+      # results for a spelling correction if none have been found with the current query
+      if autocorrect and xapian_results.empty? and !corrected_query.blank?
+        xapian_results = database.search(corrected_query, options)
+      end
+      Results.new(xapian_results, corrected_query)
+    end
+
+    alias_method :<<, :add
+
+    def directory
+      S::Site.revision_dir(@revision) / "indexes" / @index.name
+    end
+  end # Database
+end

data/lib/spontaneous/search/field.rb

@@ -0,0 +1,95 @@
+# encoding: UTF-8
+
+module Spontaneous::Search
+  class Field
+    attr_reader :prototype, :options
+
+    def initialize(prototype, options)
+      @prototype = prototype
+      @options = options
+    end
+
+    def in_index?(index)
+      indexes.any? { |opts| opts[:index] == index }
+    end
+
+    def index_id(index)
+      opts = options_for_index(index)
+      opts[:group] || prototype.schema_id.to_s
+    end
+
+    def options_for_index(index)
+      indexes.detect { |opts| opts[:index] == index }
+    end
+
+    WEIGHTINGS = {
+      :normal  => 1,
+      :high    => 4,
+      :higher  => 8,
+      :highest => 16
+    }.freeze unless defined?(WEIGHTINGS)
+
+    def field_definition(index)
+      defn = {:type => String, :store => true, :weight => 1, :index => true}
+      opts = options_for_index(index)
+      case opts[:weight]
+      when :store
+        defn.merge!({
+          :index => false,
+          :store => true
+        })
+      when Fixnum
+        defn.merge!({
+          :weight => opts[:weight]
+        })
+      when Symbol
+        defn.merge!({
+          :weight => WEIGHTINGS[opts[:weight]] || 1
+        })
+      end
+      defn
+    end
+
+    def indexes
+      @indexes ||= parse_indexes(options)
+    end
+
+    # convert the options passed to :index in a field definition to a list of
+    # options. Each entry contains the index to which it pertains and then
+    # the options that should be applied to this field within that index
+    def parse_indexes(opts)
+      all_indexes = S::Site.indexes.values
+      case opts
+      when  true
+        all_indexes.map { |index| default_index_options(index) }
+      when Symbol
+        index = find_index(opts)
+        [default_index_options(index)]
+      when Hash
+        if (index_name = opts[:name])
+          [ opts.merge(:index => find_index(index_name)) ]
+        else
+          all_indexes.map { |index| default_index_options(index).merge(opts) }
+        end
+      when Array
+        opts.map { |o| parse_indexes(o) }.flatten
+      else
+        []
+      end
+    end
+
+    def default_index_options(index)
+      index_options(index)
+    end
+
+    def index_options(index, weight = 1, group = nil)
+      {:name => index.name, :index => index, :weight => weight, :group => group}
+    end
+
+    def find_index(name)
+      index = S::Site.indexes[name]
+      logger.warn("Invalid index :#{opts} for field #{@prototype.owner}.#{@prototype.name}") unless index
+      index
+    end
+  end
+end

data/lib/spontaneous/search/index.rb

@@ -0,0 +1,184 @@
+# encoding: UTF-8
+
+require 'xapian-fu'
+
+module Spontaneous::Search
+  class Index
+
+    attr_reader :name, :search_types
+
+    def initialize(name, &definition)
+      @name = name
+      @search_types = base_search_types
+      @include_pages = nil
+      @exclude_pages = nil
+      @stemmer = @stopper = nil
+      self.instance_eval(&definition) if definition
+    end
+
+    # Index DSL methods
+    def include_pages(*page_list)
+      @include_pages = page_list
+    end
+
+    def exclude_pages(*page_list)
+      @exclude_pages = page_list
+    end
+
+    def include_types(*types)
+      @search_types = resolve_type_list(types)
+    end
+
+    def exclude_types(*types)
+      @search_types -= resolve_type_list(types)
+    end
+    # end Index DSL methods
+
+    def create_db(revision)
+      Database.create(self, revision)
+    end
+
+    def search(query, options = {})
+      database.search(query, options)
+    end
+
+    def database(revision = Spontaneous::Site.published_revision)
+      Database.new(self, revision)
+    end
+
+    def language(set_language = nil)
+      @language = set_language unless set_language.nil?
+      @language ||= :english
+    end
+
+    def stemmer(set_stemmer = nil)
+      @stemmer = set_stemmer unless set_stemmer.nil?
+      @stemmer
+    end
+
+    def stopper(set_stopper = nil)
+      @stopper = set_stopper unless set_stopper.nil?
+      @stopper
+    end
+
+    # Extract all indexable content from a page. Values are grouped across fields
+    # or across field index groups
+    def indexable_content(page)
+      values = Hash.new { |h, k| h[k] = [] }
+      # not sure that I need the include? test here as page.content only returns Pieces
+      # and I'm not sure that there is a particular need to exclude Pieces from indexes
+      pieces = [page].concat(page.content).select { |content| include?(content) }
+      pieces += page.boxes.select { |box| include?(box) }
+      pieces.each do |content|
+        content.fields.each do |field|
+          prototype = field.prototype
+          values[prototype.index_id(self)] << field.indexable_value if prototype.in_index?(self)
+        end
+      end
+      result = Hash[ values.map { |id, values| [id, values.join("\n")] } ]
+      result[:id] = page.id
+      result
+    end
+
+    def fields
+      field_lists = Hash.new { |h, k| h[k] = [] }
+      # collect all the definitions for a particular field or group
+      all_types.select { |t| include_type?(t) }.each do |type|
+        type.field_prototypes.each do |prototype|
+          if prototype.in_index?(self)
+            field_lists[prototype.index_id(self)] << prototype.options_for_index(self)
+          end
+        end
+      end
+      # then aggregate/merge them
+      fields = {}
+      field_lists.each do |id, definitions|
+        if definitions.length == 1
+          fields[id] = definitions[0]
+        else
+          # if *any* of the definitions require indexing, then index
+          index = definitions.map { |d| d[:index] }.any? { |i| i != false }
+          fields[id] = {
+            :type => String,
+            :weight => definitions.map { |d| d[:weight] }.max,
+            :store => true,
+            :index => index
+          }
+          fields[id].delete(:weight) unless index
+        end
+      end
+      fields
+    end
+
+    def include?(content)
+      include_type?(content.class) and include_page?(content.page)
+    end
+
+  protected
+
+    def include_type?(type)
+      search_types.include?(type)
+    end
+
+    def include_page?(page)
+      return true unless page.page?
+      unless @exclude_pages.nil?
+        rejected = @exclude_pages.any? { |selector| match_page(selector, page) }
+        return false if rejected
+      end
+      unless @include_pages.nil?
+        included = @include_pages.any? { |selector| match_page(selector, page) }
+        return false if !included
+      end
+      true
+    end
+
+    # TODO: move this into a generic page query language class
+    def match_page(selector, page)
+      case selector
+      when Fixnum, /\A\d+\z/
+        page.id == selector.to_i
+      when /\A#(.+)\z/
+        page.uid == $1
+      when /\A\//
+        page.path == selector
+      when /\A(>=|>)\s+(.+)\z/
+        rule, root = $1, S::Site[$2]
+        root.send(rule, page)
+      else
+        false
+      end
+    end
+
+    def resolve_page_list(pages)
+      pages.map { |p| S::Site[p] }
+    end
+
+    def base_search_types
+      all_types
+    end
+
+    def resolve_type_list(types)
+      types.map { |klass| find_type(klass) }.flatten
+    end
+
+    def find_type(type_definition)
+      return type_definition if type_definition.is_a?(Class)
+      type_definition = type_definition.to_s if Symbol === type_definition
+      if type_definition.is_a?(String)
+        scope, base = type_definition.split(/\s+/).map { |s| s.strip }
+        return scope.constantize if base.nil?
+        base_type = base.constantize
+        all_types.select { |klass| base_type.send(scope, klass) }
+      end
+    end
+
+    def all_types
+      S.schema.classes
+    end
+
+    def all_page_types
+      S.schema.classes.select { |klass| klass.page? }
+    end
+  end # Index
+end # Spontaneous