mongoid_ext 0.6.1

data/lib/mongoid_ext/file_list.rb

@@ -0,0 +1,74 @@
+module MongoidExt
+  class FileList < EmbeddedHash
+    attr_accessor :parent_document
+
+    def put(id, io, metadata = {})
+      if !parent_document.new_record?
+        filename = id
+        if io.respond_to?(:original_filename)
+          filename = io.original_filename
+        elsif io.respond_to?(:path) && io.path
+          filename = ::File.basename(io.path)
+        elsif io.kind_of?(String)
+          io = StringIO.new(io)
+        end
+
+        get(id).put(filename, io, metadata)
+      else
+        (@_pending_files ||= {})[id] = [io, metadata]
+      end
+    end
+
+    def files
+      ids = self.keys
+      ids.delete("_id")
+      ids.map {|v| get(v) }
+    end
+
+    def each_file(&block)
+      (self.keys-["_id"]).each do |key|
+        file = self.get(key)
+        yield key, file
+      end
+    end
+
+    def get(id)
+      if id.kind_of?(MongoidExt::File)
+        self[id.id] = id
+        return id
+      end
+
+      id = id.to_s.gsub(".", "_")
+      file = self[id]
+      if file.nil?
+        file = self[id] = MongoidExt::File.new
+      elsif file.class == BSON::OrderedHash
+        file = self[id] = MongoidExt::File.new(file)
+      end
+
+      file._root_document = parent_document
+      file
+    end
+
+    def sync_files
+      if @_pending_files
+        @_pending_files.each do |filename, data|
+          put(filename, data[0], data[1])
+        end
+        @_pending_files = nil
+      end
+    end
+
+    def delete(id)
+      file = self.get(id)
+      super(id)
+      file.delete
+    end
+
+    def destroy_files
+      each_file do |id, file|
+        get(id).delete
+      end
+    end
+  end
+end

data/lib/mongoid_ext/file_server.rb

@@ -0,0 +1,69 @@
+require 'time'
+require 'rack/utils'
+
+module MongoidExt
+class FileServer
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    if env["PATH_INFO"] =~ /^\/_files\/([^\/?]+)/
+      @model = $1.classify.constantize rescue nil
+      return forbidden if @model.nil?
+
+      dup._call(env)
+    else
+      @app.call(env)
+    end
+  end
+
+  def _call(env)
+    request = Rack::Request.new(env)
+    params = request.GET
+
+    @file = @model.find_file_from_params(params, request)
+    return not_found if @file.nil?
+
+    if @file.present?
+      serving
+    else
+      not_found
+    end
+  end
+
+  def forbidden
+    body = "Forbidden\n"
+    [403, {"Content-Type" => "text/plain",
+           "Content-Length" => body.size.to_s,
+           "X-Cascade" => "pass"},
+     [body]]
+  end
+
+  def serving
+    body = self
+    [200, {
+      "Last-Modified"  => Time.now.httpdate,
+      "Content-Type"   => @file.content_type,
+      "Content-Length" => @file.size.to_s
+    }, body]
+  end
+
+  def not_found
+    body = "File not found: #{@path_info}\n"
+    [404, {"Content-Type" => "text/plain",
+       "Content-Length" => body.size.to_s,
+       "X-Cascade" => "pass"},
+     [body]]
+  end
+
+  def each
+    f = @file.get
+    while part = f.read(8192)
+      yield part
+      break if part.empty?
+    end
+  end
+end
+end
+

data/lib/mongoid_ext/filter.rb

@@ -0,0 +1,266 @@
+module MongoidExt
+  module Filter
+    def self.included(klass)
+      begin
+        require 'lingua/stemmer'
+      rescue LoadError
+        $stderr.puts "install ruby-stemmer `gem install ruby-stemmer` to activate the full text search support"
+      end
+
+      klass.class_eval do
+        extend ClassMethods
+
+        field :_keywords, :type => Set, :default => Set.new
+        index :_keywords
+
+        before_save :_update_keywords
+
+        attr_accessor :search_score
+      end
+    end
+
+    module ClassMethods
+      def filterable_keys(*keys)
+        @filterable_keys ||= Set.new
+        @filterable_keys += keys
+
+        @filterable_keys
+      end
+
+      def language(lang = 'en')
+        @language ||= lang
+      end
+
+      def filter_conditions(query, opts = {})
+        stemmer = nil
+        language = opts.delete(:language) || 'en'
+
+        if defined?(Lingua::Stemmer)
+          stemmer = MongoidExt::Filter.build_stemmer(language)
+        end
+
+        @parser = MongoidExt::Filter::Parser.new(stemmer)
+        parsed_query = @parser.parse(query)
+
+        [parsed_query, _build_filter_query(parsed_query)]
+      end
+
+      def filter(query, opts = {})
+        min_score = opts.delete(:min_score) || 0.0
+        limit = opts.delete(:per_page) || 25
+        page = opts.delete(:page) || 1
+        select = opts.delete(:select) || self.fields.keys
+
+        parsed_query, conds = self.filter_conditions(query, opts)
+        query = Mongoid::Criteria.new(self)
+        conds = query.where(opts).where(conds).selector
+
+        results = self.db.nolock_eval("function(collection, q, config) { return filter(collection, q, config); }", self.collection_name, conds, {:words => parsed_query[:words].to_a, :stemmed => parsed_query[:stemmed].to_a, :limit => limit, :min_score => min_score, :select => select })
+
+        pagination = MongoidExt::Filter::ResultSet.new(results["total_entries"], parsed_query, conds, page, limit)
+
+        pagination.subject = results['results'].map do |result|
+          item = self.new(result['doc'])
+          item.search_score = result['score']
+
+          item
+        end
+
+        pagination
+      end
+
+      def _build_filter_query(parsed_query)
+        mongoquery = {}
+
+        if !parsed_query[:tokens].empty?
+          keywords = []
+
+          parsed_query[:tokens].each do |t|
+            keywords << /^#{Regexp.escape(t)}/i if t.size > 2
+          end
+
+          mongoquery[:_keywords] = { :$in => keywords }
+        end
+
+        mongoquery.merge!(map_filter_operators(parsed_query[:quotes], parsed_query[:operators]))
+
+        mongoquery
+      end
+
+      def map_filter_operators(quotes, ops)
+        mongoquery = {}
+
+        if ops["is"]
+          ops["is"].each do |d|
+            mongoquery[d] = true
+          end
+        end
+
+        if ops["not"]
+          ops["not"].each do |d|
+            mongoquery[d] = false
+          end
+        end
+
+        mongoquery
+      end
+    end
+
+    protected
+    def _update_keywords
+      lang = self.class.language
+      if lang.kind_of?(Symbol)
+        lang = send(lang)
+      elsif lang.kind_of?(Proc)
+        lang = lang.call(self)
+      end
+
+      stemmer = nil
+      if defined?(Lingua)
+        stemmer = MongoidExt::Filter.build_stemmer(lang)
+      end
+
+      stop_words = []
+      if self.respond_to?("#{lang}_stop_words")
+        stop_words = Set.new(self.send("#{lang}_stop_words"))
+      end
+
+      self._keywords = Set.new
+      self.class.filterable_keys.each do |key|
+        self._keywords += keywords_for_value(read_attribute(key), stemmer, stop_words)
+      end
+    end
+
+    private
+    def keywords_for_value(val, stemmer=nil, stop_words = [])
+      if val.kind_of?(String)
+        words = []
+        val.downcase.split.each do |word|
+          next if word.length < 3
+          next if stop_words.include?(word)
+
+          deword = word.split(%r{'|"|\/|\\|\?|\+|_|\-|\>|\>})
+
+          deword.each do |word|
+            next if word.empty?
+
+            stem = word
+            if stemmer
+              stem = stemmer.stem(word)
+            end
+            normalize_string = lambda {|str| ActiveSupport::Multibyte::Chars.new(str).normalize(:kd).gsub(/[^\x00-\x7F]/n, '').to_s }
+
+            normalized = normalize_string.call(word)
+
+            if word != normalized
+              words << normalized
+            end
+
+            words << word
+            if stem
+              normalized=normalize_string.call(stem)
+
+              words << normalized if stem != normalized
+              words << stem if stem != word
+            end
+          end
+        end
+
+        words
+      elsif val.kind_of?(Array)
+        val.map { |e| keywords_for_value(e, stemmer, stop_words) }.flatten
+      elsif val.kind_of?(Hash)
+        val.map { |k, v| keywords_for_value(v, stemmer, stop_words) }.flatten
+      elsif val
+        [val]
+      else
+        []
+      end
+    end
+
+    def self.build_stemmer(language)
+      begin
+        return Lingua::Stemmer.new(:language => language)
+      rescue Lingua::StemmerError
+        return nil
+      end
+    end
+
+    def en_stop_words
+      ["a", "about", "above", "after", "again", "against", "all", "am", "an",
+       "and", "any", "are", "aren't", "as", "at", "be", "because", "been",
+       "before", "being", "below", "between", "both", "but", "by", "cannot",
+       "can't", "could", "couldn't", "did", "didn't", "do", "does", "doesn't",
+       "doing", "don't", "down", "during", "each", "few", "for", "from",
+       "further", "had", "hadn't", "has", "hasn't", "have", "haven't", "having",
+       "he", "he'd", "he'll", "her", "here", "here's", "hers", "herself", "he's",
+       "him", "himself", "his", "how", "how's", "i", "i'd", "if", "i'll", "i'm",
+       "in", "into", "is", "isn't", "it", "its", "it's", "itself", "i've", "let's",
+       "me", "more", "most", "mustn't", "my", "myself", "no", "nor", "not", "of",
+       "off", "on", "once", "only", "or", "other", "ought", "our", "ours", "ourselves",
+       "out", "over", "own", "same", "shan't", "she", "she'd", "she'll", "she's",
+       "should", "shouldn't", "so", "some", "such", "than", "that", "that's", "the",
+       "their", "theirs", "them", "themselves", "then", "there", "there's", "these",
+       "they", "they'd", "they'll", "they're", "they've", "this", "those", "through",
+       "to", "too", "under", "until", "up", "very", "was", "wasn't", "we", "we'd",
+       "we'll", "were", "we're", "weren't", "we've", "what", "what's", "when",
+       "when's", "where", "where's",  "which", "while", "who", "whom", "who's",
+       "why", "why's", "with", "won't", "would", "wouldn't", "you", "you'd",
+       "you'll", "your", "you're", "yours", "yourself", "yourselves", "you've",
+       "the", "how"]
+    end
+
+    def es_stop_words
+      ["de", "la", "que", "el", "en", "y", "a",
+       "los", "del", "se", "las", "por", "un", "para", "con", "no", "una", "su",
+       "al", "lo", "como", "mas", "pero", "sus", "le", "ya", "o", "este", "sus",
+       "si", "porque", "esta", "entre",
+       "cuando", "muy", "sin", "sobre", "tambien", "me", "hasta", "hay",
+       "donde", "quien", "desde", "todo", "nos", "durante", "todos", "uno", "les",
+       "ni", "contra", "otros", "ese", "eso", "ante", "ellos", "e", "esto",
+       "mi", "antes", "algunos", "que", "unos", "yo", "otro",
+       "otras", "otra", "al", "tanto", "esa", "estos", "mucho", "quienes",
+       "nada", "muchos", "cual", "poco", "ella", "estar", "estas", "algunas",
+       "algo", "nosotros", "mi", "mis", "tus", "ellas", "sus", "una", "uno",
+       "nosotras", "vosostros", "vosostras", "os", "mio", "mia",
+       "mios", "mias", "tuyo", "tuya", "tuyos", "tuyas", "suyo",
+       "suya", "suyos", "suyas", "nuestro", "nuestra", "nuestros", "nuestras",
+       "vuestro", "vuestra", "vuestros", "vuestras", "esos", "esas", "estoy",
+       "estas", "esta", "estamos", "estais", "estan",
+       "este", "estes", "estemos", "esteis", "esten",
+       "estare", "estareis", "estara", "estaremos", "como",
+       "estareis", "estarin", "estaria", "estarias",
+       "estariamos", "estariais", "estarian", "estaba",
+       "estabas", "estabamos", "estabais", "estaban", "estuve",
+       "estuviste", "estuvo", "estuvimos", "estuvisteis", "estuvieron", "estuviera",
+       "estuvieras", "estuvieramos", "estuvierais", "estuvieran", "estuviese",
+       "estuvieses", "estuviesemos", "estuvieseis", "estuviesen", "estando",
+       "estado", "estada", "estados", "estadas", "estad", "he", "has", "ha", "hemos",
+       "habeis", "han", "haya", "hayas", "hayamos", "hayais", "hayan",
+       "habre", "habras", "habra", "habremos", "habreis",
+       "habran", "habria", "habrias", "habriamos",
+       "habriais", "habrian", "habia", "habias",
+       "habiamos", "habiais", "habian", "hube", "hubiste",
+       "hubo", "hubimos", "hubisteis", "hubieron", "hubiera", "hubieras", "hubieramos",
+       "hubierais", "hubieran", "hubiese", "hubieses", "hubiesemos", "hubieseis",
+       "hubiesen", "habiendo", "habido", "habida", "habidos", "habidas", "soy", "eres",
+       "es", "somos", "sois", "son", "sea", "seas", "seamos", "seais",
+       "sean", "sere", "seras", "sera", "seremos", "sereis",
+       "seran", "seria", "serias", "seriamos", "seriais",
+       "serian", "era", "eras", "eramos", "erais", "eran", "fui",
+       "fuiste", "fue", "fuimos", "fuisteis", "fueron", "fuera", "fueras",
+       "fueramos", "fuerais", "fueran", "fuese", "fueses", "fuesemos",
+       "fueseis", "fuesen", "sintiendo", "sentido", "sentida", "sentidos", "sentidas",
+       "siente", "sentid", "tengo", "tienes", "tiene", "tenemos", "teneis",
+       "tienen", "tenga", "tengas", "tengamos", "tengais", "tengan", "tendre",
+       "tendras", "tendra", "tendremos", "tendreis", "tendran",
+       "tendria", "tendrias", "tendriamos", "tendriais",
+       "tendrian", "tenia", "tenias", "teniamos",
+       "teniais", "tenian", "tuve", "tuviste", "tuvo", "tuvimos",
+       "tuvisteis", "tuvieron", "tuviera", "tuvieras", "tuvieramos",
+       "tuvierais", "tuvieran", "tuviese", "tuvieses", "tuviesemos",
+       "tuvieseis", "tuviesen", "teniendo", "tenido", "tenida", "tenidos", "tenidas", "tened"]
+    end
+  end
+end
+

data/lib/mongoid_ext/filter/parser.rb

@@ -0,0 +1,71 @@
+module MongoidExt
+  module Filter
+    class Parser
+      def initialize(stemmer)
+        @stemmer = stemmer
+      end
+
+      def parse(query)
+        query, quotes = parse_quoted_text(query)
+        query, ops = parse_operators(query)
+        query.gsub!(/\s+/, ' ')
+        query.gsub!(/\?|\!|\:|\)|\(/, "")
+
+        words = Set.new(query.downcase.split(/\s+/))
+
+        stemmed = stem(words)
+        tokens = words + stemmed
+
+        {:query => query,
+         :words => words,
+         :stemmed => stemmed,
+         :operators => ops,
+         :tokens => tokens,
+         :quotes => quotes}
+      end
+
+      private
+      def parse_quoted_text(query)
+        quotes = []
+        loop do
+          m = query.match(/"([^"]+"|-)/)
+          if m
+            query = $` + $'
+            quotes << m[1].gsub('"', "")
+          else
+            break
+          end
+        end
+
+        [query, quotes]
+      end
+
+      def parse_operators(query)
+        ops = {}
+
+        loop do
+          m = query.match(/(is|lang|not|by|score):(\S+)/)
+          if m
+            query = $` + $'
+            (ops[m[1]] ||= []) << m[2]
+          else
+            break
+          end
+        end
+        [query, ops]
+      end
+
+      def stem(words)
+        tokens = []
+        if @stemmer
+          words.each do |word|
+            stem = @stemmer.stem(word)
+            tokens << stem if word.size > 2 && !words.include?(stem)
+          end
+        end
+
+        tokens
+      end
+    end
+  end
+end