ohm-contrib 0.0.42 → 0.1.0

data/lib/ohm/contrib.rb

@@ -1,6 +1,6 @@
 module Ohm
   module Contrib
-    VERSION = "0.0.42"
+    VERSION = "0.1.0"
   end
 
   autoload :ActiveModelExtension, "ohm/contrib/active_model_extension"
@@ -14,8 +14,8 @@ module Ohm
   autoload :Typecast,             "ohm/contrib/typecast"
   autoload :Locking,              "ohm/contrib/locking"
   autoload :Callbacks,            "ohm/contrib/callbacks"
-  autoload :LunarMacros,          "ohm/contrib/lunar_macros"
   autoload :Slug,                 "ohm/contrib/slug"
   autoload :Scope,                "ohm/contrib/scope"
   autoload :SoftDelete,           "ohm/contrib/soft_delete"
-end
+  autoload :FulltextSearching,    "ohm/contrib/fulltext_searching"
+end

data/lib/ohm/contrib/fulltext_searching.rb

@@ -0,0 +1,80 @@
+begin
+  require "text"
+rescue LoadError
+  raise LoadError,
+    "Type `[sudo] gem install text` to use Ohm::FulltextSearching."
+end
+
+module Ohm
+  module FulltextSearching
+    def self.included(model)
+      model.extend ClassMethods
+    end
+  
+    module ClassMethods
+      def search(hash)
+        find(hash_to_metaphones(hash))
+      end
+  
+      def hash_to_metaphones(hash)
+        ret = Hash.new { |h, k| h[k] = [] }
+
+        hash.each do |att, string|
+          metaphones(string).each { |m| ret[:"fulltext_#{att}"] << m }
+        end
+        
+        return ret
+      end
+  
+      def double_metaphone(str)
+        return [] if STOPWORDS.include?(str.to_s.downcase)
+
+        Text::Metaphone.double_metaphone(str).compact
+      end
+
+      def metaphones(str)
+        str.to_s.strip.split(/\s+/).map { |s| double_metaphone(s) }.flatten
+      end
+
+      def fulltext(att)
+        field = :"fulltext_#{att}"
+        
+        define_method(field) { self.class.metaphones(send(att)) }
+        index(field)
+      end
+    end
+
+    STOPWORDS = %w{a about above according across actually adj after 
+      afterwards again against all almost alone along already also although 
+      always among amongst an and another any anyhow anyone anything anywhere 
+      are aren't around as at b be became because become becomes becoming 
+      been before beforehand begin behind being below beside besides between
+      beyond both but by c can can't cannot caption co co. could couldn't d 
+      did didn't do does doesn't don't down during e each eg eight eighty 
+      either else elsewhere end ending enough etc even ever every everyone 
+      everything everywhere except f few first for found from further g h 
+      had has hasn't have haven't he he'd he'll he's hence her here here's 
+      hereafter hereby herein hereupon hers herself him himself his how 
+      however hundred i i'd i'll i'm i've ie if in inc. indeed instead into is
+      isn't it it's its itself j k l last later latter latterly least less let
+      let's like likely ltd m made make makes many maybe me meantime meanwhile 
+      might miss more moreover most mostly mr mrs much must my myself n namely 
+      neither never nevertheless next nine ninety no nobody none nonetheless 
+      noone nor not nothing now nowhere o of off often on once one one's only 
+      onto or other others otherwise our ours ourselves out over overall own 
+      p per perhaps q r rather recent recently s same seem seemed seeming 
+      seems seven several she she'd she'll she's should shouldn't since so 
+      some somehow someone something sometime sometimes somewhere still such 
+      t taking than that that'll that's that've the their them themselves then
+      thence there there'd there'll there're there's there've thereafter thereby 
+      therefore therein thereupon these they they'd they'll they're they've 
+      thirty this those though three through throughout thru thus to together 
+      too toward towards u under unless unlike unlikely until up upon us used 
+      using v very via w was wasn't we we'd we'll we're we've well were weren't 
+      what what'll what's what've whatever when whence whenever where where's 
+      whereafter whereas whereby wherein whereupon wherever whether which while 
+      whither who who'd who'll who's whoever whole whom whomever whose why will 
+      with within without won't would wouldn't x y yes yet you you'd you'll 
+      you're you've your yours yourself yourselves z}
+  end
+end

data/lib/ohm/contrib/typecast.rb

@@ -1,9 +1,20 @@
 require 'bigdecimal'
 require 'time'
 require 'date'
-require 'json'
 require 'forwardable'
 
+begin
+  require "yajl/json_gem"
+rescue LoadError
+  $stderr.puts(
+    "WARNING: Currently using the `json` gem. You might encounter encoding " +
+    "problems with `Ohm::Types::Hash` and `Ohm::Types::Array`. Best to use " +
+    "the `yajl-ruby` gem to avoid problems and also for performance reasons."
+  )
+
+  require "json"
+end
+
 module Ohm
   # Provides all the primitive types. The following are included:
   #
@@ -342,5 +353,4 @@ module Ohm
       end
     end
   end
-end
-
+end

data/test/autoload_test.rb

@@ -4,6 +4,7 @@ require File.expand_path("./helper", File.dirname(__FILE__))
 
 test "autoloading of all libraries" do
   assert_nothing_raised NameError, LoadError do
+    Ohm::ActiveModelExtension
     Ohm::Boundaries
     Ohm::Timestamping
     Ohm::WebValidations
@@ -12,8 +13,7 @@ test "autoloading of all libraries" do
     Ohm::Locking
     Ohm::ExtraValidations
     Ohm::DateValidations
-    Ohm::LunarMacros
+    Ohm::FulltextSearching
     Ohm::Slug
   end
-end
-
+end

data/test/fixtures/ascii8bit.txt

@@ -0,0 +1 @@
+café

data/test/fulltext_searching_test.rb

@@ -0,0 +1,63 @@
+# encoding: UTF-8
+
+require File.expand_path("./helper", File.dirname(__FILE__))
+
+class Video < Ohm::Model
+  include Ohm::FulltextSearching
+
+  attribute :title
+  attribute :description
+   
+  fulltext :title
+  fulltext :description
+end
+
+test "finding basic words using metaphones" do
+  v = Video.create(:title => "The quick brown fox jumps over the lazy dog")
+
+  assert Video.find(:fulltext_title => "KK").include?(v)
+  assert Video.find(:fulltext_title => "PRN").include?(v)
+  assert Video.find(:fulltext_title => "FKS").include?(v)
+  assert Video.find(:fulltext_title => "JMPS").include?(v)
+  assert Video.find(:fulltext_title => "AMPS").include?(v)
+  assert Video.find(:fulltext_title => ["JMPS", "AMPS"]).include?(v)
+  assert Video.find(:fulltext_title => "LS").include?(v)
+  assert Video.find(:fulltext_title => "TK").include?(v)
+end
+
+test "finding using the actual words" do
+  v = Video.create(:title => "The quick brown fox jumps over the lazy dog")
+  
+  assert Video.search(:title => "quick").include?(v)
+  assert Video.search(:title => "brown").include?(v)
+  assert Video.search(:title => "fox").include?(v)
+  assert Video.search(:title => "jumps").include?(v)
+  assert Video.search(:title => "lazy").include?(v)
+  assert Video.search(:title => "dog").include?(v)
+end
+
+test "finding using slightly mispelled words" do
+  v = Video.create(:title => "The quick brown fox jumps over the lazy dog")
+
+  assert Video.search(:title => "quik").include?(v)
+  assert Video.search(:title => "brwn").include?(v)
+  assert Video.search(:title => "fx").include?(v)
+  assert Video.search(:title => "fax").include?(v)
+  assert Video.search(:title => "jumps").include?(v)
+  assert Video.search(:title => "lazi").include?(v)
+  assert Video.search(:title => "dag").include?(v)
+end
+
+test "stopword stripping" do
+  # This is the actual code that strips out stopwords.
+
+  assert_equal [], Video.double_metaphone("about")
+  
+  # Here we just verify that actually on a long string level,
+  # stop words are in fact stripped.
+  assert Video.metaphones(Video::STOPWORDS.join(" ")).empty?
+
+  # Finally we need to make sure that finding stop words
+  # should not even proceed.
+  assert Video.hash_to_metaphones(:title => "about").empty?
+end

data/test/typecast_array_test.rb

@@ -139,22 +139,8 @@ test "raises when trying to assign a non-array" do
   end
 end
 
-test "inspecting" do
-  post = Post.create(:addresses => [{ "address1" => "#456",
-                                      "city" => "Singapore",
-                                      "country" => "SG" }])
-
-  expected = %q{[{"address1":"#456","city":"Singapore","country":"SG"}]}
-
-  assert expected == post.addresses.inspect
-
-  post.addresses = 'FooBar'
-  assert %{"\\\"FooBar\\\""} == post.addresses.inspect
-end
-
 test "type is array" do
   post = Post.create(:addresses => ["address1"])
 
   assert post.addresses.type == Array
-end
-
+end

data/test/typecast_hash_test.rb

@@ -103,20 +103,18 @@ test "raises when trying to assign a non-hash" do
   end
 end
 
-test "inspecting" do
-  post = Post.create(:address => { "address1" => "#456",
-                                   "city" => "Singapore",
-                                   "country" => "SG" })
-
-  expected = %q{{"address1":"#456","city":"Singapore","country":"SG"}}
-  assert expected == post.address.inspect
-
-  post.address = 'FooBar'
-  assert %{"\\\"FooBar\\\""} == post.address.inspect
-end
-
 test "type is Hash" do
   post = Post.new(:address => { "address1" => "#456" })
   assert Hash == post.address.type
 end
 
+test "ascii 8bit encoding" do
+  txt = File.expand_path("fixtures/ascii8bit.txt", File.dirname(__FILE__))
+
+  data = File.read(txt, :encoding => "ascii-8bit")
+
+  post = Post.create(:address => { "address1" => data })
+  post = Post[post.id]
+
+  assert_equal post.address["address1"], data.force_encoding("UTF-8")
+end if defined?(Encoding)

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
+  - 1
   - 0
-  - 42
-  version: 0.0.42
+  version: 0.1.0
 platform: ruby
 authors: 
 - Cyril David
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-23 00:00:00 +08:00
+date: 2011-01-08 00:00:00 +08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -96,9 +96,9 @@ files:
 - lib/ohm/contrib/callbacks.rb
 - lib/ohm/contrib/date_validations.rb
 - lib/ohm/contrib/extra_validations.rb
+- lib/ohm/contrib/fulltext_searching.rb
 - lib/ohm/contrib/length_validations.rb
 - lib/ohm/contrib/locking.rb
-- lib/ohm/contrib/lunar_macros.rb
 - lib/ohm/contrib/number_validations.rb
 - lib/ohm/contrib/scope.rb
 - lib/ohm/contrib/slug.rb
@@ -115,10 +115,11 @@ files:
 - test/boundaries_test.rb
 - test/callbacks_lint.rb
 - test/date_validations_test.rb
+- test/fixtures/ascii8bit.txt
+- test/fulltext_searching_test.rb
 - test/helper.rb
 - test/instance_callbacks_test.rb
 - test/length_validations_test.rb
-- test/lunar_macros_test.rb
 - test/macro_callbacks_test.rb
 - test/membership_validation_test.rb
 - test/number_validations_test.rb

data/lib/ohm/contrib/lunar_macros.rb

@@ -1,58 +0,0 @@
-begin
-  require 'lunar'
-rescue LoadError
-  raise "You have to install Lunar in order to use Ohm::LunarMacros."
-end
-
-module Ohm
-  module LunarMacros
-    def self.included(base)
-      base.send  :include, Ohm::Callbacks
-      base.after :save,   :update_lunar_index
-      base.after :delete, :delete_lunar_index
-
-      base.extend ClassMethods
-    end
-
-    module ClassMethods
-      def fuzzy(*atts)    lunar_fields(:fuzzy,    *atts) end
-      def text(*atts)     lunar_fields(:text,     *atts) end
-      def number(*atts)   lunar_fields(:number,   *atts) end
-      def sortable(*atts) lunar_fields(:sortable, *atts) end
-
-      def lunar_fields(type, *atts)
-        @lunar_fields ||= Hash.new { |h, k| h[k] = [] }
-
-        atts.each { |att|
-          @lunar_fields[type] << att  unless @lunar_fields[type].include?(att)
-        }
-
-        @lunar_fields[type]
-      end
-    end
-
-    def update_lunar_index
-      Lunar.index self.class do |i|
-        i.id id
-
-        [:fuzzy, :text, :number, :sortable].each do |type|
-          self.class.lunar_fields(type).each do |field|
-            value = send(field)
-
-            if type == :text and value.kind_of?(Enumerable)
-              i.text field, value.join(' ')  unless value.empty?
-            else
-              i.send type, field, value  unless value.to_s.empty?
-            end
-          end
-        end
-      end
-    end
-
-  protected
-    def delete_lunar_index
-      Lunar.delete self.class, id
-    end
-  end
-end
-

data/test/lunar_macros_test.rb

@@ -1,146 +0,0 @@
-# encoding: UTF-8
-
-require File.expand_path("./helper", File.dirname(__FILE__))
-
-class Post < Ohm::Model
-  include Ohm::LunarMacros
-end
-
-test "fuzzy / text / number / sortable" do
-  Post.fuzzy :name
-  assert [:name] == Post.fuzzy
-
-  Post.text :name
-  assert [:name] == Post.text
-
-  Post.number :name
-  assert [:name] == Post.number
-
-  Post.sortable :name
-  assert [:name] == Post.sortable
-end
-
-test "fuzzy / text / number / sortable done twice" do
-  Post.fuzzy :name
-  Post.fuzzy :name
-  assert [:name] == Post.fuzzy
-
-  Post.text :name
-  Post.text :name
-  assert [:name] == Post.text
-
-  Post.number :name
-  Post.number :name
-  assert [:name] == Post.number
-
-  Post.sortable :name
-  Post.sortable :name
-  assert [:name] == Post.sortable
-end
-
-class Document < Ohm::Model
-  include Ohm::LunarMacros
-
-  fuzzy    :filename
-  text     :content
-  number   :author_id
-  sortable :views
-
-  attribute :filename
-  attribute :content
-  attribute :author_id
-  attribute :views
-end
-
-test "fuzzy indexing" do
-  doc = Document.create(:filename => "amazing.mp3")
-
-  strs = %w{a am ama amaz amazi amazin amazing amazing. amazing.m
-            amazing.mp amazing.mp3}
-
-  strs.each do |str|
-    r = Lunar.search(Document, :fuzzy => { :filename => str })
-    assert r.include?(doc)
-  end
-
-  doc.update(:filename => "crazy.mp3")
-
-  strs = %w{c cr cra craz crazy crazy. crazy.m crazy.mp crazy.mp3}
-
-  strs.each do |str|
-    r = Lunar.search(Document, :fuzzy => { :filename => str })
-    assert r.include?(doc)
-  end
-end
-
-test "text indexing" do
-  doc = Document.create(:content => "The quick brown fox")
-
-  queries = ['quick', 'brown', 'fox', 'quick brown' 'quick fox', 'fox brown',
-             'the quick brown fox']
-
-  queries.each do |q|
-    r = Lunar.search(Document, :q => q)
-    assert r.include?(doc)
-  end
-
-  doc.update(:content => "lazy dog jumps over")
-
-  queries = ['lazy', 'dog', 'jumps', 'over', 'lazy dog', 'jumps over']
-
-  queries.each do |q|
-    r = Lunar.search(Document, :q => q)
-    assert r.include?(doc)
-  end
-end
-
-test "number indexing" do
-  doc = Document.create(:author_id => 99)
-
-  [99..100, 98..99, 98..100].each do |range|
-    r = Lunar.search(Document, :author_id => range)
-    assert r.include?(doc)
-  end
-
-  [97..98, 100..101].each do |range|
-    r = Lunar.search(Document, :author_id => range)
-    assert ! r.include?(doc)
-  end
-
-  doc.update(:author_id => 49)
-
-  [49..50, 48..49, 48..50].each do |range|
-    r = Lunar.search(Document, :author_id => range)
-    assert r.include?(doc)
-  end
-
-  [47..48, 50..51].each do |range|
-    r = Lunar.search(Document, :author_id => range)
-    assert ! r.include?(doc)
-  end
-end
-
-test "sortable indexing" do
-  osx    = Document.create(:content => "apple mac osx", :views => 500)
-  iphone = Document.create(:content => "apple iphone",  :views => 10_000)
-
-  assert [iphone, osx] == Lunar.search(Document, :q => "apple").sort_by(:views, :order => "DESC")
-  assert [osx, iphone] == Lunar.search(Document, :q => "apple").sort_by(:views, :order => "ASC")
-
-  osx.update(:content => "ios mac osx", :views => 1000)
-  iphone.update(:content => "ios iphone", :views => 500)
-
-  assert [osx, iphone] == Lunar.search(Document, :q => "ios").sort_by(:views, :order => "DESC")
-  assert [iphone, osx] == Lunar.search(Document, :q => "ios").sort_by(:views, :order => "ASC")
-end
-
-test "on delete" do
-  doc = Document.create(:filename => "amazing.mp3", :content => "apple mac osx",
-                        :author_id => 99, :views => 500)
-
-  doc.delete
-
-  assert Lunar.search(Document, :q => "apple").size.zero?
-  assert Lunar.search(Document, :fuzzy => { :filename => "amazing" }).size.zero?
-end
-