silver 0.2.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,18 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.3.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.2"
+  gem "rcov", ">= 0"
+  gem "redis", "~>  2.1.1"
+  gem "yajl-ruby", ">=  0.7.7"
+  gem "text", "~>  0.2.0"
+  gem "dm-core", "~> 1.0.0"
+  gem "dm-sqlite-adapter", "~> 1.0.0"
+end

data/Gemfile.lock

@@ -0,0 +1,51 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.2.3)
+    data_objects (0.10.2)
+      addressable (~> 2.1)
+    diff-lcs (1.1.2)
+    dm-core (1.0.2)
+      addressable (~> 2.2)
+      extlib (~> 0.9.15)
+    dm-do-adapter (1.0.2)
+      data_objects (~> 0.10.2)
+      dm-core (~> 1.0.2)
+    dm-sqlite-adapter (1.0.2)
+      dm-do-adapter (~> 1.0.2)
+      do_sqlite3 (~> 0.10.2)
+    do_sqlite3 (0.10.2)
+      data_objects (= 0.10.2)
+    extlib (0.9.15)
+    git (1.2.5)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.8.7)
+    rcov (0.9.9)
+    redis (2.1.1)
+    rspec (2.3.0)
+      rspec-core (~> 2.3.0)
+      rspec-expectations (~> 2.3.0)
+      rspec-mocks (~> 2.3.0)
+    rspec-core (2.3.1)
+    rspec-expectations (2.3.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.3.0)
+    text (0.2.0)
+    yajl-ruby (0.7.9)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  dm-core (~> 1.0.0)
+  dm-sqlite-adapter (~> 1.0.0)
+  jeweler (~> 1.5.2)
+  rcov
+  redis (~> 2.1.1)
+  rspec (~> 2.3.0)
+  text (~> 0.2.0)
+  yajl-ruby (>= 0.7.7)

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Erik Hin-tone
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,92 @@
+# Silver
+
+Silver is a lightweight, redis-backed database cacher and indexer.
+
+## Getting Started
+
+As it says on the tin, Silver is going to make your database queries much faster. Now it is no secret that Redis is fantastic to use as a cache/index. However, you have to write the same boilerplate to use it as cache over and over: find most recent cached entry, look for newer entries in the database, cache it to redis, combine with old results. Rinse, repeat.
+The goal of Silver is so that you never have to do that again. Rather than connecting to the database/service for you, you simply wrap your calls in Silver and it does the rest. This means you can use silver to speed up calls to databases, calls APIs, CURLS, really whatever you want.
+
+### A Simple Caching Example
+
+First make sure you have Silver installed.
+    gem install silver
+Now, let's pretend you have an app that queries your database for entries frequently. Entries are added frequently. Furthermore, you only want Entries that come from a specific blog, blog #12. Also you want to grab something from an association of the Entry row in the database. Let's say the author's name.
+First, instantiate a new cache object.
+    cache = Silver::Cache.new("12_entries","created_time") do |date|
+      Entry.all(:order => :created_time.desc, :created_time.gt => date, :blog_id => 12)
+    end
+The first paramater passed to the constructor is the name you want to give to this cache in Redis as Silver allows to creates as many caches for  different queries as you would like. The second paramater is the name of the field that you will be using to determine if there are new entries. Finally, you pass the constructor a block that will receive the date of the newest cached entry from Redis. You must return the entries in reverse chronological order for Silver to be able to keep them in order. Silver will then query the database/service for newer entries when the instance's find method is called.
+    results = cache.find do |entry|
+      attrs = entry.attributes
+      author = {:author_name => entry.author[:name]}
+      attrs.merge author
+    end
+The find method of a cache instance takes a block that will be called for every new entry. The results of the block call should be a hash that will be stored in the cache. The whole thing will be converted into JSON and stashed in the Redis cache. From now on the database will never have to be hit again to return this value. The find method returns an array of all the results old and new from the Redis cache. 
+If you just want to read from the cache without hitting the database, simply call find without a block and with a single param:    false
+    results = cache.find(false)
+Currently, the cache does not support the changing of cached entries and is, thus, intended for data that is unlikely to change once it has been written to the database. This feature will be included in future releases of Silver.
+Finally, Silver provides a cull method.
+    cache.cull(30)
+This will cut the Redis cache down to the 30 most recent items.
+
+### A Simple Indexing Example
+
+However, Silver is not just a simple cache. It can also be used to index a database. It is optimized to index based on short text, such as names, captions, tag lists, excerpts, tweets etc. There is nothing stopping you from using on longer fields such as body text except the size of your memory alloted to Redis. Silver uses a stupidly simple fuzzy text search. The search will likely be augmented in the future.
+Here's how you would index a mess of photos by their captions, falling back on their filename if no caption is given.
+First, instantiate a new index object.
+    index = Silver::Index.new("blog_pictures","created_time") do |date|
+      Picture.all(:order => :created_time.desc, :created_time.gt => date)
+    end
+This is the same deal as before with Silver::Cache: redis key name, time field, ordering block.
+Next, call the find_and_update method of the instance.
+    index.find_and_update do |result|
+      output = result.label || result.filename || ""
+      id = result.id
+      [id,output]
+    end
+Find_and_update takes a block that will be called for each db-fetched result. This block should return a two item array of the row's id, first, and the value we are using for indexing second. As you can see in the example, Silver allows you to mix fields to use to index. It let's you do anything you want actually as long as an id and a corresponding value are returned. After calling find_and_update, your database is indexed and ready to be searched. Say, we wanted to search for photos of "Barack Obama":
+    search = Silver::Search.new("Barack Obama","blog_pictures")
+The constructor takes a string to search for and the name of Redis key storing the index. To actually perform the search:
+    search.perform{|id| Picture.get(id)}
+The perform method takes a block that will be passed the ids of all the id's whose indexes match the query. Perform will return an array of database/service objects for you to then interact with as you please.
+
+### A note about the shortcoming of the search.
+
+As Silver is currently in beta, it's search could use some work (feel free to contribute code). Currently, it does a fuzzy search based on the double-metaphone of each word in the search and then interects the results. This means several things. First, a search for "barack Obama" will only return those entries indexed with "Barack Obama"(case-insensitive) in their indexed field. It will excludes those with just, say, "obama". Second, no levensteining is done yet. It should be.
+
+### Non-standard configurations
+
+Every initializer in Silver takes, in addition to the parameters shown above, an optional options hash for Redis as the third parameter.
+    cache = Silver::Cache.new("12_entries","created_time",{:host => "127.0.0.1",:port => "6969"})
+Also, the search initializer for Silver's indexing takes optional number and offset paramater for pagination. Default is no offest and 30 results returned.
+    search = Silver::Search.new("Barack Obama","blog_pictures",{:host => "127.0.0.1",:port => "6969"},50,10)
+This will return results 10-60.
+
+### Rocco Annotated Source
+
+* [Cacher](http://tpm.github.com/Silver/cache.html)
+* [Indexer](http://tpm.github.com/Silver/indexer.html)
+* [Searcher](http://tpm.github.com/Silver/search.html)
+
+## Improvements to be made
+
+* Better Search
+* Allow for changes to be made in the database for already cached values. (Make an uncache function that fetches that ID again and replaces it in the cache)
+* Etc.
+
+## Contributing to silver
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+Copyright (c) 2011 Erik Hin-tone. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,50 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  gem.name = "silver"
+  gem.homepage = "http://github.com/tpm/silver"
+  gem.license = "MIT"
+  gem.summary = %Q{Makes your queries faster with the power of Redis.}
+  gem.description = %Q{A lightweight, Redis-backed cacher and indexer for databases, REST API's, really anything you can query.}
+  gem.email = "hinton.erik@gmail.com"
+  gem.authors = ["Erik Hin-tone"]
+  gem.add_runtime_dependency  "redis", "~>  2.1.1"
+  gem.add_runtime_dependency  "yajl-ruby", ">=  0.7.7"
+  gem.add_runtime_dependency  "text", "~>  0.2.0"
+  gem.add_development_dependency "dm-core", "~> 1.0.0"
+  gem.add_development_dependency "dm-sqlite-adapter", "~> 1.0.0"
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "silver #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/lib/silver/cache.rb

@@ -0,0 +1,152 @@
+#### Hash extensions
+
+# Creates a new class, BareHash, that is alike a Hash in every way except that
+# it may be accessed by a symbol or a string for every key. Really the same thing
+# as HashWithIndifferentAccess but without ActiveSupport
+
+class BareHash < Hash
+
+    def [](key)
+
+        if self.include? key
+            self.fetch(key)
+        else
+            key.class == String ? self.fetch(key.to_sym,nil) : self.fetch(key.to_s, nil)
+        end
+
+    end
+
+end
+
+# Monkey patches Hash to allow for conversion to a BareHash where all values are strings. This
+# is good for mixing results with Redis results which are always stored as strings.
+
+class Hash
+
+    def to_bare
+
+        bhash = BareHash.new
+        self.each{|k,v| bhash[k] = v}
+        bhash
+
+    end
+
+end
+
+#### Caching
+
+module Silver
+
+  class Cache
+    
+    attr_reader :key, :time_field, :query
+
+    # Creates a new cached search object.
+    #
+    # Silver does not connect to a database, the query is passed as a block. This means you can use
+    # Silver to cache databases, REST APIs, or anything else that can be queried.
+    #
+    # key is string to identify this and successive queries to Redis
+    # time_field is the name of a field used to determine whether or not there are new entries that
+    # are not yet cached.
+    # query is a block that take a date and queries the database for all entries after that date. The results
+    # must be returned in descending order.
+    # 
+    # Example to prepare a cache and query for the database of new stories in blog #2:
+    #
+    #     cache = Silver::Cache.new("news_stories",
+    #                               "created_time") do |date|
+    #       
+    #       Stories.all(:order => :created_time.desc,
+    #                   :created_time.gt => date
+    #                   :blog_id => 2)
+    #
+    #     end 
+
+
+    def initialize(key,time_field,redis_options={},&query)
+    
+        @key = key
+        @time_field = time_field
+        @query = query
+        @r = Redis.new(redis_options)
+        @r.select 12
+    
+    end
+
+    # Queries Redis, returns new entries and inserts them into Redis.
+    # 
+    # update is an optional parameter that, if false, will just return cached results and not look for new ones.
+    # callback is block that gets called for every new results, receives the result
+    # and returns the hash to be cached. This can used to query associations.
+    #
+    # Example to cache and the query the database and include any categories the entry might have:
+    #
+    #     cache.find do |entry| 
+    #       attrs = entry.attributes
+    #       cats = {:categories => entry.categories}
+    #       attrs.merge cats
+    #     end
+      
+    def find(update=true,&callback)
+      
+      old_results = @r.lrange(@key,0,-1).map{|q| JSON.parse(q)}
+     
+      if update
+        last_date = @r.get("#{@key}:last") || "1970-01-01"
+        new_results = @query.call(DateTime.parse(last_date))
+        results = new_results.map do |result| 
+          callback.call(result)
+        end 
+
+        if results.empty?
+            final_results = old_results
+        else 
+            write_new(results)
+            
+            # Why do we go back to Redis here instead of just merging old and new? Because it's faster and cleaner than 
+            # selectively determining which types are changed by the to_json (like Dates) and which are preservered (like
+            # Hashes).
+
+            final_results = @r.lrange(@key,0,-1).map{|q| JSON.parse(q)}
+        end
+      else
+        final_results = old_results
+      end
+      
+      final_results = final_results.map do |result| 
+          result.to_bare
+      end
+    end
+
+    # A helper method to keep the redis list at a reasonable size.
+    # 
+    # length is the number of entries to reduce the redis to 
+
+    def cull(length)
+
+        @r.ltrim(@key,0,length-1)
+
+    end
+    
+    private
+
+    # Writes the results to redis by pushing them in reverse order on the head
+    # of the redis list. This ensures that the first result will always be the newest.
+    # Also turns every result hash into JSON before writing because Redis is string based.
+    # Find will automatically parse these JSON strings upon retrieval.
+
+    def write_new(results)
+          
+          new_date = results[0][@time_field].to_s
+          @r.set("#{@key}:last",new_date)
+          
+          results.reverse.each do |result|
+            @r.lpush(@key,result.to_json)
+          end
+
+    end
+
+  end
+
+end

data/lib/silver/common_words.rb

@@ -0,0 +1 @@
+COMMON_WORDS = %w[about, after, again, air, all, along, also, an, and, another, any, are, around, as, at, away, back, be, because, been, before, below, between, both, but, by, came, can, come, could, day, did, different, do, does, don't, down, each, end, even, every, few, find, first, for, found, from, get, give, go, good, great, had, has, have, he, help, her, here, him, his, home, house, how, I, if, in, into, is, it, its, just, know, large, last, left, like, line, little, long, look, made, make, man, many, may, me, men, might, more, most, Mr., must, my, name, never, new, next, no, not, now, number, of, off, old, on, one, only, or, other, our, out, over, own, part, people, place, put, read, right, said, same, saw, say, see, she, should, show, small, so, some, something, sound, still, such, take, tell, than, that, the, them, then, there, these, they, thing, think, this, those, thought, three, through, time, to, together, too, two, under, up, us, use, very, want, water, way, we, well, went, were, what, when, where, which, while, who, why, will, with, word, work, world, would, write, year, you, your, was]

data/lib/silver/indexer.rb

@@ -0,0 +1,126 @@
+require File.expand_path(File.dirname(__FILE__) + '/common_words.rb')
+
+#### Boring new error
+
+# Just makes a new sublcass of RuntimeError so we can raise database
+# errors later
+
+class AttrError < RuntimeError
+end
+
+#### Indexing
+
+module Silver
+
+    class Index
+
+        # Create a new index for some given database or web service. Silver
+        # doesn't do any connecting to a database, it requires you to pass
+        # a query to it as a block. Use whatever ORM you like best. 
+        #
+        # key is the name this index will be stored with in Redis. 
+        # time_field is the name of the field used for determining whether or not
+        # there are new entries.
+        # query is a block that takes the date of the most recently indexed item
+        # as a parameter. Query should always sort entries in descending order.
+        #
+        # Example to index all large pictures based on their captions:
+        #
+        #      index = Silver::Index.new("picturecaptions",
+        #                                "created_time") do |time|
+        #        
+        #        Pictures.all(:order => :created_time.desc, 
+        #                     :created_time.gt => time, 
+        #                     :size => "large")
+        #      end
+
+        def initialize(key,time_field,redis_options={},&query)
+            @r = Redis.new(redis_options)
+            @r.select 12
+            @key = key
+            @time_field = time_field
+            @query = query
+            @new_results = []
+            @indexed_items = []
+        end
+
+        # Looks for the most recent date indexed and only fetch
+        # entries newer than that date from the database.
+        # 
+        # Takes a block that takes a results from the database and returns the value by which to index.
+        # accessor should be a block that takes an individual results and returns and array containing, first,
+        # the item's id and, second, the value by which to index.
+        #
+        # Ex:
+        #
+        #     index.find_and_update do |result|
+        #       output = result.caption || result.label || "" 
+        #       id = result.id
+        #       [id,output]
+        #     end
+
+        def find_and_update(&accessor)
+
+            last_date = @r.get("#{@key}:last") || "Jan. 1, 1970"
+            new_results = @query.call(DateTime.parse(last_date))
+            if new_results.empty?
+                false
+            else
+                new_date = new_results[0][@time_field].to_s
+                @r.set("#{@key}:last",new_date)
+                @new_results = new_results
+                parse(accessor)
+                true
+            end
+        end
+
+
+        private
+        
+        # Sends results off to a double metaphone method to allow for fuzzy searching. 
+
+        def parse(accessor)
+
+            @new_results.each do |result|
+                begin 
+                    value = accessor.call(result)
+                    time = Time.parse(result[@time_field].to_s).to_i
+                    raise AttrError, "Specified attribute not found in item #{value[0]}" if !value
+                    morphed_words = morph(value[1])
+                    morphed_words.each{|word| write(word,value[0],time)}
+                rescue AttrError => e
+                    puts e
+                end
+           
+            end
+           
+        end
+        
+        # Takes a metaphoned string, the id of the row that contains it, and the time created/modified of that row
+        # and writes to the database to phonemes by which that row is now indexed.
+
+        def write(word,id,time)
+            
+            text = word[0]
+            phonemes = word[1].compact
+            phonemes.each do |phoneme|
+                @r.zadd "#{@key}:#{phoneme}", time, id
+            end
+        end
+
+        # Note: this method needs some work/ is not perfect.
+        # Takes a string, removes any file extension, splits it on non-alpha characters, throws out empty words
+        # words smaller than 5 characters that are not capitalized and all common words. Finally it converts each
+        # word to its metaphoned equivalents and returns an array of the original words in its metaphones.
+
+        def morph(string)
+            string.gsub!(/\.[a-zA-Z]{1,4}$/,"")
+            words = string.split(/[^a-zA-Z]/).reject{|q| q == "" || (q.length < 5 && q.capitalize != q)} - COMMON_WORDS 
+            morphed_words = words.map{|q| [q,Text::Metaphone.double_metaphone(q)]}
+            morphed_words
+        end
+
+    end
+
+end
+

data/lib/silver/search.rb

@@ -0,0 +1,71 @@
+module Silver
+
+    #### Search
+
+    # Searches an indexed database. What else would it do?
+
+    class Search
+
+        # Takes a query and a redis key from a previous indexing. There is an optional offset
+        # that can be used to paginate. It defaults to returning 30 results.
+        # 
+        # Example, query picture captions for "Barack Obama":
+        #     search = Silver::Search.new("barack obama","picturecaption")
+       
+        def initialize(query,key,redis_options={},number=30,offset=0)
+            @query = query
+            @key = key
+            @number = number
+            @offset = offset
+            @r = Redis.new(redis_options)
+            @r.select 12
+        end
+
+        # Send a query to be metaphoned, finds the matching ids for the query and then
+        # returns the results. Finally it only returns entries that are shared by both words.
+        #
+        # Takes a block that takes an id and then queries the database for that row. Again, Silver
+        # can be used for services that "row" is a bad metaphor like REST apis. However, it is easy 
+        # to write.
+        #
+        # Ex:
+        #     search.perform{|id| Picture.get(id) }
+
+        def perform(&accessor)
+            morphed_words = morph_words
+            morphed_words.map! do |word|
+                phones = word[1]
+                phones = self.find_matching_phones(phones)
+                phones
+            end
+            results = morphed_words.reduce{|memo,obj| memo & obj}.slice(@offset,@offset+@number)
+            results.map{|result| accessor.call(result)}
+        end
+
+        # Takes the instance's query, splits it into words, metaphones each word and returns the array of metaphoned words.
+
+        def morph_words
+            words = @query.split(/[^a-zA-Z0-9]/)
+            morphed_words = words.map{|word| [word,Text::Metaphone.double_metaphone(word)]}
+            morphed_words
+        end
+
+        # Takes an array of metaphones and returns the matching keys in the index. Since we are using double metaphone,
+        # it unions the results for the two possible metaphones.i
+
+        def find_matching_phones(phones)
+            phones.map! do |phone|
+                if phone
+                    "#{@key}:#{phone}"
+                else
+                    nil
+                end
+            end
+            phones = @r.zunionstore "temp", phones.compact
+            phones = @r.zrevrange "temp", 0, -1
+            phones
+        end
+
+    end
+
+end

data/lib/silver.rb

@@ -0,0 +1,7 @@
+require 'silver/indexer'
+require 'silver/search'
+require 'silver/cache'
+require 'rubygems'
+require 'redis'
+require 'text'
+require 'yajl/json_gem'

data/silver.gemspec

@@ -0,0 +1,102 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{silver}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Erik Hin-tone"]
+  s.date = %q{2011-01-31}
+  s.description = %q{A lightweight, Redis-backed cacher and indexer for databases, REST API's, really anything you can query.}
+  s.email = %q{hinton.erik@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.markdown"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.markdown",
+    "Rakefile",
+    "VERSION",
+    "lib/silver.rb",
+    "lib/silver/cache.rb",
+    "lib/silver/common_words.rb",
+    "lib/silver/indexer.rb",
+    "lib/silver/search.rb",
+    "silver.gemspec",
+    "spec/silver_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/support/db.rb",
+    "spec/support/spec.db"
+  ]
+  s.homepage = %q{http://github.com/tpm/silver}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Makes your queries faster with the power of Redis.}
+  s.test_files = [
+    "spec/silver_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/support/db.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<redis>, ["~> 2.1.1"])
+      s.add_development_dependency(%q<yajl-ruby>, [">= 0.7.7"])
+      s.add_development_dependency(%q<text>, ["~> 0.2.0"])
+      s.add_development_dependency(%q<dm-core>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<dm-sqlite-adapter>, ["~> 1.0.0"])
+      s.add_runtime_dependency(%q<redis>, ["~> 2.1.1"])
+      s.add_runtime_dependency(%q<yajl-ruby>, [">= 0.7.7"])
+      s.add_runtime_dependency(%q<text>, ["~> 0.2.0"])
+      s.add_development_dependency(%q<dm-core>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<dm-sqlite-adapter>, ["~> 1.0.0"])
+    else
+      s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<redis>, ["~> 2.1.1"])
+      s.add_dependency(%q<yajl-ruby>, [">= 0.7.7"])
+      s.add_dependency(%q<text>, ["~> 0.2.0"])
+      s.add_dependency(%q<dm-core>, ["~> 1.0.0"])
+      s.add_dependency(%q<dm-sqlite-adapter>, ["~> 1.0.0"])
+      s.add_dependency(%q<redis>, ["~> 2.1.1"])
+      s.add_dependency(%q<yajl-ruby>, [">= 0.7.7"])
+      s.add_dependency(%q<text>, ["~> 0.2.0"])
+      s.add_dependency(%q<dm-core>, ["~> 1.0.0"])
+      s.add_dependency(%q<dm-sqlite-adapter>, ["~> 1.0.0"])
+    end
+  else
+    s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<redis>, ["~> 2.1.1"])
+    s.add_dependency(%q<yajl-ruby>, [">= 0.7.7"])
+    s.add_dependency(%q<text>, ["~> 0.2.0"])
+    s.add_dependency(%q<dm-core>, ["~> 1.0.0"])
+    s.add_dependency(%q<dm-sqlite-adapter>, ["~> 1.0.0"])
+    s.add_dependency(%q<redis>, ["~> 2.1.1"])
+    s.add_dependency(%q<yajl-ruby>, [">= 0.7.7"])
+    s.add_dependency(%q<text>, ["~> 0.2.0"])
+    s.add_dependency(%q<dm-core>, ["~> 1.0.0"])
+    s.add_dependency(%q<dm-sqlite-adapter>, ["~> 1.0.0"])
+  end
+end
+

data/spec/silver_spec.rb

@@ -0,0 +1,263 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require File.expand_path(File.dirname(__FILE__) + '/support/db.rb')
+
+describe "indexer" do
+
+   it "correctly metaphones searches" do
+       
+       search = Silver::Search.new("Barack Obama","specparents")
+       morph = search.morph_words
+       morph.should eq([["Barack",["PRK",nil]],["Obama",["APM",nil]]])
+
+   end
+
+   it "correctly metaphones items" do
+
+       index = Silver::Index.new("specparents","date") do |date|
+           Parent.all(:order => :date.desc, :date.gt => date)
+       end
+       morph = index.send("morph","barack-obama.JpEg")
+       morph.should eq([["barack",["PRK",nil]],["obama",["APM",nil]]])
+
+   end
+
+   it "indexes the database" do
+       r = Redis.new
+       r.select 12
+       keys = r.keys("specparents:*")
+       keys.each{|key| r.del key}
+       r.keys("specparents:*").should eq([])
+
+       index = Silver::Index.new("specparents","date") do |date|
+           Parent.all(:order => :date.desc, :date.gt => date)
+       end
+       
+       output = index.find_and_update do |result|
+           output = result.name 
+           id = result.id
+           [id,output]
+       end
+       
+       r.keys("specparents:*").should include("specparents:PS")
+       r.keys("specparents:*").should include("specparents:ARK")
+
+   end
+
+
+   it "returns false when there are no new updates" do
+
+       index = Silver::Index.new("specparents","date") do |date|
+           Parent.all(:order => :date.desc, :date.gt => date)
+       end
+       
+       output = index.find_and_update do |result|
+           output = result.name 
+           id = result.id
+           [id,output]
+       end
+
+       output.should eq(false)
+
+   end
+
+   it "searches the index" do
+       
+       search = Silver::Search.new("Erik","specparents")
+       results = search.perform{|id| Parent.get(id)}
+       results[0]["name"].should eq("Erik")
+       results[0]["age"].should eq(24)
+       search = Silver::Search.new("Erik Hinton","specparents")
+       results = search.perform{|id| Parent.get(id)}
+       results[0]["name"].should eq("Erik Hinton")
+       results[0]["age"].should eq(2)
+
+   end
+   
+   it "updates the index" do
+
+       r = Redis.new
+       r.select 12
+
+       index = Silver::Index.new("specparents","date") do |date|
+           Parent.all(:order => :date.desc, :date.gt => date)
+       end
+
+       p = Parent.create(:name => "Update",
+                         :age => 1,
+                         :date => DateTime.now)
+
+       output = index.find_and_update do |result|
+           output = result.name 
+           id = result.id
+           [id,output]
+       end
+
+       output.should eq(true)
+       r.keys("specparents:*").should include("specparents:APTT")
+       p.destroy
+
+   end
+
+end
+
+describe "cacher" do
+
+   it "caches new additions to the database" do
+       r = Redis.new
+       r.select 12
+       r.del "parents"
+       r.del "parents:last"
+
+       cache = Silver::Cache.new("parents","date") do |date|
+           Parent.all(:order => :date.desc, :date.gt => date)
+       end
+
+       results = cache.find do |result|
+         result.attributes
+       end
+       results.should eq([{"name"=>"Baz", "id"=>2, "age"=>33, "date"=>"2011-01-28T10:40:45-05:00"}, {"name"=>"Erik", "id"=>1, "age"=>24, "date"=>"2011-01-28T10:34:10-05:00"}, {"name"=>"Erik Hinton", "id"=>11, "age"=>2, "date"=>"1980-01-02T00:00:00+00:00"}])
+       cached_results = r.lrange "parents",0,-1
+       cached_results.map{|result| JSON.parse(result)}.should eq([{"name"=>"Baz", "id"=>2, "age"=>33, "date"=>"2011-01-28T10:40:45-05:00"}, {"name"=>"Erik", "id"=>1, "age"=>24, "date"=>"2011-01-28T10:34:10-05:00"}, {"name"=>"Erik Hinton", "id"=>11, "age"=>2, "date"=>"1980-01-02T00:00:00+00:00"}])
+   end
+
+   it "caches associations" do
+       r = Redis.new
+       r.select 12
+       r.del "parents"
+       r.del "parents:last"
+
+       cache = Silver::Cache.new("parents","date") do |date|
+           Parent.all(:order => :date.desc, :date.gt => date)
+       end
+
+       results = cache.find do |parent|
+           children = parent.children
+           names = children.map{|child| child["name"]}
+           attrs = parent.attributes
+           c = {:children => names}
+           attrs.merge c
+       end
+
+       results.should eq([{"name"=>"Baz", "id"=>2, "age"=>33, "date"=>"2011-01-28T10:40:45-05:00", "children"=>["Bar"]}, 
+                         {"name"=>"Erik", "id"=>1, "age"=>24, "date"=>"2011-01-28T10:34:10-05:00", "children"=>["Foo"]}, 
+                         {"name"=>"Erik Hinton", "id"=>11, "age"=>2, "date"=>"1980-01-02T00:00:00+00:00", "children"=>[]}])
+       cached_results = r.lrange "parents",0,-1
+       cached_results.map{|result| JSON.parse(result)}.should eq([{"name"=>"Baz", "id"=>2, "age"=>33, "date"=>"2011-01-28T10:40:45-05:00", "children"=>["Bar"]}, 
+                         {"name"=>"Erik", "id"=>1, "age"=>24, "date"=>"2011-01-28T10:34:10-05:00", "children"=>["Foo"]}, 
+                         {"name"=>"Erik Hinton", "id"=>11, "age"=>2, "date"=>"1980-01-02T00:00:00+00:00", "children"=>[]}])
+
+   end
+
+   it "retrieves just old results" do
+
+      cache = Silver::Cache.new("parents","date") do |date|
+          Parent.all(:order => :date.desc, :date.gt => date)
+      end
+      results = cache.find(false)
+      results.should eq([{"name"=>"Baz", "id"=>2, "age"=>33, "date"=>"2011-01-28T10:40:45-05:00", "children"=>["Bar"]}, 
+                         {"name"=>"Erik", "id"=>1, "age"=>24, "date"=>"2011-01-28T10:34:10-05:00", "children"=>["Foo"]}, 
+                         {"name"=>"Erik Hinton", "id"=>11, "age"=>2, "date"=>"1980-01-02T00:00:00+00:00", "children"=>[]}])
+
+   end
+
+end
+
+describe "barehash" do
+
+    it "makes an indifferent hash" do
+        a = BareHash.new
+        a[:name] = "Erik"
+        a["age"] = 14
+        a["name"].should eq("Erik")
+        a[:age].should eq(14)
+    end
+
+    it "converts a hash to a barehash" do
+
+        a = {:name => "Erik", "age" => 24, "DOB" => DateTime.parse("Jan. 3, 1987")}
+        b = a.to_bare
+        b["name"].should eq("Erik")
+        b["age"].should eq(24)
+        b["DOB"].should eq(DateTime.parse("Jan. 3, 1987"))
+
+    end
+
+end
+
+#describe "reader" do
+
+    #it "reads the database" do
+        #index = Silver::Index.new("Asset","label",{:id => 15})
+        #items = index.read
+        #items[0].label.should eq("petraeus-testify-med.jpg")
+    #end
+
+#end
+
+#describe "parser" do
+    #it "registers the correct time" do
+        #index = Silver::Index.new("Asset","label",{:id => 15})
+        #items = index.read
+        #parsed_item = index.parse(items[0])
+        #parsed_item[0].should eq(1207685164)
+    #end
+
+    #it "parses the string" do
+        #index = Silver::Index.new("Asset","label",{:id => 15})
+        #items = index.read
+        #parsed_item = index.parse(items[0])
+        #parsed_item[1].should eq([["petraeus", ["PTRS", nil]], ["testify", ["TSTF", nil]], ["med", ["MT", nil]]])
+    #end
+#end
+
+#describe "morpher" do
+    #it "breaks strings into words" do
+        #string = "barack-obama-happy.jpg"
+        #parsed_string = Silver::Index.morph(string)
+        #parsed_string[0][0].should eq("barack")
+        #parsed_string[2][0].should eq("happy")
+    #end
+    #it "correctly sounds out words" do
+        #string = "cat-hard-tack.jpg"
+        #parsed_string = Silver::Index.morph(string)
+        #parsed_string[0][1].should eq(["KT",nil])
+        #parsed_string[2][1].should eq(["TK",nil])
+    #end
+#end
+
+#describe "writer" do
+
+    #it "writes to redis" do
+        #index = Silver::Index.new("Asset","caption/label",{:parent => nil, :limit => 2})
+        #r = Redis.new
+        #r.select 4
+        #items = index.read
+        #if items == []
+            #puts "no new items to test"
+        #else
+            #parsed_item = index.parse(items[1])
+            #index.process
+            #r.keys.should include("captionlabel:#{parsed_item[1][0][1][0]}")
+        #end
+    #end
+
+#end
+
+#describe "searcher" do
+   
+    #it "finds matching phones" do
+        #search = Silver::Search.new("obama","Asset","captionlabel",0)
+        #results = search.find_matching_phones(["SMT","XMT"])
+        #results.should include("36662")
+    #end
+
+    #it "perform searches" do
+        #search = Silver::Search.new("barack obama","Asset","captionlabel",0)
+        #results = search.perform
+        #results.length.should eq(30)
+        #words = (results[0].caption || "")+" "+(results[0].label || "")
+        #results[0].class.should eq(Asset)
+        #words.should =~ /obama/i
+    #end
+
+#end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rubygems'
+require 'rspec'
+require 'silver'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end

data/spec/support/db.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'dm-core'
+
+DB = "sqlite://#{File.expand_path(File.dirname(__FILE__))}/spec.db"
+
+DataMapper.setup(:default,DB)
+DataMapper.finalize
+
+class Parent
+    include DataMapper::Resource
+
+    property :id, Serial
+    property :name, String
+    property :date, DateTime
+    property :age, Integer
+
+    has n, :children
+end
+
+class Child
+    include DataMapper::Resource
+
+    property :id, Serial
+    property :name, String
+    property :wow_factor, Integer
+
+    belongs_to :parent
+end
+

metadata

@@ -0,0 +1,308 @@
+--- !ruby/object:Gem::Specification 
+name: silver
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  - 0
+  version: 0.2.0
+platform: ruby
+authors: 
+- Erik Hin-tone
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-31 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 2
+        - 3
+        - 0
+        version: 2.3.0
+  name: rspec
+  requirement: *id001
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  name: bundler
+  requirement: *id002
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  name: jeweler
+  requirement: *id003
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  name: rcov
+  requirement: *id004
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 2
+        - 1
+        - 1
+        version: 2.1.1
+  name: redis
+  requirement: *id005
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 0
+        - 7
+        - 7
+        version: 0.7.7
+  name: yajl-ruby
+  requirement: *id006
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  name: text
+  requirement: *id007
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  name: dm-core
+  requirement: *id008
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  name: dm-sqlite-adapter
+  requirement: *id009
+- !ruby/object:Gem::Dependency 
+  type: :runtime
+  prerelease: false
+  version_requirements: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 2
+        - 1
+        - 1
+        version: 2.1.1
+  name: redis
+  requirement: *id010
+- !ruby/object:Gem::Dependency 
+  type: :runtime
+  prerelease: false
+  version_requirements: &id011 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 0
+        - 7
+        - 7
+        version: 0.7.7
+  name: yajl-ruby
+  requirement: *id011
+- !ruby/object:Gem::Dependency 
+  type: :runtime
+  prerelease: false
+  version_requirements: &id012 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  name: text
+  requirement: *id012
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id013 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  name: dm-core
+  requirement: *id013
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  version_requirements: &id014 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  name: dm-sqlite-adapter
+  requirement: *id014
+description: A lightweight, Redis-backed cacher and indexer for databases, REST API's, really anything you can query.
+email: hinton.erik@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.markdown
+files: 
+- .document
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.markdown
+- Rakefile
+- VERSION
+- lib/silver.rb
+- lib/silver/cache.rb
+- lib/silver/common_words.rb
+- lib/silver/indexer.rb
+- lib/silver/search.rb
+- silver.gemspec
+- spec/silver_spec.rb
+- spec/spec_helper.rb
+- spec/support/db.rb
+- spec/support/spec.db
+has_rdoc: true
+homepage: http://github.com/tpm/silver
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Makes your queries faster with the power of Redis.
+test_files: 
+- spec/silver_spec.rb
+- spec/spec_helper.rb
+- spec/support/db.rb