scripsi 0.0.1

data/Manifest

@@ -0,0 +1,4 @@
+README.md
+Rakefile
+lib/scripsi.rb
+Manifest

data/README.md

@@ -0,0 +1,29 @@
+# Scripsi
+
+A flexible text-searching library built on top of redis.
+
+## Sorted suffix indexing
+
+Sorted suffix indexing allows you to search for any substring within a set of documents. First, index a collection of documents and associated ids:
+
+    require 'scripsi'
+    Scripsi.connect  # connect to a running redis server
+
+    ssi = Scripsi::SortedSuffixIndexer.new "myindexer"
+    ssi.index(1,"Epistulam ad te scripsi.")
+    ssi.index(2,"I've written you a letter.")
+    ssi.index(3,"Quisnam Tusculo espistulam me misit?")
+    ssi.index(4,"Who in Tusculum would've sent me a letter?")
+
+You can then search for any substring, and the indexer will return the ids of the documents where that substring appears.
+
+    ssi = Scripsi.indexer "myindexer"
+    ssi.search("te")        # => [1,2,4]
+    ssi.search("Tuscul")    # => [3,4]
+    ssi.search("Tusculu")   # => [4]
+    ssi.search("you a le")  # => [2]
+
+You can also retrive the stored documents efficiently:
+
+    ssi.documents  # lazy list of documents
+    ssi.documents[3]  # document with id string

data/Rakefile

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'rake'
+require 'echoe'
+
+Echoe.new('scripsi','0.0.1') do |p|
+  p.description = "a flexible text-searching library built on top of redis"
+  p.url = "https://github.com/alecbenzer/scripsi"
+  p.author = "Alec Benzer"
+  p.email = "alecbenzer@gmail.com"
+  p.ignore_pattern = ["*.rdb"]
+  p.development_dependencies = ["redis >=2.1.1"]
+end

data/lib/scripsi.rb

@@ -0,0 +1,184 @@
+require 'redis'
+require 'set'
+
+module Scripsi
+  # connect to a redis server
+  def self.connect(options = {})
+    @@redis = Redis.new(options)
+  end
+
+  def self.redis
+    @@redis
+  end
+
+  @@partition_size = 10
+
+  def self.partition_size
+    @@partition_size
+  end
+
+  # generate a 'score' for a string
+  # used for storing it in a sorted set
+  #
+  # This method effectively turns a string into a base 27 floating point number,
+  # where 0 corresponds to no letter, 1 to A, 2 to B, etc.
+  #
+  # @param [String] str the string we are computing a score for
+  # @return [Number] the string's score
+  def self.score(str)
+    str = str.downcase
+    scrs = []
+    str.split('').each_slice(partition_size) do |s|
+      mult = 1.0
+      scr = 0.0
+      s.each do |char|
+        mult /= 27
+        scr += (char.ord-'a'.ord+1)*mult if ('a'..'z').include? char
+      end
+      scrs << scr
+    end
+    scrs
+  end
+
+  # get the indexer with the given id
+  def self.indexer(id)
+    type = Scripsi.redis.hget "scripsi:used", id.to_s
+    if type == "ssi"
+      SortedSuffixIndexer.build(id)
+    end
+  end
+
+  # (see #indexer)
+  def self.find(id)
+    indexer(id)
+  end
+
+  class SortedSuffixIndexer
+    def initialize(id=nil,check=true)
+      if check
+        if id and Scripsi.redis.hexists "scripsi:used", id.to_s
+          raise "id '#{id}' in use"
+        end
+        @id = id ? id.to_s : Scripsi.redis.incr("scripsi:next_id")
+        Scripsi.redis.hset "scripsi:used", @id, "ssi"
+      else
+        @id = id
+      end
+      @index_key = "scripsi:index:#{@id}"
+      @document_key = "scripsi:document:#{@id}"
+      @documents_key = "scripsi:documents:#{@id}"
+      @search_length = 30
+    end
+
+    # adds a document to this indexer
+    #
+    # @param [Integer] id a number representing the id of this document
+    # @param [String] str the text of the document
+    # @return [Boolean] returns true if the document was successfully indexed
+    def index(id,str)
+      id = id.to_s
+      return false if Scripsi.redis.hexists @documents_key, id
+      offset = Scripsi.redis.strlen @document_key
+      sfxs = suffixes(str).sort_by{|s,i| s}
+      sfxs.each do |suffix,i|
+        Scripsi.score(suffix).each_with_index do |scr,j|
+          Scripsi.redis.zadd "#{@index_key}:#{j}", scr, i+offset
+        end
+      end
+      doc = str + "\0#{id}\0"
+      Scripsi.redis.append @document_key, doc
+      endpoints = Marshal.dump([offset, offset + str.size - 1])
+      Scripsi.redis.hset @documents_key, id, endpoints
+    end
+
+    # a lazy list of documents associated with a SortedSuffixIndexer
+    class Documents
+      def initialize(doc_key, endpoints_key)
+        @doc_key = doc_key
+        @endpoints_key = endpoints_key
+      end
+
+      def [](id)
+        endpoints = Scripsi.redis.hget(@endpoints_key, id)
+        return nil unless endpoints
+        a,b = Marshal.load(endpoints)
+        Scripsi.redis.getrange @doc_key, a.to_i, b.to_i
+      end
+    end
+
+    # retrive the document with the given id
+    def documents
+      Documents.new(@document_key,@documents_key)
+    end
+
+    # searches for documents containing the substring term
+    #
+    # @param [String] term the substring to search for
+    def search(term)
+      term, length = term.downcase, term.length
+      set = nil
+      Scripsi.score(term).each_with_index do |scr,i|
+        a,b = scr.to_s, "#{scr+1.0/(27**length)}"
+        b = "(" + b unless a == b
+        ids = Scripsi.redis.zrangebyscore("#{@index_key}:#{i}",a,b)
+        set = set ? set & Set.new(ids) : Set.new(ids)
+        length -= Scripsi.partition_size
+      end
+      set.map{|i| read_to_id(i.to_i)}.uniq
+    end
+
+    # creates an indexer with the given id WITHOUT CHECKING
+    # this method is used internally - calling it yourself may result in deleting an indexer, unless you know the id you're using is valid
+    def self.build(id)
+      new(id,false)
+    end
+
+    def inspect
+      "#<Scripsi::SortedSuffixIndexer id=#{@id}>"
+    end
+
+    private
+
+    def suffixes(str)
+      str = str.downcase
+      (0...str.length).map {|i| [str[i..-1],i] }
+    end
+
+    def document_index(index)
+      doc_index = Scripsi.redis.zrange(@index_key, index, index).first.to_i
+    end
+
+    def compare_with_index(str,doc_index)
+      str.split('').each_with_index do |char,offset|
+        s = Scripsi.redis.getrange @document_key, doc_index+offset, doc_index+offset
+        STDERR.puts "comparing #{char} and #{s.downcase}"
+        comp = char <=> s.downcase
+        return comp unless comp == 0
+      end
+      0
+    end
+
+    def read_to_id(doc_index)
+      last = Scripsi.redis.strlen @document_key
+      (doc_index..last).each do |i|
+        char = Scripsi.redis.getrange(@document_key, i, i)
+        if char == "\0"
+          id = ""
+          offset = 1
+          loop do
+            next_char = Scripsi.redis.getrange(@document_key,i+offset,i+offset)
+            if next_char == "\0"
+              break
+            else
+              id << next_char
+              offset += 1
+            end
+          end
+          return id.to_i
+        end
+      end
+      raise "index is corrupt"
+    end
+  end
+
+end

data/scripsi.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{scripsi}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Alec Benzer"]
+  s.date = %q{2011-02-09}
+  s.description = %q{a flexible text-searching library built on top of redis}
+  s.email = %q{alecbenzer@gmail.com}
+  s.extra_rdoc_files = ["README.md", "lib/scripsi.rb"]
+  s.files = ["README.md", "Rakefile", "lib/scripsi.rb", "Manifest", "scripsi.gemspec"]
+  s.homepage = %q{https://github.com/alecbenzer/scripsi}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Scripsi", "--main", "README.md"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{scripsi}
+  s.rubygems_version = %q{1.5.0}
+  s.summary = %q{a flexible text-searching library built on top of redis}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<redis>, [">= 2.1.1"])
+    else
+      s.add_dependency(%q<redis>, [">= 2.1.1"])
+    end
+  else
+    s.add_dependency(%q<redis>, [">= 2.1.1"])
+  end
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification 
+name: scripsi
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Alec Benzer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-09 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.1.1
+  type: :development
+  version_requirements: *id001
+description: a flexible text-searching library built on top of redis
+email: alecbenzer@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+- lib/scripsi.rb
+files: 
+- README.md
+- Rakefile
+- lib/scripsi.rb
+- Manifest
+- scripsi.gemspec
+has_rdoc: true
+homepage: https://github.com/alecbenzer/scripsi
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Scripsi
+- --main
+- README.md
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "1.2"
+requirements: []
+
+rubyforge_project: scripsi
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: a flexible text-searching library built on top of redis
+test_files: []
+