code_zauker 0.0.1 → 0.0.2

data/lib/code_zauker/constants.rb

@@ -0,0 +1,24 @@
+module CodeZauker
+  DEFAULT_EXCLUDED_EXTENSION=[
+                              ".pdf",
+                              ".xps",
+                              ".zip",
+                              ".ppt",".xls",".rtf",".vsd",  
+                              ".dll",".exe",".out",".elf",".lib",".so",
+                              ".jar",".class",
+                              ".tar",
+                              ".gz",
+                              ".dropbox",
+                              ".svn-base",".pdb",".cache",
+                              # MS Office zip-like files...
+                              ".pptx",".docx",".xlsx",
+                              # Music exclusion
+                              ".mp3",".mp4",".wav",
+                              # Image exclusion
+                              ".png",".gif",
+                              # Temp stuff
+                              ".tmp","~",
+                              # Oracle exports...
+                              ".exp"
+                              ]
+end

data/lib/code_zauker/grep.rb

@@ -0,0 +1,138 @@
+# GG Customized to suite Code Zauker needs
+# Refer to https://rubygems.org/gems/grep
+# for the original gem/code
+# Skeleton module for the 'grep' routine.
+#
+# Ideally, one would do this in their code to import the "grep" call
+# directly into their current namespace:
+#
+#     require 'grep'
+#     include Grep
+#     # do something with grep()
+#
+#
+# It is recommended that you look at the documentation for the grep()
+# call directly for specific usage.
+#
+#--
+#
+# The compilation of software known as grep.rb is distributed under the
+# following terms:
+# Copyright (C) 2005-2006 Erik Hollensbe. All rights reserved.
+#
+# Redistribution and use in source form, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+#
+# THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+#++
+
+module Grep
+
+  #
+  # Grep works like a shell grep. `file' can be either a string,
+  # containing the name of a file to load and handle, or an IO object
+  # (such as $stdin) to deal with. `pattern' can be either a string or
+  # Regexp object which contains a pattern. Patterns as strings treat
+  # no part of the string as `special', such as '.' or '?' in a
+  # regex. `pre_context' and `post_context' determine the amount of
+  # lines to return that came before or after the content that was
+  # matched, respectively. If there are overlaps in the context, no
+  # duplicates will be printed.
+  #
+
+  def grep(file, pattern, pre_context=0, post_context=0, print_filename=true)
+    currentline=0
+    if file.kind_of? String
+      fileName=file
+      file = File.new(file, "r")
+    else
+      fileName=""
+    end
+
+    if ! file.kind_of? IO
+      throw IOError.new("File must be the name of an existing file or IO object")
+    end
+
+    if pattern.kind_of? String
+      pattern = /#{Regexp.escape(pattern)}/
+    end
+
+    if ! pattern.kind_of? Regexp
+      throw StandardError.new("Pattern must be string or regexp")
+    end
+
+    cache = []
+    lines = []
+
+    loop do
+      begin
+        line = file.readline
+        currentline +=1
+        cache.shift unless cache.length < pre_context
+
+      # GG Patch
+      # if print_filename==true
+      #   cache.push("#{fileName}:#{line}")
+      # else
+        cache.push("#{currentline}:#{line}")
+      # end
+
+
+
+        
+        if line =~ pattern
+          lines += cache
+          cache = []
+          if post_context > 0
+            post_context.times do
+              begin
+                lines.push(file.readline) 
+                currentline +=1
+              rescue IOError => e
+                break
+              end
+            end
+          end
+        end
+      rescue IOError => e
+        break
+      end
+    end
+    
+
+    file.each_line do |line|
+      cache.shift unless cache.length < pre_context
+      cache.push(line)
+
+      if line =~ pattern
+        lines += cache
+        if post_context > 0
+          post_context.times do
+            begin
+              lines.push(file.readline) 
+              currentline +=1
+            rescue Exception => e
+              break
+            end
+          end
+        end
+      end
+    end
+
+    return lines
+  end
+end

data/lib/code_zauker/version.rb

@@ -1,3 +1,3 @@
 module CodeZauker
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/code_zauker.rb

@@ -1,10 +1,11 @@
 # -*- mode:ruby ; -*- -*
 require "code_zauker/version"
+require "code_zauker/constants"
 require 'redis/connection/hiredis'
 require 'redis'
 require 'set'
-# This module try to implement a simple reverse indexer 
-# based on redis
+# This module implements a simple reverse indexer 
+# based on Redis
 # The idea is ispired by http://swtch.com/~rsc/regexp/regexp4.html
 module CodeZauker
   GRAM_SIZE=3
@@ -12,19 +13,52 @@ module CodeZauker
   # Scan a file and push it inside redis...
   # then it can provide handy method to find file scontaining the trigram...
   class FileScanner
-    def initialize()
+    def initialize(redisConnection=nil)
+      if redisConnection==nil
+        @redis=Redis.new
+      else
+        @redis=redisConnection
+      end
+    end
+    def disconnect()
+      @redis.quit
     end
+
+   
+
+    def pushTrigramsSet(s, fid, filename)
+      error=false
+      if s.length > 5000
+        puts " >Pushing...#{s.length} for id #{fid}=#{filename}"
+      end
+      s.each do | trigram |
+        @redis.sadd "trigram:#{trigram}",fid
+        @redis.sadd "fscan:trigramsOnFile:#{fid}", trigram
+        # Add the case-insensitive-trigram
+        begin
+          @redis.sadd "trigram:ci:#{trigram.downcase}",fid
+        rescue ArgumentError 
+          error=true          
+        end
+      end
+      if s.length > 5000
+        puts " <Pushed #{s.length}..."
+        puts "WARN: Some invalid UTF-8  char on #{filename} Case insensitive search will be compromised" if error
+      end
+    end
+
+    private :pushTrigramsSet
+
     def load(filename, noReload=false)
-      # Define my redis id...
-      r=Redis.new
+      # Define my redis id...      
       # Already exists?...
-      fid=r.get "fscan:id:#{filename}"
+      fid=@redis.get "fscan:id:#{filename}"
       if fid==nil 
-        r.setnx "fscan:nextId",0
-        fid=r.incr "fscan:nextId"
+        @redis.setnx "fscan:nextId",0
+        fid=@redis.incr "fscan:nextId"
         # BUG: Consider storing it at the END of the processing
-        r.set "fscan:id:#{filename}", fid
-        r.set "fscan:id2filename:#{fid}",filename
+        @redis.set "fscan:id:#{filename}", fid
+        @redis.set "fscan:id2filename:#{fid}",filename
       else
         if noReload 
           puts "Already found #{filename} as id:#{fid} and NOT RELOADED"
@@ -54,12 +88,7 @@ module CodeZauker
             # push the trigram to redis (highly optimized)
             s.add(trigram)
             if s.length > adaptiveSize
-              puts " >Pushing...#{s.length}"
-              s.each do | trigram |
-                r.sadd "trigram:#{trigram}",fid
-                r.sadd "fscan:trigramsOnFile:#{fid}", trigram
-              end
-              puts " <Pushed #{s.length}..."
+              pushTrigramsSet(s,fid,filename)
               s=Set.new()             
             end
             trigramScanned += 1
@@ -69,19 +98,18 @@ module CodeZauker
       end
 
       if s.length > 0
-        s.each do | trigram |
-          r.sadd "trigram:#{trigram}",fid
-          r.sadd "fscan:trigramsOnFile:#{fid}", trigram
-        end
+        pushTrigramsSet(s,fid,filename)
+        s=nil
         #puts "Final push of #{s.length}"
       end
 
 
-      trigramsOnFile=r.scard "fscan:trigramsOnFile:#{fid}"
-      r.sadd "fscan:processedFiles", "fscan:id:#{filename}"
+      trigramsOnFile=@redis.scard "fscan:trigramsOnFile:#{fid}"
+      @redis.sadd "fscan:processedFiles", "#{filename}"
       trigramRatio=( (trigramsOnFile*1.0) / trigramScanned )* 100.0
-      puts "File processed. Unique Trigrams for #{filename}: #{trigramsOnFile} Total Scanned: #{trigramScanned} Ratio:#{trigramRatio}"
-      r.quit
+      if trigramRatio < 10 or trigramRatio >75
+        puts "#{filename}\n\tRatio:#{trigramRatio.round}%  Unique Trigrams:#{trigramsOnFile} Total Scanned: #{trigramScanned} "      
+      end
       return nil
     end
 
@@ -89,6 +117,9 @@ module CodeZauker
     # Find a list of file candidates to a search string
     # The search string is padded into trigrams
     def search(term)
+      if term.length < GRAM_SIZE
+        raise "FATAL: #{term} is shorter then the minimum size of #{GRAM_SIZE} character"
+      end
       #puts " ** Searching: #{term}"
       # split the term in a padded trigram      
       trigramInAnd=[]
@@ -104,25 +135,65 @@ module CodeZauker
       #puts "Trigam conversion /#{term}/ into #{trigramInAnd}"
       if trigramInAnd.length==0
         return []
-      end
-      r=Redis.new      
-      fileIds=    r.sinter(*trigramInAnd)
+      end      
+      fileIds=    @redis.sinter(*trigramInAnd)
       filenames=[]
       # fscan:id2filename:#{fid}....
       fileIds.each do | id |
-        filenames.push(r.get("fscan:id2filename:#{id}")) 
-      end
-      r.quit
+        filenames.push(@redis.get("fscan:id2filename:#{id}")) 
+      end      
       #puts " ** Files found:#{filenames} from ids #{fileIds}"
       return filenames
     end
-    
-    # This function accepts a very simple search query like
-    # Gio*
-    # will match Giovanni, Giovedi, Giorno...
-    # Giova*ni
-    # will match Giovanni, Giovani, Giovannini
-    def searchSimpleRegexp(termWithStar)
+
+    def reindex(fileList)
+      #puts "Reindexing... #{fileList.length} files..."
+      fileList.each do |current_file |
+        self.remove([current_file])        
+        self.load(current_file,noReload=false)
+      end
     end
+
+    # Remove all the keys
+    def removeAll()
+      self.remove(nil)
+    end
+
+    # Remove the files from the index, updating trigrams
+    def remove(filePaths=nil)
+      if filePaths==nil
+        fileList=[]
+        storedFiles=@redis.keys "fscan:id:*"
+        storedFiles.each do |fileKey|
+          filename=fileKey.split("fscan:id:")[1]
+          fileList.push(filename)
+        end
+      else
+        fileList=filePaths
+      end
+      # puts "Files to remove from index...#{fileList.length}"      
+      fileList.each do |filename|
+        fid=@redis.get "fscan:id:#{filename}"
+        trigramsToExpurge=@redis.smembers "fscan:trigramsOnFile:#{fid}"
+        if trigramsToExpurge.length==0
+          puts "?Nothing to do on #{filename}"
+        end
+        puts "#{filename} id=#{fid} Trigrams: #{trigramsToExpurge.length} Expurging..."
+        trigramsToExpurge.each do | ts |
+          @redis.srem "trigram:#{ts}", fid
+          begin
+            @redis.srem "trigram:ci:#{ts.downcase}",fid
+          rescue ArgumentError
+            # Ignore  "ArgumentError: invalid byte sequence in UTF-8"
+            # and proceed...
+          end
+        end
+        
+        @redis.del "fscan:id:#{filename}", "fscan:trigramsOnFile:#{fid}", "fscan:id2filename:#{fid}"
+        @redis.srem "fscan:processedFiles",  filename
+      end
+      return nil
+    end
+
   end
 end

data/readme.org

@@ -1,10 +1,34 @@
+* Code Zauker: your code, indexed
+Code Zauker is a search engine for programming languages.
+Code Zauker is based from ideas taken by old Google Code Search and uses Redis as a basic platform
+
+For news and discussion: http://gioorgi.com/tag/code-zauker/
+
+
 * INSTALL
-To install Code Zauker, you must simply build and install the gem as usual
+To install Code Zauker,simply issue
+#+BEGIN_SRC sh
+  gem install code_zauker
+#+END_SRC
+You need also [[http://redis.io/][redis-2.4.6]] or better. 
+For a sample redis configuration see the etc/ dreictory of the project
 
-* DEVELOPING
-For developing with Code Zauker you need bundler 1.0.21 or above
+
+* Release Notes
+** 0.0.1
+   First "we are here" release.
+** 0.0.2
+   Code Cleanup, reindexing features, and sample redis server config
+   czsearch now do not relay on unix grep, so it is easier to use with jruby
+   czsearch/czindexer supports options
 
 * Release History
   | Version | Date        | Summary                                           |
   |   0.0.1 | 26 Jan 2012 | First RubyGems Release (for testing purpose only) |
+  |   0.0.2 | 29 Jan 2012 | Removed dependency on unix find for czindexer.    |
   |         |             |                                                   |
+
+* DEVELOPING
+For developing with Code Zauker you need bundler 1.0.21 or above
+See devel.org file
+

data/test/test_search.rb

@@ -13,7 +13,7 @@ class FileScannerBasicSearch < Test::Unit::TestCase
     fs=CodeZauker::FileScanner.new()
     time = Benchmark.bm(7) do |x|
       x.report ("kurukku.txt") { fs.load("./test/fixture/kurukku.txt") } 
-      x.report ("BigFile") { fs.load("./test/fixture/TEST_LICENSE.txt")}
+      #x.report ("BigFile") { fs.load("./test/fixture/TEST_LICENSE.txt")}
       x.report("Search common words"){ fs.search("and"); fs.search("terms")  }
     end
     puts "Bench Result..."
@@ -47,11 +47,13 @@ class FileScannerBasicSearch < Test::Unit::TestCase
     assert(files.include?("./test/fixture/kurukku.txt") ==true)   
   end
 
-  def test_less_then3_must_not_give_Redis_error
+  def test_less_then3_must_give_error
     fs=CodeZauker::FileScanner.new()
     fs.load("./test/fixture/kurukku.txt")    
-    files=fs.search("di")
-    assert_equal 0, files.length
+    assert_raise RuntimeError do
+      files=fs.search("di")
+    end
+    #assert_equal 0, files.length
   end
 
   def test_small4
@@ -69,6 +71,29 @@ class FileScannerBasicSearch < Test::Unit::TestCase
     assert files.include?("./test/fixture/TEST_LICENSE.txt")==true
   end
 
+  def test_remove
+    fs=CodeZauker::FileScanner.new()
+    fs.load("./test/fixture/kurukku.txt", noReload=true)  
+    fs.remove(["./test/fixture/kurukku.txt"])
+    files=fs.search("\"Be hungry, be foolish\"")    
+    assert files.length ==0, 
+    "Expected zero search results after removal from index. Found instead:#{files}"
+    #assert(files[0].include?("test/fixture/kurukku.txt")==true)
+  end
+
+  def test_removeAll
+    require 'redis/connection/hiredis'
+    require 'redis'
+    redis=Redis.new
+    fs=CodeZauker::FileScanner.new(redis)
+    fs.load("./test/fixture/kurukku.txt", noReload=true) 
+    fs.removeAll()
+    foundKeys=redis.keys "*"
+    #puts "Keys at empty db:#{foundKeys}"
+    assert foundKeys.length==1, "Expected only one key at empty db. Found instead #{foundKeys}"
+    assert foundKeys[0]=="fscan:nextId", "Expected only the fscan:nextId key at empty db. Found instead #{foundKeys}"
+  end
+
 
 end
  

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: code_zauker
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-26 00:00:00.000000000 Z
+date: 2012-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &72883350 !ruby/object:Gem::Requirement
+  requirement: &78315190 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '0.7'
   type: :development
   prerelease: false
-  version_requirements: *72883350
+  version_requirements: *78315190
 - !ruby/object:Gem::Dependency
   name: hiredis
-  requirement: &72883030 !ruby/object:Gem::Requirement
+  requirement: &78314880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '0.3'
   type: :runtime
   prerelease: false
-  version_requirements: *72883030
+  version_requirements: *78314880
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &72882680 !ruby/object:Gem::Requirement
+  requirement: &78314600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: '2.2'
   type: :runtime
   prerelease: false
-  version_requirements: *72882680
+  version_requirements: *78314600
 description: Code Zauker is based from ideas taken by old Google Code Search and uses
   Redis as a basic platform
 email:
@@ -51,6 +51,7 @@ email:
 executables:
 - czindexer
 - czsearch
+- startRedis
 extensions: []
 extra_rdoc_files: []
 files:
@@ -59,8 +60,28 @@ files:
 - Rakefile
 - bin/czindexer
 - bin/czsearch
+- bin/startRedis
 - code_zauker.gemspec
+- devel.org
+- doc/CodeZauker.html
+- doc/CodeZauker/FileScanner.html
+- doc/_index.html
+- doc/class_list.html
+- doc/css/common.css
+- doc/css/full_list.css
+- doc/css/style.css
+- doc/file_list.html
+- doc/frames.html
+- doc/index.html
+- doc/js/app.js
+- doc/js/full_list.js
+- doc/js/jquery.js
+- doc/method_list.html
+- doc/top-level-namespace.html
+- etc/redis.conf
 - lib/code_zauker.rb
+- lib/code_zauker/constants.rb
+- lib/code_zauker/grep.rb
 - lib/code_zauker/version.rb
 - readme.org
 - test/fixture/TEST_LICENSE.txt