code_zauker 0.0.5 → 0.0.6

data/.gitignore

@@ -5,3 +5,4 @@ pkg/*
 code_zauker_index.rdb
 .yardoc
 publishMe
+bin/aws*

data/bin/czindexer

@@ -2,11 +2,18 @@
 # Suggested execution is mixing find / xargs with the parallel (P) parameters:
 # find test/fixture/ -type f | xargs -P 5  -n 10 ./bin/czindexer
 # will fire 5 czindexer each with 10 files to process...
+require 'code_zauker/cli'
+require 'redis/connection/hiredis'
+require 'redis'
 require 'optparse'
 options={}
 optparse= OptionParser.new do  |opts|
   opts.banner="Usage: czindexer [options] [file1] [file2]..."
   options[:verbose] = false
+  options[:redis_host]="127.0.0.1"
+  options[:redis_port]=6379
+  options[:redis_password]=nil
+
 
   opts.on( '-v', '--verbose', 'Output more information' ) do
     options[:verbose] = true
@@ -17,6 +24,20 @@ optparse= OptionParser.new do  |opts|
     options[:reindex]=true
   end
 
+  opts.on('-h','--redis-server pass@SERVER:port', String,
+          'Specify the alternate redis server to use')do |server|
+    myoptions=CodeZauker::CliUtil.new().parse_host_options(server)
+    options[:redis_host]=myoptions[:redis_host]
+    options[:redis_port]=myoptions[:redis_port]
+    options[:redis_password]=myoptions[:redis_password]
+    
+    if options[:redis_password]
+      puts "Server: #{options[:redis_host]} Port:#{options[:redis_port]} WithPassword"
+    else
+      puts "Server: #{options[:redis_host]} Port:#{options[:redis_port]}"
+    end
+  end
+
   opts.on( '-h', '--help', 'Display this screen' ) do
     puts opts
     exit
@@ -82,7 +103,8 @@ end
 begin  
   $CUMULATED_TIME=0
   # Allocated here to recycle connection
-  fs=CodeZauker::FileScanner.new()
+  redisConnection=Redis.new(:host => options[:redis_host], :port => options[:redis_port], :password=> options[:redis_password])
+  fs=CodeZauker::FileScanner.new(redisConnection)
   $PROCESSED_FILES=0
   puts "Code Zauker v#{CodeZauker::VERSION}" if  options[:verbose]
   puts "Reindexing..." if options[:verbose]==true and options[:reindex]==true

data/bin/czsearch

@@ -4,6 +4,9 @@
 $VERBOSE=nil 
 require 'code_zauker'
 require 'code_zauker/grep'
+require 'code_zauker/cli'
+require 'redis/connection/hiredis'
+require 'redis'
 require 'tempfile'
 require 'pdf/reader'
 include Grep
@@ -17,6 +20,9 @@ optparse= OptionParser.new do  |opts|
   options[:postcontext]=0
   options[:extensions_to_ignore]=[]
   options[:file_to_exclude]=[]
+  options[:redis_host]="127.0.0.1"
+  options[:redis_port]=6379
+  options[:redis_password]=nil
 
   opts.on('-i', '--ignore-case','ignore case distinctions') do 
       options[:ignorecase]=true    
@@ -44,9 +50,18 @@ optparse= OptionParser.new do  |opts|
   end
 
 
-  opts.on('-h','--redis-server SERVER', String,
+  opts.on('-h','--redis-server pass@SERVER:port', String,
           'Specify the alternate redis server to use')do |server|
-    raise 'Still to be done'
+    myoptions=CodeZauker::CliUtil.new().parse_host_options(server)
+    options[:redis_host]=myoptions[:redis_host]
+    options[:redis_port]=myoptions[:redis_port]
+    options[:redis_password]=myoptions[:redis_password]
+    
+    if options[:redis_password]
+      puts "Server: #{options[:redis_host]} Port:#{options[:redis_port]} WithPassword"
+    else
+      puts "Server: #{options[:redis_host]} Port:#{options[:redis_port]}"
+    end
   end
 
 
@@ -69,7 +84,8 @@ optparse.parse!
 ARGV.each do | s |
   #puts "Code Zauker Searching for #{s}"  
   util=CodeZauker::Util.new()
-  fs=CodeZauker::FileScanner.new()
+  redisConnection=Redis.new(:host => options[:redis_host], :port => options[:redis_port], :password=> options[:redis_password])
+  fs=CodeZauker::FileScanner.new(redisConnection)
   if options[:ignorecase]==false
     files=fs.search(s)
     pattern=/#{Regexp.escape(s)}/

data/lib/code_zauker/cli.rb

@@ -0,0 +1,40 @@
+module CodeZauker
+
+  class CliUtil
+    
+    def parse_host_options(connection_string)
+      #puts "Parsing... #{connection_string}"
+      options={}
+      options[:redis_host]="127.0.0.1"
+      options[:redis_port]=6379
+      options[:redis_password]=nil
+      r=/(\w+)@([a-zA-Z0-9.]+):([0-9]+)?/
+      rNoPass=/([a-zA-Z0-9.]+):([0-9]+)?/
+      rHostAndPass=/(\w+)@([a-zA-Z0-9.]+)/
+      m=r.match(connection_string)    
+      if m                   
+        options[:redis_password]=m.captures[0]
+        options[:redis_host]=m.captures[1]      
+        options[:redis_port]=m.captures[2]
+        
+      else
+        m=rNoPass.match(connection_string)
+        if m        
+          options[:redis_host]=m.captures[0]      
+          options[:redis_port]=m.captures[1]
+        else
+          # Check the auth@host case right here
+          m2=rHostAndPass.match(connection_string)
+          if m2
+            options[:redis_password]=m2.captures[0]
+            options[:redis_host]=m2.captures[1]
+          else
+            #puts "SERVER ONLY"
+            options[:redis_host]=connection_string
+          end
+        end
+      end
+      return options
+    end
+  end
+end

data/lib/code_zauker/constants.rb

@@ -1,5 +1,7 @@
 module CodeZauker
-  MAX_PUSH_TRIGRAM_RETRIES=3
+  # Under Amazon AWS, a lot of timeout can happen.
+  # We put a higer retry here
+  MAX_PUSH_TRIGRAM_RETRIES=15
   # Stats 
   # It is difficult to decide what is the best trigram push size.
   # a larger one ensure a best in memory processing but can lead to longer transactions

data/lib/code_zauker/version.rb

@@ -1,3 +1,3 @@
 module CodeZauker
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/code_zauker/webgui.rb

@@ -3,6 +3,8 @@ require 'sinatra'
 require "code_zauker/version"
 require "code_zauker"
 require "erb"
+require 'code_zauker/grep'
+include Grep
 
 # See http://www.sinatrarb.com/intro
 get '/' do
@@ -14,7 +16,22 @@ get '/search' do
   # Process the search and show the results...
   fs=CodeZauker::FileScanner.new()
   files=fs.isearch(params[:q])
-  erb :show_results, :locals => {:files => files, :q => params[:q] }
+  util=CodeZauker::Util.new()
+  abstracts=[]
+  files.each do |f|
+    if util.is_pdf?(f)==false
+      askedQuery=params[:q]
+      pattern=/#{Regexp.escape(askedQuery)}/i
+      lines=grep(f,pattern, pre_context=2, post_context=2);
+      desc=""
+      lines.each do |l |
+        hilighted=l.gsub(/(#{Regexp.escape(askedQuery)})/i){ "<b>#{$1}</b>"}
+        desc=desc+ "#{f}:#{hilighted}\n"
+      end
+      abstracts.push(desc)
+    end
+  end
+  erb :show_results, :locals => {:files => abstracts, :q => params[:q] }
 end
 
 configure do

data/readme.org

@@ -1,6 +1,6 @@
 * Code Zauker: your code, indexed
 Code Zauker is a search engine for programming languages.
-Code Zauker is based from ideas taken by old Google Code Search and uses Redis as a basic platform
+Code Zauker is based from ideas taken by old Google Code Search and uses Redis as storage engine.
 
 For news and discussion: http://gioorgi.com/tag/code-zauker/
 
@@ -13,17 +13,33 @@ To install Code Zauker,simply issue
 You need also [[http://redis.io/][redis-2.4.6]] or better. 
 For a sample redis configuration see the etc/ directory of the project
 
+* Try it out
+Take a look to czindex and czsearch commands help for usage patterns.
+
+
 
 * Release History
-  | Version | Date        | Summary                                                       |
-  |---------+-------------+---------------------------------------------------------------|
-  |   0.0.5 | 09 Apr 2012 | Added Sinatra-based web search page, featuring bootrstrap css |
-  |   0.0.4 | 12 Feb 2012 | PDF Searching                                                 |
-  |   0.0.3 | 03 Feb 2012 | Added Case insensitive search.UTF-8 trigram database          |
-  |   0.0.2 | 29 Jan 2012 | Removed dependency on unix find for czindexer.                |
-  |   0.0.1 | 26 Jan 2012 | First RubyGems Release (for testing purpose only)             |
-  |         |             |                                                               |
-  |         |             |                                                               |
+  | Version | Date        | Summary                                                         |
+  |---------+-------------+-----------------------------------------------------------------|
+  |   0.0.6 | 04 May 2012 | New redis-server option. Better web search with results hilight |
+  |   0.0.5 | 09 Apr 2012 | Added Sinatra-based web search page, featuring bootrstrap css   |
+  |   0.0.4 | 12 Feb 2012 | PDF Searching                                                   |
+  |   0.0.3 | 03 Feb 2012 | Added Case insensitive search.UTF-8 trigram database            |
+  |   0.0.2 | 29 Jan 2012 | Removed dependency on unix find for czindexer.                  |
+  |   0.0.1 | 26 Jan 2012 | First RubyGems Release (for testing purpose only)               |
+  |         |             |                                                                 |
+  |         |             |                                                                 |
+
+* Aws tests
+** Micro instance
+Without multiplexing you get
+ real    4m39.599s
+for indexing code_zauker
+
+With
+ time find . -type f -print0 | xargs -0 -P 10  -n 20  ./bin/czindexer -v --redis-server awsserver
+You get about
+  real    0m31.284s
 
 
 * DEVELOPING

data/templates/show_results.erb

@@ -32,15 +32,20 @@
 
 	  </form>
 	</p>
+	<style>
+	  b{
+	   color: #A1A100;
+	  }
+	</style>
 	<span class="badge badge-success"> <%=files.length%> Results</span>
 	<ol>
 	  <%
 	     files.each do |f|
-	     %>
+	  %>
 	  <li><%=f%>
-	    <%   
+	  <%   
 	       end
-	       %>
+	  %>
 	</ol>
       </div>
     </div>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: code_zauker
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-09 00:00:00.000000000 Z
+date: 2012-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &73314420 !ruby/object:Gem::Requirement
+  requirement: &74198570 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '0.7'
   type: :development
   prerelease: false
-  version_requirements: *73314420
+  version_requirements: *74198570
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &73314170 !ruby/object:Gem::Requirement
+  requirement: &74197690 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '0.9'
   type: :development
   prerelease: false
-  version_requirements: *73314170
+  version_requirements: *74197690
 - !ruby/object:Gem::Dependency
   name: hiredis
-  requirement: &73313940 !ruby/object:Gem::Requirement
+  requirement: &74196880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '0.3'
   type: :runtime
   prerelease: false
-  version_requirements: *73313940
+  version_requirements: *74196880
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &73313690 !ruby/object:Gem::Requirement
+  requirement: &74196090 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '2.2'
   type: :runtime
   prerelease: false
-  version_requirements: *73313690
+  version_requirements: *74196090
 - !ruby/object:Gem::Dependency
   name: pdf-reader
-  requirement: &73313440 !ruby/object:Gem::Requirement
+  requirement: &74187710 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *73313440
+  version_requirements: *74187710
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &73313150 !ruby/object:Gem::Requirement
+  requirement: &74186780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: '1.3'
   type: :runtime
   prerelease: false
-  version_requirements: *73313150
+  version_requirements: *74186780
 description: Code Zauker is based from ideas taken by old Google Code Search and uses
   Redis as a basic platform
 email:
@@ -129,6 +129,7 @@ files:
 - htdocs/js/bootstrap.js
 - htdocs/js/bootstrap.min.js
 - lib/code_zauker.rb
+- lib/code_zauker/cli.rb
 - lib/code_zauker/constants.rb
 - lib/code_zauker/grep.rb
 - lib/code_zauker/version.rb
@@ -163,7 +164,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: code_zauker
-rubygems_version: 1.8.10
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: A search engine for programming languages