code_zauker 0.0.6 → 0.0.7

data/bin/mczindexer

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -x -e
+echo "Parallel Indexer"
+time find $* -type f -print0 | xargs -0 -P 10  -n 25  czindexer  -v

data/bin/report.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+require 'set'
+require 'code_zauker'
+require 'code_zauker/grep'
+require 'code_zauker/cli'
+require 'redis/connection/hiredis'
+require 'redis'
+require 'tempfile'
+require 'pdf/reader'
+
+puts "Redis Code Zauker Report"
+require 'optparse'
+options={}
+optparse= OptionParser.new do  |opts|
+  opts.banner="Usage: report [options] "
+
+  options[:redis_host]="127.0.0.1"
+  options[:redis_port]=6379
+  options[:redis_password]=nil
+
+  opts.on('-h','--redis-server pass@SERVER:port', String,
+          'Specify the alternate redis server to use')do |server|
+    myoptions=CodeZauker::CliUtil.new().parse_host_options(server)
+    options[:redis_host]=myoptions[:redis_host]
+    options[:redis_port]=myoptions[:redis_port]
+    options[:redis_password]=myoptions[:redis_password]
+    
+    if options[:redis_password]
+      puts "Server: #{options[:redis_host]} Port:#{options[:redis_port]} WithPassword"
+    else
+      puts "Server: #{options[:redis_host]} Port:#{options[:redis_port]}"
+    end
+  end
+end
+optparse.parse!
+
+util=CodeZauker::CliUtil.new()
+redisConnection=Redis.new(:host => options[:redis_host], :port => options[:redis_port], :password=> options[:redis_password])
+util.do_report(redisConnection)
+

data/code_zauker.gemspec

@@ -27,6 +27,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "redis", "~> 2.2"
   s.add_runtime_dependency "pdf-reader", "~> 1.0.0"
   s.add_runtime_dependency "sinatra", "~> 1.3"
+  s.add_runtime_dependency "redis_logger", "~> 0.1"
+  # Eval http://github.com/masonoise/redis_logger-web  
+
   # thin unable to be installed for missing g++ on my dev platform (shame on debian)
   #s.add_runtime_dependency "thin"
   ## Install and require the hiredis gem before redis-rb for maximum performances.

data/lib/code_zauker/cli.rb

@@ -36,5 +36,47 @@ module CodeZauker
       end
       return options
     end
+
+    def do_report(redis)
+      puts "Simple Reporting....on #{redis.client.host}"
+      id2filename=redis.keys "fscan:id2filename:*"
+      puts "File Stored:\t\t#{id2filename.length}"
+      processedFiles=redis.scard("fscan:processedFiles")
+      puts "Processed Files:\t#{processedFiles}"
+      # Complex... compute average "fscan:trigramsOnFile:#{fid}"
+      sum=0.0
+      max=0
+      min=90000000000000000
+      fileName2Ids=redis.keys "fscan:id:*"
+      count=fileName2Ids.length+0.0
+      puts "Finding ids..."   
+      ids=redis.mget(*(redis.keys("fscan:id:*")))
+      puts "Scanning ids..."      
+      ids.each do | fid |
+        # Forma fscan:trigramsOnFile:5503
+        trigramsOnFile=redis.scard("fscan:trigramsOnFile:#{fid}")
+        sum = sum + trigramsOnFile
+        # if trigramsOnFile == 0 or trigramsOnFile >=max
+        #   fname=redis.get("fscan:id2filename:#{fid}")
+        #   puts "Note fscan:trigramsOnFile:#{fid} -> #{trigramsOnFile} #{fname}"
+        # end
+        max=trigramsOnFile if trigramsOnFile >max
+        min=trigramsOnFile if trigramsOnFile <min and trigramsOnFile>0        
+      end
+      av=sum/count
+      puts "Average Trigrams per file:#{av} Min: #{min} Max: #{max}"
+      tagCharSize=max/80
+      #tagCharSize=max/10 if tagCharSize>80
+      puts "Graphic summary... +=#{tagCharSize}"
+      ids.each do | fid |
+        trigramsOnFile=redis.scard("fscan:trigramsOnFile:#{fid}")
+        if trigramsOnFile>= (tagCharSize*3)
+          fname=redis.get("fscan:id2filename:#{fid}")
+          bar="+"*(trigramsOnFile/tagCharSize)
+          puts "#{bar} #{fname}"
+        end
+      end
+
+    end
   end
 end

data/lib/code_zauker/constants.rb

@@ -10,9 +10,11 @@ module CodeZauker
   DEFAULT_EXCLUDED_EXTENSION=[
                               # Documents                             
                               ".xps",
-                              ".zip",".7z",
+                              ".zip",".7z","rar",
                               # MS Office zip-like files...
                               ".pptx",".docx",".xlsx",
+                              # Avoid slurping text document too...
+                              ".doc",
                               ".ppt",".xls",".rtf",".vsd", ".odf",
                               # Binary bad stuff
                               ".dll",".exe",".out",".elf",".lib",".so",
@@ -25,7 +27,9 @@ module CodeZauker
                               ".tar",
                               ".gz",".Z",
                               ".dropbox",
-                              ".svn-base",".pdb",".cache",                             
+                              ".svn-base",".pdb",".cache", 
+                              #IDE STUFF
+                              ".wlwLock",
                               # Music exclusion
                               ".mp3",".mp4",".wav",
                               # Image exclusion

data/lib/code_zauker/version.rb

@@ -1,3 +1,3 @@
 module CodeZauker
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/lib/code_zauker/webgui.rb

@@ -24,7 +24,8 @@ get '/search' do
       pattern=/#{Regexp.escape(askedQuery)}/i
       lines=grep(f,pattern, pre_context=2, post_context=2);
       desc=""
-      lines.each do |l |
+      lines.each do | l |
+        # http://stackoverflow.com/questions/1287630/ruby-gsub-and-regex
         hilighted=l.gsub(/(#{Regexp.escape(askedQuery)})/i){ "<b>#{$1}</b>"}
         desc=desc+ "#{f}:#{hilighted}\n"
       end

data/readme.org

@@ -14,13 +14,60 @@ You need also [[http://redis.io/][redis-2.4.6]] or better.
 For a sample redis configuration see the etc/ directory of the project
 
 * Try it out
-Take a look to czindex and czsearch commands help for usage patterns.
+** To index all your files
+The following command will index all the code file below /full_path_of_sources/ providing 
+a feedback on standard output
+#+BEGIN_SRC sh
+ czindexer -v /full_path_of_sources/
+#+END_SRC
+If your redis server is on another machine, you can use the following command
+#+BEGIN_SRC sh
+ czindexer --redis-server myredisserver  -v /full_path_of_sources/
+#+END_SRC
+Also authentication over redis is supported:
+#+BEGIN_SRC sh
+ czindexer --redis-server password@rediserverpwp  -v /full_path_of_sources/
+#+END_SRC
+
+
+To search for the 'my_beautiful_method' try out:
+#+BEGIN_SRC sh
+ czsearch  my_beautiful_method
+#+END_SRC
 
+Looking case insensitive can be userful too....
+#+BEGIN_SRC sh
+ czsearch -i  'Select USER_id'
+#+END_SRC
+
+Tip:Take a look to czindex and czsearch commands help for full options list
+** Web interface
+There is a new shiny web interface based on sinatra. To try it out, fire
+#+BEGIN_SRC sh
+ ./bin/webgui
+#+END_SRC
+It is still beta, but it is *very* fast, thank to redis!
+
+
+** Parallel execution
+If you want to speed up indexing, you can use the mczindexer command.
+For instance:
+mczindexer eclipse-sources/
+will fire at most 10 parallel czindexer.
+
+** Simple stats
+You can ask code zauker to print a nice stats on standard output.
+Run
+#+BEGIN_SRC sh
+ ./bin/report.rb
+#+END_SRC
+and enjoy!
 
 
 * Release History
   | Version | Date        | Summary                                                         |
   |---------+-------------+-----------------------------------------------------------------|
+  |   0.0.7 | 13 May 2012 | Better documentation, mczindexer, new report command
   |   0.0.6 | 04 May 2012 | New redis-server option. Better web search with results hilight |
   |   0.0.5 | 09 Apr 2012 | Added Sinatra-based web search page, featuring bootrstrap css   |
   |   0.0.4 | 12 Feb 2012 | PDF Searching                                                   |

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: code_zauker
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-04 00:00:00.000000000 Z
+date: 2012-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &74198570 !ruby/object:Gem::Requirement
+  requirement: &83355880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '0.7'
   type: :development
   prerelease: false
-  version_requirements: *74198570
+  version_requirements: *83355880
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &74197690 !ruby/object:Gem::Requirement
+  requirement: &83355530 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '0.9'
   type: :development
   prerelease: false
-  version_requirements: *74197690
+  version_requirements: *83355530
 - !ruby/object:Gem::Dependency
   name: hiredis
-  requirement: &74196880 !ruby/object:Gem::Requirement
+  requirement: &83355130 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '0.3'
   type: :runtime
   prerelease: false
-  version_requirements: *74196880
+  version_requirements: *83355130
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &74196090 !ruby/object:Gem::Requirement
+  requirement: &83354730 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '2.2'
   type: :runtime
   prerelease: false
-  version_requirements: *74196090
+  version_requirements: *83354730
 - !ruby/object:Gem::Dependency
   name: pdf-reader
-  requirement: &74187710 !ruby/object:Gem::Requirement
+  requirement: &83354000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *74187710
+  version_requirements: *83354000
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &74186780 !ruby/object:Gem::Requirement
+  requirement: &83130590 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,18 @@ dependencies:
         version: '1.3'
   type: :runtime
   prerelease: false
-  version_requirements: *74186780
+  version_requirements: *83130590
+- !ruby/object:Gem::Dependency
+  name: redis_logger
+  requirement: &83130330 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: *83130330
 description: Code Zauker is based from ideas taken by old Google Code Search and uses
   Redis as a basic platform
 email:
@@ -84,6 +95,8 @@ email:
 executables:
 - czindexer
 - czsearch
+- mczindexer
+- report.rb
 - startRedis
 - webgui
 extensions: []
@@ -96,6 +109,8 @@ files:
 - Rakefile
 - bin/czindexer
 - bin/czsearch
+- bin/mczindexer
+- bin/report.rb
 - bin/startRedis
 - bin/webgui
 - code_zauker.gemspec