zeiger 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7ca514e457ea27de32736575f07c6aabcd63e097
-  data.tar.gz: d01654cdb2af19a41953a1726ddf114fafc3db9e
+  metadata.gz: 23b194662e74f40eada24a6f9ce7e25efcaa34ac
+  data.tar.gz: 5bb3af605cfc175cd66423aed69186124cd54f6e
 SHA512:
-  metadata.gz: 0d61f59cd5a5b1d56dee4555bddc4c3b7573c37ebb6f52aa76a09ce6f80bc7f39b2a04db0d9ac463c6cbc439db8b312ac172ae09d4c5c696853fafeb72690c04
-  data.tar.gz: 5fa8f50cbefcff8298cc2896ff4053ab81d4bc360eb5f2c3dcd94196b8d97e844a6075630fa1b34d5b589ccd1290688ad25dc98c3417c61d73c21a9bc037904b
+  metadata.gz: 510a9d31b51b965f131411e08b682f817dd9f2571ba1dba99020696ed360825ba651933d6619d85fedb72fe393f321a496c3b88a4dc868b1a090db82e913da72
+  data.tar.gz: 3a5f35825f6c516d7609623829fdaa248f989a446d739f37ef93a2ca6bdaa42022bf67d8e3868b9a3cb9405b9d8e4b5bd0dca41a3fdae5764083e436ec4a85d2

data/.zeiger.yml

@@ -1,9 +1,30 @@
 search:
+  - "app/**/*"
   - "bin/**/*"
   - "lib/**/*"
 ignore:
-  - .gz$
-  - .png$
-  - .gif$
-  - .zip$
-  - .jpg$
+  - "\\.gz$"
+  - "\\.png$"
+  - "\\.gif$"
+  - "\\.zip$"
+  - "\\.jpg$"
+stats:
+  code:
+    - "bin/.*"
+    - "lib/.*\\.rb$"
+  test:
+    - "spec/.*"
+    - "test/.*"
+  other:
+    - ".*\\.yml$"
+comments:
+  "\\.yml$":
+    - "^\\s*#"
+  "\\.rb$":
+    - "^\\s*#"
+  "\\.nydp$":
+    - "^\\s*;"
+  "\\.arc$":
+    - "^\\s*;"
+  "\\.c$":
+    - "^\\s*//"

data/README.md

@@ -29,13 +29,13 @@ Zeiger will rescan files in the current directory every ten seconds so the index
 
 ## Usage
 
-`zeiger server` runs the server and opens a unix filesystem socket called `zeiger-index` in the current directory.
+`zeiger server` runs the server and opens a unix filesystem socket called `/tmp/zeiger-index`
 
 `zeiger search "foo"` writes the query to the socket and displays the result
 
 `zeiger files "xed"` asks for the list of filenames corresponding to the argument ("xed"). With no argument, return all filenames. Files are sorted by length of filename. This sounds odd, but works nicely with 'completing-read in emacs: you will find the file you want in fewer keystrokes.
 
-By default, Zeiger searches only in these subdirectories : %w{ app bin config lib spec test }, and excludes filenames matching these patterns: %w{ .gz$ .png$ .jpg$ .pdf$ }.
+By default, Zeiger searches only in these subdirectories : `%w{ app bin config lib spec test }`, and excludes filenames matching these patterns: `%w{ .gz$ .png$ .jpg$ .pdf$ }`.
 
 To override, create a file called `.zeiger.yml` in your project root with the following format:
 

data/bin/zeiger

@@ -5,11 +5,13 @@ require 'zeiger'
 
 pwd     = `pwd`.strip
 command = $*[0]
+query   = $*[1]
 
 case command
 when "server" ; Zeiger::Server.new.run *$*
-when "search" ; Zeiger::QueryClient.run pwd, *$*
-when "files"  ; Zeiger::FileListClient.run pwd, *$*
+when "search" ; Zeiger::Client.new.send({ command: :search, pwd: pwd, search: query })
+when "files"  ; Zeiger::Client.new.send({ command: :files , pwd: pwd, files: query  })
+when "stats"  ; Zeiger::Client.new.send({ command: :stats , pwd: pwd                })
 end
 
 # define file groups

data/lib/zeiger/{query_client.rb → client.rb}

@@ -1,8 +1,8 @@
 module Zeiger
-  class QueryClient
-    def self.run pwd, command, q, *args
+  class Client
+    def send data
       Socket.unix(SOCKET_NAME) { |sock|
-        s = YAML.dump({ pwd: pwd, search: q })
+        s = YAML.dump(data)
         sock.write([s.bytesize].pack("I"))
         sock.write(s)
 

data/lib/zeiger/file_info.rb

@@ -2,14 +2,17 @@ require 'set'
 
 module Zeiger
   class FileInfo
-    attr_accessor :dir, :filename, :ngrams
+    attr_accessor :index, :filename, :ngrams, :lines, :stats_group, :comment_regexes, :nc_nb_line_count
 
-    def initialize dir, filename
-      @dir, @filename, @ngrams = dir, filename, Set.new
+    def initialize index, filename, stats
+      @index, @filename, @ngrams, @lines = index, filename, Set.new, []
+      @stats_group      = stats.stats_group filename
+      @comment_regexes  = stats.comment_rules(filename)
+      @nc_nb_line_count = 0
     end
 
     def local_filename
-      @_local_filename ||= filename.gsub(/^#{Regexp.escape dir}\//, "")
+      @_local_filename ||= filename.gsub(/^#{Regexp.escape index.dir}\//, "")
     end
 
     def match regex
@@ -19,5 +22,14 @@ module Zeiger
     def add_ngram ngram
       @ngrams << ngram
     end
+
+    def add_line line
+      lines << line
+      @nc_nb_line_count += 1 unless line.blank? || (comment_regexes.any? { |regex| line.matches? regex })
+    end
+
+    def summary
+      "[#{index.name}] #{stats_group}\t#{nc_nb_line_count}/#{lines.count}\t: #{local_filename}"
+    end
   end
 end

data/lib/zeiger/index.rb

@@ -5,17 +5,30 @@ module Zeiger
     ROOT_FILES = %w{ .zeiger.yml .git .hg Makefile Rakefile Gemfile build.xml }
     NGRAM_SIZE = 3
 
-    attr_accessor :index, :dir, :includes, :ignore, :files, :monitor
+    attr_accessor :index, :dir, :name, :includes, :ignore, :files, :config, :monitor, :stats
 
     def initialize dir
       attrs = File.exist?(".zeiger.yml") ? YAML.load(File.read ".zeiger.yml") : { }
       self.dir      = File.expand_path dir
+      self.name     = File.basename self.dir
+      self.config   = load_config
       self.index    = Hash.new { |h, k| h[k] = [] }
       self.files    = Hash.new
-      self.monitor  = Monitor.new dir, self
+      self.monitor  = Monitor.new dir, self, config
+      self.stats    = Stats.new self, config
       rescan
     end
 
+    def load_config
+      conf_file = File.join dir, ".zeiger.yml"
+      if File.exist?(conf_file)
+        puts "reading config from #{conf_file.inspect}"
+        YAML.load(File.read conf_file)
+      else
+        { }
+      end
+    end
+
     def self.from_path path
       raise "no path! #{path.inspect}" if path == nil || path.strip == ''
       return INDICES[path] if INDICES[path]
@@ -35,20 +48,23 @@ module Zeiger
     end
 
     def add_to_index file
-      info = files[file] = FileInfo.new(dir, file)
+      info = files[file] = FileInfo.new(self, file, stats)
 
       File.read(file).split(/\n/).each_with_index { |txt, line|
-        Line.new(info, line + 1, txt).ngrams(NGRAM_SIZE) do |trig, line|
+        line = Line.new(info, line + 1, txt).ngrams(NGRAM_SIZE) do |trig, line|
           index[trig] << line
           info.add_ngram trig
         end
       }
+
+      puts "#{info.summary} : re-index"
     end
 
-    def rescan                   ; monitor.build_index                                            ; end
-    def get_ngram_lines   ngrams ; ngrams.map { |ngram| result = index[ngram] || [] }.reduce(&:&) ; end
-    def exec_query regex, ngrams ; get_ngram_lines(ngrams).select { |line| line.matches? regex }  ; end
-    def sort_by_filename   lines ; lines.sort_by { |line| line.file.local_filename }              ; end
+    def glob             pattern ; Dir.glob(File.join(dir, pattern))                                     ; end
+    def rescan                   ; monitor.build_index                                                   ; end
+    def get_ngram_lines   ngrams ; ngrams.map { |ngram| result = index[ngram] || [] }.reduce(&:&)        ; end
+    def exec_query regex, ngrams ; get_ngram_lines(ngrams).select { |line| line.matches? regex }         ; end
+    def sort_by_filename   lines ; lines.sort_by { |line| [line.file.local_filename, line.line_number] } ; end
 
     def query txt
       puts "got query #{txt.inspect}"

data/lib/zeiger/line.rb

@@ -6,12 +6,14 @@ module Zeiger
     def initialize file, line_number, content
       @file, @line_number, @content = file, line_number, content
       @hash = "#{file.filename}##{line_number}".hash
+      file.add_line self
     end
 
     def to_s            ; "#{file.local_filename}:#{line_number}:#{content}"               ; end
-    def matches? regex  ; content.match regex                                              ; end
+    def blank?          ; content.strip == ''                                              ; end
+    def matches?  regex ; content.match regex                                              ; end
     def ngram_list size ; @ngrams ||= content.ngrams(size)                                 ; end
-    def ngrams    size  ; ngram_list(size).each { |ngram| yield ngram, self }              ; end
-    def ==       other  ; self.file == other.file && self.line_number == other.line_number ; end
+    def ngrams     size ; ngram_list(size).each { |ngram| yield ngram, self }              ; end
+    def ==        other ; self.file == other.file && self.line_number == other.line_number ; end
   end
 end

data/lib/zeiger/monitor.rb

@@ -2,23 +2,12 @@ module Zeiger
   class Monitor
     attr_accessor :dir, :index, :stat, :includes, :ignore
 
-    def initialize dir, index
+    def initialize dir, index, attrs
       @dir, @index, @stat = dir, index, Hash.new
-      attrs = load_config
       @includes = attrs["search"] || %w{ app/**/* bin/**/* config/**/* lib/**/* spec/**/* test/**/* }
       @ignore   = attrs["ignore"] || %w{ .gz$ .png$ .jpg$ .pdf$ }
     end
 
-    def load_config
-      conf_file = File.join dir, ".zeiger.yml"
-      if File.exist?(conf_file)
-        puts "reading config from #{conf_file.inspect}"
-        YAML.load(File.read conf_file)
-      else
-        { }
-      end
-    end
-
     def ignore? filename
       ignore.any? { |ig| filename.match ig }
     end
@@ -31,12 +20,11 @@ module Zeiger
       started = Time.now
       files = Set.new
       includes.each do |inc|
-        Dir.glob(File.join(dir, inc)).sort.each do |file|
+        index.glob(inc).sort.each do |file|
           if File.file?(file) && !ignore?(file)
             files << file
             mtime = File.stat(file).mtime
             if !uptodate?(file, mtime)
-              puts "re-indexing #{file}"
               index.remove_from_index file
               index.add_to_index file
               stat[file] = mtime
@@ -44,7 +32,13 @@ module Zeiger
           end
         end
       end
-      (Set.new(stat.keys) - files).each { |f| index.remove_from_index f }
+      (Set.new(stat.keys) - files).each { |f|
+        info = index.files[f]
+        index.remove_from_index f
+        stat.delete f
+        help = info ? info.summary : "[#{index.name}] #{f}"
+        puts "#{help} : deleted"
+      }
       finished = Time.now
       puts "ngrams       : #{index.index.length}"
       puts "files        : #{index.files.length}"

data/lib/zeiger/server.rb

@@ -10,7 +10,7 @@ module Zeiger
           while true do
             indices = Zeiger::INDICES.values
             indices.each do |index|
-              puts "scanning #{index.dir}"
+              puts "scanning #{index.name} at #{index.dir}"
               index.rescan
             end
             sleep 10
@@ -22,7 +22,6 @@ module Zeiger
       end
 
       Socket.unix_server_loop(SOCKET_NAME) { |sock, client|
-        puts "query thread: server loop"
         begin
           length   = sock.read(4).unpack("I")[0]
           query    = sock.read(length)
@@ -30,11 +29,14 @@ module Zeiger
           puts incoming.to_yaml
 
           index = Index.from_path incoming[:pwd]
-          puts "querying index at #{index.dir}"
+          puts "querying index #{index.name}"
 
-          if incoming[:search]
+          case incoming[:command]
+          when :search
             index.query(incoming[:search]).each { |res| sock.puts res.to_s }
-          elsif incoming.key? :files
+          when :stats
+            sock.puts index.stats.stats.to_yaml
+          when :files
             index.file_list(incoming[:files]).each { |f| sock.puts f.local_filename }
           end
 

data/lib/zeiger/stats.rb

@@ -0,0 +1,33 @@
+module Zeiger
+  class Stats
+    attr_accessor :index, :group_cfg, :comment_cfg
+
+    def initialize index, config
+      @index = index
+      @group_cfg = config["stats"]
+      @comment_cfg = compile_comment_cfg config["comments"]
+    end
+
+    def compile_comment_cfg cfg
+      (cfg || { }).each_with_object({ }) do |(file_pattern, regex_list), result|
+        result[Regexp.compile(file_pattern)] = regex_list.map { |r| Regexp.compile r }
+      end
+    end
+
+    def stats_group filename
+      (group_cfg || []).each { |group, regexes| return group if regexes.any? { |r| filename.match r } }
+      "undefined"
+    end
+
+    def comment_rules filename
+      file_type = comment_cfg.keys.detect { |rxp| filename.match rxp }
+      comment_cfg[file_type] || []
+    end
+
+    def stats
+      index.files.values.each_with_object(Hash.new { |h,k| h[k] = 0 }) do |file, results|
+        results[file.stats_group] += file.nc_nb_line_count
+      end
+    end
+  end
+end

data/lib/zeiger/version.rb

@@ -1,3 +1,3 @@
 module Zeiger
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/zeiger.rb

@@ -1,25 +1,22 @@
 require "set"
 require 'yaml'
-require 'zeiger/query_client'
-require 'zeiger/file_list_client'
+require 'zeiger/client'
 require 'zeiger/server'
 require "zeiger/version"
 require 'zeiger/monitor'
+require 'zeiger/stats'
 require 'zeiger/file_info'
 require 'zeiger/index'
 require 'zeiger/line'
 
 class String
   def ngrams size
-    padding = " " * (size - 1)
-    txt = self
-    ngrams = Hash.new { |h,k| h[k] = 0 }
-    regex = Regexp.compile("." * size)
+    s = size - 1
 
     result = []
 
-    (txt.length - 2).times do |i|
-      result << txt[i..(i + 2)]
+    (length - s).times do |i|
+      result << self[i..(i + s)]
     end
 
     result

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zeiger
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - conanite
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-23 00:00:00.000000000 Z
+date: 2017-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -55,13 +55,13 @@ files:
 - Rakefile
 - bin/zeiger
 - lib/zeiger.rb
+- lib/zeiger/client.rb
 - lib/zeiger/file_info.rb
-- lib/zeiger/file_list_client.rb
 - lib/zeiger/index.rb
 - lib/zeiger/line.rb
 - lib/zeiger/monitor.rb
-- lib/zeiger/query_client.rb
 - lib/zeiger/server.rb
+- lib/zeiger/stats.rb
 - lib/zeiger/version.rb
 - zeiger.gemspec
 homepage: https://github.com/conanite/zeiger

data/lib/zeiger/file_list_client.rb

@@ -1,15 +0,0 @@
-module Zeiger
-  class FileListClient
-    def self.run pwd, command, q=nil, *args
-      Socket.unix(SOCKET_NAME) { |sock|
-        s = YAML.dump({ pwd: pwd, files: q })
-        sock.write([s.bytesize].pack("I"))
-        sock.write(s)
-
-        while !sock.eof?
-          puts sock.readline
-        end
-      }
-    end
-  end
-end