zeiger 0.0.1 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c01cd920ba6af86d17bd79b07e648d5e1bb37ad2
-  data.tar.gz: d90ce8ed808e1853aad0175d6fa862d0b3ff2a29
+  metadata.gz: 68b9dfc1fb8332bcba98e6add4af92cdde895c7d
+  data.tar.gz: 32df5d5004fc6fe5fe448c5b478d7b72d18feb1c
 SHA512:
-  metadata.gz: 5c24d14cba4ac6840aa2ec817c22a608bc98a476289643bdaa6b06fcd50649f82a189f45fdc5c14ef882a7cb815bdcd900c4f387d2d3baa2f4ffeb8edb90b223
-  data.tar.gz: 491a7131fc1f5b5ba29b207d31d2c5b232ed9307650a17aecd6d9383f101debefd6f7e5366c79a2e8e456e82f3d52b3fa6c799a7282b64b4ff5ab2bd683e9aff
+  metadata.gz: 6af416b2b517cf755fdd7304f7fc099b4a91aa115f95f011e54cbcb2f42bf9f3924ba3787e0a22e2873e54876133b72d9b45745866adf3044277fc1447bb6985
+  data.tar.gz: 6fd40c1ca39823a89c03a9eb36b3ee55463e9f0e2b3a67cbd19d7874171aca84e53c6d11b528cbc3bfc16c16653d04e3ec50f47a64c5c0771f37999b071700d8

data/.zeiger.yml

@@ -1,9 +1,30 @@
 search:
-  - bin
-  - lib
+  - "app/**/*"
+  - "bin/**/*"
+  - "lib/**/*"
 ignore:
-  - .gz$
-  - .png$
-  - .gif$
-  - .zip$
-  - .jpg$
+  - "\\.gz$"
+  - "\\.png$"
+  - "\\.gif$"
+  - "\\.zip$"
+  - "\\.jpg$"
+stats:
+  code:
+    - "bin/.*"
+    - "lib/.*\\.rb$"
+  test:
+    - "spec/.*"
+    - "test/.*"
+  other:
+    - ".*\\.yml$"
+comments:
+  "\\.yml$":
+    - "^\\s*#"
+  "\\.rb$":
+    - "^\\s*#"
+  "\\.nydp$":
+    - "^\\s*;"
+  "\\.arc$":
+    - "^\\s*;"
+  "\\.c$":
+    - "^\\s*//"

data/README.md

@@ -1,42 +1,43 @@
 # Zeiger
 
-Zeiger is the German word for "pointer", "indicator", "index", "locator". This gem is built in the "index" sense : run
+Zeiger is the German word for "pointer", "indicator", "index", "locator". This gem is built in the "index" sense. To install,
 
 ```
-cd myproject
-zeiger server
+gem install zeiger
 ```
 
-and this gem will create an in-memory index of all text in the filesystem subtree rooted in the current directory.
+In a terminal, run
 
-Query the index thus:
+```
+zeiger server
+```
+
+Zeiger will open a unix-socket under `/tmp` and listen for search and file-list requests. Run this in a terminal:
 
 ```
 cd myproject
 zeiger search "muppets"
 ```
 
-This example returns one line for each line in your projects containing the word "muppets". Output is in the same format as `grep` (so you can hook it up with your emacs for quick project browsing).
+(Replace "myproject" with something meaningful!)
 
-## Installation
+Zeiger will index the current directory if it is not already indexed, then search for lines containing "muppets". Output is in the same format as `grep` (so you can hook it up with your emacs for quick project browsing).
+
+Zeiger will rescan files in the current directory every ten seconds so the index is mostly up-to-date.
 
-```
-gem install 'zeiger'
-```
 
-This is built as a standalone commandline tool ; I don't have any use-cases for integrating it directly into a larger project. But if you do, I'm all ears.
 
 ## Usage
 
-`zeiger server` runs the server and opens a unix filesystem socket called `zeiger-index` in the current directory.
+`zeiger server` runs the server and opens a unix filesystem socket called `/tmp/zeiger-index`
 
 `zeiger search "foo"` writes the query to the socket and displays the result
 
-`zeiger files "xed"` asks for the list of filenames corresponding to the argument ("xed"). With no argument, return all filenames.
+`zeiger files "xed"` asks for the list of filenames corresponding to the argument ("xed"). With no argument, return all filenames. Files are sorted by length of filename. This sounds odd, but works nicely with 'completing-read in emacs: you will find the file you want in fewer keystrokes.
 
-By default, Zeiger searches only in these subdirectories : %w{ app bin config lib spec test }, and excludes filenames matching these patterns: %w{ .gz$ .png$ .jpg$ .pdf$ }.
+By default, Zeiger searches only in these subdirectories : `%w{ app bin config lib spec test }`, and excludes filenames matching these patterns: `%w{ .gz$ .png$ .jpg$ .pdf$ }`.
 
-To override, create a file in your working directory with the following format:
+To override, create a file called `.zeiger.yml` in your project root with the following format:
 
 ```yaml
 search:
@@ -49,10 +50,20 @@ ignore:
   - .gif$
   - .zip$
   - .jpg$
+  - .ico$
   - .xcf$
   - .mpg$
 ```
 
+You should change the contents to suit your indexing needs. The `search` key is a list of regexps, zeiger will index only those files whose name matches at least one of these regexps. The `ignore` key is likewise a list of regexps, zeiger will not index any file whose name matches any of these regexps. The `ignore` rule supercedes the `search` rule.
+
+When you invoke `zeiger search ...` or `zeiger files ...`, zeiger will consider whether an index already exists for the current directory. If not, and the current directory is a project root, it will create an index for the current directory. If there is no index, and the current directory is not a project root, it moves up one directory and tries again.
+
+Zeiger considers a project root any directory containing any one of the following: `%w{ .zeiger.yml .git .hg Makefile Rakefile Gemfile build.xml }` (see `ROOT_FILES` constant in `index.rb`)
+
+## TODO
+
+Use `Listen` gem instead of manual filesystem scan
 
 ## Contributing
 

data/bin/zeiger

@@ -3,14 +3,16 @@
 require 'socket'
 require 'zeiger'
 
-SOCKET_NAME = "./zeiger-index"
-
+pwd     = `pwd`.strip
 command = $*[0]
+query   = $*[1]
 
 case command
-when "server" ; Zeiger::Server.run *$*
-when "search" ; Zeiger::QueryClient.run *$*
-when "files"  ; Zeiger::FileListClient.run *$*
+when "server" ; Zeiger::Server.new.run *$*
+when "search" ; Zeiger::Client.new.send({ command: :search, pwd: pwd, search: query })
+when "files"  ; Zeiger::Client.new.send({ command: :files , pwd: pwd, files: query  })
+when "stats"  ; Zeiger::Client.new.send({ command: :stats , pwd: pwd                })
+else puts "Zeiger version #{Zeiger::VERSION}, usage `zeiger server|search|files|stats`"
 end
 
 # define file groups

data/lib/zeiger.rb

@@ -1,25 +1,22 @@
 require "set"
 require 'yaml'
-require 'zeiger/query_client'
-require 'zeiger/file_list_client'
+require 'zeiger/client'
 require 'zeiger/server'
 require "zeiger/version"
 require 'zeiger/monitor'
+require 'zeiger/stats'
 require 'zeiger/file_info'
 require 'zeiger/index'
 require 'zeiger/line'
 
 class String
   def ngrams size
-    padding = " " * (size - 1)
-    txt = self
-    ngrams = Hash.new { |h,k| h[k] = 0 }
-    regex = Regexp.compile("." * size)
+    s = size - 1
 
     result = []
 
-    (txt.length - 2).times do |i|
-      result << txt[i..(i + 2)]
+    (length - s).times do |i|
+      result << self[i..(i + s)]
     end
 
     result

data/lib/zeiger/{file_list_client.rb → client.rb}

@@ -1,8 +1,11 @@
 module Zeiger
-  class FileListClient
-    def self.run command, q=nil, *args
+  class Client
+    def send data
       Socket.unix(SOCKET_NAME) { |sock|
-        sock.puts("FILES: #{q}")
+        s = YAML.dump(data)
+        sock.write([s.bytesize].pack("I"))
+        sock.write(s)
+
         while !sock.eof?
           puts sock.readline
         end

data/lib/zeiger/file_info.rb

@@ -2,14 +2,17 @@ require 'set'
 
 module Zeiger
   class FileInfo
-    attr_accessor :dir, :filename, :ngrams
+    attr_accessor :index, :filename, :ngrams, :lines, :stats_group, :comment_regexes, :nc_nb_line_count
 
-    def initialize dir, filename
-      @dir, @filename, @ngrams = dir, filename, Set.new
+    def initialize index, filename, stats
+      @index, @filename, @ngrams, @lines = index, filename, Set.new, []
+      @stats_group      = stats.stats_group filename
+      @comment_regexes  = stats.comment_rules(filename)
+      @nc_nb_line_count = 0
     end
 
     def local_filename
-      filename.gsub(/^#{Regexp.escape dir}\//, "")
+      @_local_filename ||= filename.gsub(/^#{Regexp.escape index.dir}\//, "")
     end
 
     def match regex
@@ -19,5 +22,14 @@ module Zeiger
     def add_ngram ngram
       @ngrams << ngram
     end
+
+    def add_line line
+      lines << line
+      @nc_nb_line_count += 1 unless line.blank? || (comment_regexes.any? { |regex| line.matches? regex })
+    end
+
+    def summary
+      "[#{index.name}] #{stats_group}\t#{nc_nb_line_count}/#{lines.count}\t: #{local_filename}"
+    end
   end
 end

data/lib/zeiger/index.rb

@@ -1,14 +1,53 @@
 module Zeiger
+  INDICES = { }
+
   class Index
+    ROOT_GLOBS  = [%w{ *.gemspec       }]
+    ROOT_GROUPS = [%w{ .zeiger.yml     },
+                   %w{ .git            },
+                   %w{ .hg             },
+                   %w{ Makefile        },
+                   %w{ Rakefile        },
+                   %w{ Gemfile         },
+                   %w{ build.xml       },
+                   %w{ lib LICENSE     },
+                   %w{ lib spec        },
+                   %w{ lib MIT-LICENSE },
+                   %w{ lib README.md   },
+                  ]
     NGRAM_SIZE = 3
 
-    attr_accessor :index, :dir, :includes, :ignore, :files
+    attr_accessor :index, :dir, :name, :includes, :ignore, :files, :config, :monitor, :stats
 
     def initialize dir
       attrs = File.exist?(".zeiger.yml") ? YAML.load(File.read ".zeiger.yml") : { }
       self.dir      = File.expand_path dir
+      self.name     = File.basename self.dir
+      self.config   = load_config
       self.index    = Hash.new { |h, k| h[k] = [] }
       self.files    = Hash.new
+      self.monitor  = Monitor.new dir, self, config
+      self.stats    = Stats.new self, config
+      rescan
+    end
+
+    def load_config
+      conf_file = File.join dir, ".zeiger.yml"
+      if File.exist?(conf_file)
+        puts "reading config from #{conf_file.inspect}"
+        YAML.load(File.read conf_file)
+      else
+        { }
+      end
+    end
+
+    def self.from_path path
+      raise "no path! #{path.inspect}" if path == nil || path.strip == ''
+      return INDICES[path] if INDICES[path]
+      return nil if path == '/'
+      return (INDICES[path] = new(path)) if ROOT_GROUPS.any? { |rg| rg.all? { |f| File.exist?(File.join path, f)       } }
+      return (INDICES[path] = new(path)) if ROOT_GLOBS.any?  { |rg| rg.all? { |f| Dir.glob(File.join path, f).size > 0 } }
+      return from_path(File.dirname path)
     end
 
     def remove_from_index file
@@ -22,29 +61,56 @@ module Zeiger
     end
 
     def add_to_index file
-      info = files[file] = FileInfo.new(dir, file)
+      info = files[file] = FileInfo.new(self, file, stats)
 
       File.read(file).split(/\n/).each_with_index { |txt, line|
-        Line.new(info, line + 1, txt).ngrams(NGRAM_SIZE) do |trig, line|
+        line = Line.new(info, line + 1, txt).ngrams(NGRAM_SIZE) do |trig, line|
           index[trig] << line
           info.add_ngram trig
         end
       }
-    end
 
-    def exec_query regex, ngrams
-      ngrams.map { |ngram| index[ngram] }.reduce(&:&).select { |line| line.matches? regex }
+      puts "#{info.summary} : re-index"
+    rescue Exception => e
+      puts "Not able to read #{file}"
+      while e
+        puts e.message
+        puts e.backtrace.join("\n")
+        puts
+        e = e.cause
+      end
     end
 
+    def glob             pattern ; Dir.glob(File.join(dir, pattern))                                     ; end
+    def rescan                   ; monitor.build_index                                                   ; end
+    def get_ngram_lines   ngrams ; ngrams.map { |ngram| index[ngram] || [] }.reduce(&:&)                 ; end
+    def exec_query regex, ngrams ; get_ngram_lines(ngrams).select { |line| line.matches? regex }         ; end
+    def sort_by_filename   lines ; lines.sort_by { |line| [line.file.local_filename, line.line_number] } ; end
+    def all_matching           q ; index.keys.select { |k| k.match q }.map { |k| index[k] }.reduce(&:|)  ; end
+
     def query txt
       puts "got query #{txt.inspect}"
-      exec_query Regexp.compile(txt), txt.ngrams(NGRAM_SIZE)
+
+      lines = if (txt.strip.to_s == '')
+                puts "empty query, not searching!"
+                return []
+              elsif txt.length <= NGRAM_SIZE
+                all_matching Regexp.compile(Regexp.escape txt)
+              else
+                exec_query Regexp.compile(Regexp.escape txt), txt.ngrams(NGRAM_SIZE)
+              end
+
+      sort_by_filename(lines)
     end
 
     def file_list name
-      r = Regexp.compile name
-      puts "file names matching #{r.inspect}"
-      files.values.select { |f| f.match r }
+      if name
+        r = Regexp.compile name
+        puts "file names matching #{r.inspect}"
+        files.values.select { |f| f.match r }
+      else
+        files.values
+      end.sort_by { |f| f.local_filename.length }
     end
   end
 end

data/lib/zeiger/line.rb

@@ -6,12 +6,14 @@ module Zeiger
     def initialize file, line_number, content
       @file, @line_number, @content = file, line_number, content
       @hash = "#{file.filename}##{line_number}".hash
+      file.add_line self
     end
 
     def to_s            ; "#{file.local_filename}:#{line_number}:#{content}"               ; end
-    def matches? regex  ; content.match regex                                              ; end
+    def blank?          ; content.strip == ''                                              ; end
+    def matches?  regex ; content.match regex                                              ; end
     def ngram_list size ; @ngrams ||= content.ngrams(size)                                 ; end
-    def ngrams    size  ; ngram_list(size).each { |ngram| yield ngram, self }              ; end
-    def ==       other  ; self.file == other.file && self.line_number == other.line_number ; end
+    def ngrams     size ; ngram_list(size).each { |ngram| yield ngram, self }              ; end
+    def ==        other ; self.file == other.file && self.line_number == other.line_number ; end
   end
 end

data/lib/zeiger/monitor.rb

@@ -2,11 +2,10 @@ module Zeiger
   class Monitor
     attr_accessor :dir, :index, :stat, :includes, :ignore
 
-    def initialize dir, index
+    def initialize dir, index, attrs
       @dir, @index, @stat = dir, index, Hash.new
-      attrs = File.exist?(".zeiger.yml") ? YAML.load(File.read ".zeiger.yml") : { }
-      @includes = attrs["includes"] || %w{ app bin config lib spec test }
-      @ignore   = attrs["ignore"]   || %w{ .gz$ .png$ .jpg$ .pdf$ }
+      @includes = attrs["search"] || %w{ app/**/* bin/**/* config/**/* lib/**/* spec/**/* test/**/* }
+      @ignore   = attrs["ignore"] || %w{ .gz$ .png$ .jpg$ .pdf$ .ico$ }
     end
 
     def ignore? filename
@@ -21,12 +20,11 @@ module Zeiger
       started = Time.now
       files = Set.new
       includes.each do |inc|
-        Dir.glob(File.join(dir, inc, "**", "*")).sort.each do |file|
+        index.glob(inc).sort.each do |file|
           if File.file?(file) && !ignore?(file)
             files << file
             mtime = File.stat(file).mtime
             if !uptodate?(file, mtime)
-              puts "re-indexing #{file}"
               index.remove_from_index file
               index.add_to_index file
               stat[file] = mtime
@@ -34,7 +32,13 @@ module Zeiger
           end
         end
       end
-      (Set.new(stat.keys) - files).each { |f| index.remove_from_index f }
+      (Set.new(stat.keys) - files).each { |f|
+        info = index.files[f]
+        index.remove_from_index f
+        stat.delete f
+        help = info ? info.summary : "[#{index.name}] #{f}"
+        puts "#{help} : deleted"
+      }
       finished = Time.now
       puts "ngrams       : #{index.index.length}"
       puts "files        : #{index.files.length}"

data/lib/zeiger/server.rb

@@ -1,18 +1,22 @@
+require 'yaml'
+
 module Zeiger
-  class Server
-    def self.run command, *args
-      dir = File.expand_path(".")
-      z = Zeiger::Index.new dir
+  SOCKET_NAME = "/tmp/zeiger-index"
 
+  class Server
+    def run command, sleepytime=nil, *args
+      sleepytime = (sleepytime || 10).to_f
+      raise "sleep time must be greater than zero, got #{sleepytime.inspect}" unless sleepytime > 0.0
+      puts "starting server with sleep interval of #{sleepytime} seconds"
       Thread.new do
         begin
-          puts "monitor thread"
-          monitor = Zeiger::Monitor.new dir, z
-          puts "created monitor"
           while true do
-            puts "scanning..."
-            monitor.build_index
-            sleep 10
+            indices = Zeiger::INDICES.values
+            indices.each do |index|
+              puts "#{Time.now} scanning #{index.name} at #{index.dir}"
+              index.rescan
+            end
+            sleep sleepytime
           end
         rescue Exception => e
           puts e.message
@@ -20,18 +24,31 @@ module Zeiger
         end
       end
 
-      puts "query thread..."
-
       Socket.unix_server_loop(SOCKET_NAME) { |sock, client|
-        puts "query thread: server loop"
         begin
-          incoming = sock.readline.strip.split(/:/, 2).map &:strip
-          case incoming[0]
-          when "SEARCH"
-            z.query(incoming[1]).each { |res| sock.puts res.to_s }
-          when "FILES"
-            z.file_list(incoming[1]).each { |f| sock.puts f.local_filename }
+          length   = sock.read(4).unpack("I")[0]
+          query    = sock.read(length)
+          incoming = YAML.load(query)
+          puts incoming.to_yaml
+
+          index = Index.from_path incoming[:pwd]
+
+          if index
+            puts "querying index #{index.name}"
+
+            case incoming[:command]
+            when :search
+              index.query(incoming[:search]).each { |res| sock.puts res.to_s }
+            when :stats
+              sock.puts index.stats.stats.to_yaml
+            when :files
+              index.file_list(incoming[:files]).each { |f| sock.puts f.local_filename }
+            end
+
+          else
+            puts "no index found for path #{incoming[:pwd].inspect}"
           end
+
         ensure
           sock.close
         end

data/lib/zeiger/stats.rb

@@ -0,0 +1,33 @@
+module Zeiger
+  class Stats
+    attr_accessor :index, :group_cfg, :comment_cfg
+
+    def initialize index, config
+      @index = index
+      @group_cfg = config["stats"]
+      @comment_cfg = compile_comment_cfg config["comments"]
+    end
+
+    def compile_comment_cfg cfg
+      (cfg || { }).each_with_object({ }) do |(file_pattern, regex_list), result|
+        result[Regexp.compile(file_pattern)] = regex_list.map { |r| Regexp.compile r }
+      end
+    end
+
+    def stats_group filename
+      (group_cfg || []).each { |group, regexes| return group if regexes.any? { |r| filename.match r } }
+      "undefined"
+    end
+
+    def comment_rules filename
+      file_type = comment_cfg.keys.detect { |rxp| filename.match rxp }
+      comment_cfg[file_type] || []
+    end
+
+    def stats
+      index.files.values.each_with_object(Hash.new { |h,k| h[k] = 0 }) do |file, results|
+        results[file.stats_group] += file.nc_nb_line_count
+      end
+    end
+  end
+end

data/lib/zeiger/version.rb

@@ -1,3 +1,3 @@
 module Zeiger
-  VERSION = "0.0.1"
+  VERSION = "0.0.6"
 end

data/zeiger.gemspec

@@ -18,6 +18,6 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10.0"
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: zeiger
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.6
 platform: ruby
 authors:
 - conanite
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-25 00:00:00.000000000 Z
+date: 2021-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -55,13 +55,13 @@ files:
 - Rakefile
 - bin/zeiger
 - lib/zeiger.rb
+- lib/zeiger/client.rb
 - lib/zeiger/file_info.rb
-- lib/zeiger/file_list_client.rb
 - lib/zeiger/index.rb
 - lib/zeiger/line.rb
 - lib/zeiger/monitor.rb
-- lib/zeiger/query_client.rb
 - lib/zeiger/server.rb
+- lib/zeiger/stats.rb
 - lib/zeiger/version.rb
 - zeiger.gemspec
 homepage: https://github.com/conanite/zeiger
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: Provide text index of files in current directory tree and a unix socket to

data/lib/zeiger/query_client.rb

@@ -1,12 +0,0 @@
-module Zeiger
-  class QueryClient
-    def self.run command, q, *args
-      Socket.unix(SOCKET_NAME) { |sock|
-        sock.puts("SEARCH: #{q}")
-        while !sock.eof?
-          puts sock.readline
-        end
-      }
-    end
-  end
-end