zeiger 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c01cd920ba6af86d17bd79b07e648d5e1bb37ad2
+  data.tar.gz: d90ce8ed808e1853aad0175d6fa862d0b3ff2a29
+SHA512:
+  metadata.gz: 5c24d14cba4ac6840aa2ec817c22a608bc98a476289643bdaa6b06fcd50649f82a189f45fdc5c14ef882a7cb815bdcd900c4f387d2d3baa2f4ffeb8edb90b223
+  data.tar.gz: 491a7131fc1f5b5ba29b207d31d2c5b232ed9307650a17aecd6d9383f101debefd6f7e5366c79a2e8e456e82f3d52b3fa6c799a7282b64b4ff5ab2bd683e9aff

data/.gitignore

@@ -0,0 +1,19 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*.gem
+test/tmp
+test/version_tmp
+tmp
+.#*

data/.zeiger.yml

@@ -0,0 +1,9 @@
+search:
+  - bin
+  - lib
+ignore:
+  - .gz$
+  - .png$
+  - .gif$
+  - .zip$
+  - .jpg$

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in zeiger.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2017 conanite
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,63 @@
+# Zeiger
+
+Zeiger is the German word for "pointer", "indicator", "index", "locator". This gem is built in the "index" sense : run
+
+```
+cd myproject
+zeiger server
+```
+
+and this gem will create an in-memory index of all text in the filesystem subtree rooted in the current directory.
+
+Query the index thus:
+
+```
+cd myproject
+zeiger search "muppets"
+```
+
+This example returns one line for each line in your projects containing the word "muppets". Output is in the same format as `grep` (so you can hook it up with your emacs for quick project browsing).
+
+## Installation
+
+```
+gem install 'zeiger'
+```
+
+This is built as a standalone commandline tool ; I don't have any use-cases for integrating it directly into a larger project. But if you do, I'm all ears.
+
+## Usage
+
+`zeiger server` runs the server and opens a unix filesystem socket called `zeiger-index` in the current directory.
+
+`zeiger search "foo"` writes the query to the socket and displays the result
+
+`zeiger files "xed"` asks for the list of filenames corresponding to the argument ("xed"). With no argument, return all filenames.
+
+By default, Zeiger searches only in these subdirectories : %w{ app bin config lib spec test }, and excludes filenames matching these patterns: %w{ .gz$ .png$ .jpg$ .pdf$ }.
+
+To override, create a file in your working directory with the following format:
+
+```yaml
+search:
+  - bin
+  - lib
+  - config.rb
+ignore:
+  - .gz$
+  - .png$
+  - .gif$
+  - .zip$
+  - .jpg$
+  - .xcf$
+  - .mpg$
+```
+
+
+## Contributing
+
+1. Fork it ( https://github.com/conanite/zeiger/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/zeiger

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'zeiger'
+
+SOCKET_NAME = "./zeiger-index"
+
+command = $*[0]
+
+case command
+when "server" ; Zeiger::Server.run *$*
+when "search" ; Zeiger::QueryClient.run *$*
+when "files"  ; Zeiger::FileListClient.run *$*
+end
+
+# define file groups
+
+# file_group :app, patterns: [/^app/ ]
+# file_group :helpers, patterns: [/^app\/helpers/ ]
+# file_group :spec, patterns: [/^spec/ ]
+# file_group :sass, patterns: [/\.sass$/ ]
+# file_group :all, groups: %w{app spec sass}
+
+
+# writing to socket: nc -U /tmp/uss

data/lib/zeiger/file_info.rb

@@ -0,0 +1,23 @@
+require 'set'
+
+module Zeiger
+  class FileInfo
+    attr_accessor :dir, :filename, :ngrams
+
+    def initialize dir, filename
+      @dir, @filename, @ngrams = dir, filename, Set.new
+    end
+
+    def local_filename
+      filename.gsub(/^#{Regexp.escape dir}\//, "")
+    end
+
+    def match regex
+      local_filename.match regex
+    end
+
+    def add_ngram ngram
+      @ngrams << ngram
+    end
+  end
+end

data/lib/zeiger/file_list_client.rb

@@ -0,0 +1,12 @@
+module Zeiger
+  class FileListClient
+    def self.run command, q=nil, *args
+      Socket.unix(SOCKET_NAME) { |sock|
+        sock.puts("FILES: #{q}")
+        while !sock.eof?
+          puts sock.readline
+        end
+      }
+    end
+  end
+end

data/lib/zeiger/index.rb

@@ -0,0 +1,50 @@
+module Zeiger
+  class Index
+    NGRAM_SIZE = 3
+
+    attr_accessor :index, :dir, :includes, :ignore, :files
+
+    def initialize dir
+      attrs = File.exist?(".zeiger.yml") ? YAML.load(File.read ".zeiger.yml") : { }
+      self.dir      = File.expand_path dir
+      self.index    = Hash.new { |h, k| h[k] = [] }
+      self.files    = Hash.new
+    end
+
+    def remove_from_index file
+      info = files[file]
+      if info
+        info.ngrams.each { |ngram|
+          index[ngram].reject! { |line| line.file == info }
+        }
+      end
+      files.delete file
+    end
+
+    def add_to_index file
+      info = files[file] = FileInfo.new(dir, file)
+
+      File.read(file).split(/\n/).each_with_index { |txt, line|
+        Line.new(info, line + 1, txt).ngrams(NGRAM_SIZE) do |trig, line|
+          index[trig] << line
+          info.add_ngram trig
+        end
+      }
+    end
+
+    def exec_query regex, ngrams
+      ngrams.map { |ngram| index[ngram] }.reduce(&:&).select { |line| line.matches? regex }
+    end
+
+    def query txt
+      puts "got query #{txt.inspect}"
+      exec_query Regexp.compile(txt), txt.ngrams(NGRAM_SIZE)
+    end
+
+    def file_list name
+      r = Regexp.compile name
+      puts "file names matching #{r.inspect}"
+      files.values.select { |f| f.match r }
+    end
+  end
+end

data/lib/zeiger/line.rb

@@ -0,0 +1,17 @@
+module Zeiger
+  class Line
+    attr_accessor :file, :line_number, :content
+    attr_reader :hash
+
+    def initialize file, line_number, content
+      @file, @line_number, @content = file, line_number, content
+      @hash = "#{file.filename}##{line_number}".hash
+    end
+
+    def to_s            ; "#{file.local_filename}:#{line_number}:#{content}"               ; end
+    def matches? regex  ; content.match regex                                              ; end
+    def ngram_list size ; @ngrams ||= content.ngrams(size)                                 ; end
+    def ngrams    size  ; ngram_list(size).each { |ngram| yield ngram, self }              ; end
+    def ==       other  ; self.file == other.file && self.line_number == other.line_number ; end
+  end
+end

data/lib/zeiger/monitor.rb

@@ -0,0 +1,44 @@
+module Zeiger
+  class Monitor
+    attr_accessor :dir, :index, :stat, :includes, :ignore
+
+    def initialize dir, index
+      @dir, @index, @stat = dir, index, Hash.new
+      attrs = File.exist?(".zeiger.yml") ? YAML.load(File.read ".zeiger.yml") : { }
+      @includes = attrs["includes"] || %w{ app bin config lib spec test }
+      @ignore   = attrs["ignore"]   || %w{ .gz$ .png$ .jpg$ .pdf$ }
+    end
+
+    def ignore? filename
+      ignore.any? { |ig| filename.match ig }
+    end
+
+    def uptodate? filename, mtime
+      stat.include?(filename) && stat[filename] == mtime
+    end
+
+    def build_index
+      started = Time.now
+      files = Set.new
+      includes.each do |inc|
+        Dir.glob(File.join(dir, inc, "**", "*")).sort.each do |file|
+          if File.file?(file) && !ignore?(file)
+            files << file
+            mtime = File.stat(file).mtime
+            if !uptodate?(file, mtime)
+              puts "re-indexing #{file}"
+              index.remove_from_index file
+              index.add_to_index file
+              stat[file] = mtime
+            end
+          end
+        end
+      end
+      (Set.new(stat.keys) - files).each { |f| index.remove_from_index f }
+      finished = Time.now
+      puts "ngrams       : #{index.index.length}"
+      puts "files        : #{index.files.length}"
+      puts "#build_index : #{(finished - started)} sec"
+    end
+  end
+end

data/lib/zeiger/query_client.rb

@@ -0,0 +1,12 @@
+module Zeiger
+  class QueryClient
+    def self.run command, q, *args
+      Socket.unix(SOCKET_NAME) { |sock|
+        sock.puts("SEARCH: #{q}")
+        while !sock.eof?
+          puts sock.readline
+        end
+      }
+    end
+  end
+end

data/lib/zeiger/server.rb

@@ -0,0 +1,41 @@
+module Zeiger
+  class Server
+    def self.run command, *args
+      dir = File.expand_path(".")
+      z = Zeiger::Index.new dir
+
+      Thread.new do
+        begin
+          puts "monitor thread"
+          monitor = Zeiger::Monitor.new dir, z
+          puts "created monitor"
+          while true do
+            puts "scanning..."
+            monitor.build_index
+            sleep 10
+          end
+        rescue Exception => e
+          puts e.message
+          puts e.backtrace
+        end
+      end
+
+      puts "query thread..."
+
+      Socket.unix_server_loop(SOCKET_NAME) { |sock, client|
+        puts "query thread: server loop"
+        begin
+          incoming = sock.readline.strip.split(/:/, 2).map &:strip
+          case incoming[0]
+          when "SEARCH"
+            z.query(incoming[1]).each { |res| sock.puts res.to_s }
+          when "FILES"
+            z.file_list(incoming[1]).each { |f| sock.puts f.local_filename }
+          end
+        ensure
+          sock.close
+        end
+      }
+    end
+  end
+end

data/lib/zeiger/version.rb

@@ -0,0 +1,3 @@
+module Zeiger
+  VERSION = "0.0.1"
+end

data/lib/zeiger.rb

@@ -0,0 +1,27 @@
+require "set"
+require 'yaml'
+require 'zeiger/query_client'
+require 'zeiger/file_list_client'
+require 'zeiger/server'
+require "zeiger/version"
+require 'zeiger/monitor'
+require 'zeiger/file_info'
+require 'zeiger/index'
+require 'zeiger/line'
+
+class String
+  def ngrams size
+    padding = " " * (size - 1)
+    txt = self
+    ngrams = Hash.new { |h,k| h[k] = 0 }
+    regex = Regexp.compile("." * size)
+
+    result = []
+
+    (txt.length - 2).times do |i|
+      result << txt[i..(i + 2)]
+    end
+
+    result
+  end
+end

data/zeiger.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'zeiger/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "zeiger"
+  spec.version       = Zeiger::VERSION
+  spec.authors       = ["conanite"]
+  spec.email         = ["conan@conandalton.net"]
+  spec.summary       = %q{Provide text index of files in current directory tree and a unix socket to query on}
+  spec.description   = %q{Maintain an in-memory text index of files in current directory tree, allow querying via unix socket in current directory }
+  spec.homepage      = "https://github.com/conanite/zeiger"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: zeiger
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- conanite
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 'Maintain an in-memory text index of files in current directory tree,
+  allow querying via unix socket in current directory '
+email:
+- conan@conandalton.net
+executables:
+- zeiger
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".zeiger.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/zeiger
+- lib/zeiger.rb
+- lib/zeiger/file_info.rb
+- lib/zeiger/file_list_client.rb
+- lib/zeiger/index.rb
+- lib/zeiger/line.rb
+- lib/zeiger/monitor.rb
+- lib/zeiger/query_client.rb
+- lib/zeiger/server.rb
+- lib/zeiger/version.rb
+- zeiger.gemspec
+homepage: https://github.com/conanite/zeiger
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Provide text index of files in current directory tree and a unix socket to
+  query on
+test_files: []