frett 0.0.2

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2012-06-27
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,16 @@
+History.txt
+Manifest.txt
+README.rdoc
+Rakefile
+bin/frett
+bin/frett_service
+lib/frett.rb
+lib/frett/adapter.rb
+lib/frett/cli.rb
+lib/frett/config.rb
+lib/frett/indexer.rb
+lib/frett/search.rb
+script/console
+test/test_frett.rb
+test/test_frett_cli.rb
+test/test_helper.rb

data/README.rdoc

@@ -0,0 +1,55 @@
+= frett
+
+* http://github.com/dbldots/frett
+
+== DESCRIPTION:
+
+the goal of frett is to provide a much quicker search functionality than ack on large projects.
+it's built on top of the 'ferret' and 'listen' gems.
+
+== FEATURES/PROBLEMS:
+
+* files changes are not tracked until the initial index has been build
+
+== SYNOPSIS:
+
+go to any folder you want to index for searching. then hit
+* 'frett_service start' or
+* 'frett_service start clean' to ensure the index is build from scratch
+* 'frett_service status' 
+
+start your search using 'frett'. see
+* 'frett -h' for usage instructions
+
+== REQUIREMENTS:
+
+frett uses the 'listen' gem. should work out of the box on any platform. (except windows?)
+
+== INSTALL:
+
+* gem install frett
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2012 FIXME full name
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,25 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/frett'
+
+Hoe.plugin :newgem
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'frett' do
+  self.developer 'johannes-kostas goetzinger', 'dbldots@gmail.com'
+  self.extra_deps           = [
+    ['daemons','>= 1.1.8'],
+    ['ferret','>= 0.11.8.4'],
+    ['listen','>= 0.4.7'],
+    ['ptools','>= 1.2.2'],
+    ['colorize','>= 0.5.8'],
+    ['mime-types','>= 1.19']
+  ]
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }

data/bin/frett

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+#
+#  Created on 2012-6-27.
+#  Copyright (c) 2012. All rights reserved.
+
+require File.expand_path(File.dirname(__FILE__) + "/../lib/frett")
+require File.expand_path(File.dirname(__FILE__) + "/../lib/frett/config")
+
+Frett::Config.load_config(Dir.pwd)
+Frett::CLI.execute(STDOUT, ARGV)

data/bin/frett_service

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+#
+#  Created on 2012-6-27.
+#  Copyright (c) 2012. All rights reserved.
+
+require 'rubygems'
+require "daemons"
+require File.expand_path(File.dirname(__FILE__) + "/../lib/frett")
+require File.expand_path(File.dirname(__FILE__) + "/../lib/frett/config")
+
+Frett::Config.load_config(Dir.pwd)
+
+Daemons.run_proc(Frett::Config.service_name,
+  :log_dir => Frett::Config.index_path,
+  :ARGV => ARGV,
+  :log_output => Frett::Config.log) do
+    Frett::Indexer.new(:clean => (ARGV.last == 'clean'))
+end

data/lib/frett/adapter.rb

@@ -0,0 +1,32 @@
+require 'ferret'
+
+class Frett::Adapter
+
+  def initialize(options = {})
+    new_index = options[:clean] || !Frett::Config.consider_mtime
+    unless File.directory?(Frett::Config.index_path)
+      new_index = true
+      Dir.mkdir(Frett::Config.index_path)
+    end
+    @index = Ferret::Index::Index.new(:path => Frett::Config.index_path, :create => new_index)
+    if new_index
+      File.open(Frett::Config.mtime_path, "w") {}
+      past = Time.local(1970,"jan",1,0,0,0)
+      File.utime(past, past, Frett::Config.mtime_path)
+      @index.field_infos.add_field :file,        :store => :yes, :index => :untokenized, :term_vector => :no
+      @index.field_infos.add_field :content,     :store => :yes, :index => :yes
+      @index.field_infos.add_field :line,        :store => :yes, :index => :yes,         :term_vector => :no
+    end
+  end
+
+  def write(&block)
+    yield(@index)
+    @index.optimize
+    File.utime(Time.now, Time.now, Frett::Config.mtime_path)
+  end
+
+  def read
+    yield(@index)
+  end
+
+end

data/lib/frett/cli.rb

@@ -0,0 +1,59 @@
+require 'optparse'
+
+module Frett
+  class CLI
+    def self.execute(stdout, arguments=[])
+      options = {}
+      banner = "Usage: frett [options] 'search string' [directory path]"
+
+      optparse = OptionParser.new do|opts|
+        opts.banner = banner
+
+        options[:escape] = nil
+        opts.on( '-n', '--escape', 'escape special characters' ) do
+          options[:escape] = true
+        end
+        opts.on( '-N', '--no-escape', 
+                 'don\'t escape special chars. you may want to use:',
+                 '\'?\' for any character, \'*\' for multiple characters' ) do
+          options[:escape] = false
+        end
+
+        options[:use_wildcard] = nil
+        opts.on( '-w', '--use-wildcard', 'adds a \'*\' in front & to the end of your search string' ) do
+          options[:use_wildcard] = true
+        end
+        opts.on( '-W', '--no-wildcard', 'don\'t wildcard the search string' ) do
+          options[:use_wildcard] = false
+        end
+
+        options[:use_or] = nil
+        opts.on( '-o', '--use-or', 'build search query using OR for the terms of your search string' ) do
+          options[:use_or] = true
+        end
+        opts.on( '-a', '--use-and', 'build search query using AND for the terms of your search string' ) do
+          options[:use_or] = false
+        end
+
+        opts.on( '-h', '--help', 'Display this screen' ) do
+          puts opts
+          exit
+        end
+      end
+
+      optparse.parse!(arguments)
+      search_options = options.inject({}) { |hsh, (key, value)| value.nil? ? hsh : hsh.merge(key => value) }
+      path = File.join(Frett::Config.working_dir, arguments.last) if arguments.size > 1
+      arguments.pop if path && File.exist?(path)
+      needle = arguments.join(" ")
+
+      puts "WARNING: frett_service is NOT running..".red unless File.exist?(File.join(Frett::Config.working_dir, Frett::Config.service_name << ".pid"))
+
+      if needle.strip.empty?
+        puts banner
+      else
+        Frett::Search.new(search_options).search(needle, path)
+      end
+    end
+  end
+end

data/lib/frett/config.rb

@@ -0,0 +1,42 @@
+class Frett::Config
+
+  class << self
+    def default_options
+      {
+        :exclude => "^tags$|log\/|tmp\/",
+        :num_docs => 100000,
+        :directory => ".frett",
+        :log => true,
+        :consider_mtime => true,
+        :service_name => "frett_service",
+        :search => {
+          :use_wildcard => false,
+          :escape => true,
+          :use_or => false
+        }
+      }
+    end
+
+    def load_config(working_dir)
+      @options = default_options.merge(:working_dir => working_dir)
+      @options.merge!(YAML.load(File.read(self.config_path))) if File.exists?(self.config_path)
+    end
+
+    def index_path
+      File.join(File.expand_path(self.working_dir), self.directory)
+    end
+
+    def mtime_path
+      File.join(index_path, 'mtime')
+    end
+
+    def config_path
+      File.join(File.expand_path(self.working_dir), '.frett.yml')
+    end
+
+    def method_missing(meth, *args, &block)
+      @options[meth]
+    end
+  end
+
+end

data/lib/frett/indexer.rb

@@ -0,0 +1,105 @@
+require 'listen'
+require 'ptools'
+require 'mime/types'
+require 'colorize'
+
+class Frett::Indexer
+
+  def initialize(options = {})
+    @options = options
+    index!
+
+    ::Listen.to(Frett::Config.working_dir) do |modified, added, removed|
+      modified.each do |filename|
+        update_file(filename)
+      end
+
+      added.each do |filename|
+        update_file(filename)
+      end
+
+      removed.each do |filename|
+        remove_file(filename)
+      end
+    end
+  end
+
+  def index!
+    adapter.write do |index|
+      Dir.glob(File.join(Frett::Config.working_dir, "**/*"), File::FNM_CASEFOLD) do |filename|
+        if process?(filename) && needs_index?(filename)
+          index_file(index, filename)
+        end
+      end
+    end
+  end
+
+  def remove_file(filename)
+    return if filename.include?(Frett::Config.directory)
+    adapter.write do |index|
+      remove_from_index(index, filename)
+    end
+  end
+
+  def update_file(filename)
+    return unless process?(filename)
+    adapter.write do |index|
+      remove_from_index(index, filename)
+      index_file(index, filename)
+    end
+  end
+
+  private
+
+  def index_file(index, filename)
+    puts "INDEX #{filename}".green
+    file = File.new(filename, 'r')
+    line = 1
+    now = Time.now.to_i
+    while (text = file.gets)
+      content = text.gsub(/\n/, "").strip
+      index << {
+        :line => line,
+        :content => content,
+        :file => filename
+      }
+      line += 1
+    end
+  end
+
+  def remove_from_index(index, filename)
+    query = Ferret::Search::PrefixQuery.new(:file, filename)
+    doc_ids = index.scan(query, :limit => Frett::Config.num_docs)
+    puts "DELETE #{doc_ids.size} entries for #{filename}".red
+    doc_ids.each do |doc_id|
+      index.delete(doc_id)
+    end
+  end
+
+  def needs_index?(filename)
+    return true unless Frett::Config.consider_mtime
+    File.mtime(filename) > File.mtime(Frett::Config.mtime_path)
+  end
+
+  def process?(filename)
+    File.file?(filename) && 
+      !filename.include?(Frett::Config.directory) &&
+      !filename.include?(Frett::Config.service_name) &&
+      !(filename =~ /.*\/(_darcs|\..+?)\/.*/) &&
+      !excluded?(filename) &&
+      !MIME::Types.of(filename).any? { |type| type.binary? } &&
+      !File.binary?(filename)
+  end
+
+  def excluded?(filename)
+    filename.gsub(File.join(Frett::Config.working_dir,"/"), '') =~ exclude_regexp
+  end
+
+  def exclude_regexp
+    @exclude_regexp ||= Regexp.new(Frett::Config.exclude)
+  end
+
+  def adapter
+    @adapter ||= Frett::Adapter.new(@options)
+  end
+end

data/lib/frett/search.rb

@@ -0,0 +1,45 @@
+class Frett::Search
+
+  def initialize(options = {})
+    @options = Frett::Config.search.merge(:default_field => :content).
+      merge(options)
+  end
+
+  def search(needle, path = nil)
+    results = []
+    adapter.read do |index|
+      index.search_each(build_query(needle, path), :limit => Frett::Config.num_docs) do |doc_id, score|
+        results.push result(index[doc_id])
+      end
+    end
+
+    puts ( results.size == 1 ? "1 match" : "#{results.size} matches" ).white
+    results.flatten.map { |result_line| puts result_line }
+  end
+
+  private
+
+  def result(doc)
+    file = doc[:file].gsub!(File.join(Frett::Config.working_dir,"/"), '')
+    content = ( doc[:content].length > 200 ) ? ( doc[:content][0..77] + "..." ) : doc[:content]
+    ["", file.light_yellow, "#{doc[:line]}: #{content.light_blue}"]
+  end
+
+  def adapter
+    @adapter ||= Frett::Adapter.new
+  end
+
+  def build_query(needle, path)
+    needle = escape(needle) if @options[:escape]
+    needle = "*#{needle}*"  if @options[:use_wildcard]
+    #parsed = Ferret::QueryParser.new(:default_field => @options[:default_field], :or_default => @options[:use_or]).
+      #parse(needle.downcase)
+    q = "content:#{needle.downcase}" << ( path ? " + file:#{path}*" : "" )
+    Ferret::QueryParser.new(:fields => [:content, :file], :tokenized_fields => :content, :or_default => @options[:use_or]).
+      parse(q)
+  end
+
+  def escape(needle)
+    Regexp.escape(needle).gsub(/([:~!<>="])/,'\\\\\1')
+  end
+end

data/lib/frett.rb

@@ -0,0 +1,11 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+module Frett
+  VERSION = '0.0.2'
+end
+
+require 'rubygems'
+%w(config adapter cli indexer search).each do |file|
+  require File.join(File.expand_path(File.dirname(__FILE__)), "frett", file)
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/frett.rb'}"
+puts "Loading frett gem"
+exec "#{irb} #{libs} --simple-prompt"

data/test/test_frett.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestFrett < Test::Unit::TestCase
+
+  def setup
+  end
+  
+  def test_truth
+    assert true
+  end
+end

data/test/test_frett_cli.rb

@@ -0,0 +1,14 @@
+require File.join(File.dirname(__FILE__), "test_helper.rb")
+require 'frett/cli'
+
+class TestFrettCli < Test::Unit::TestCase
+  def setup
+    Frett::CLI.execute(@stdout_io = StringIO.new, [])
+    @stdout_io.rewind
+    @stdout = @stdout_io.read
+  end
+  
+  def test_print_default_output
+    assert_match(/To update this executable/, @stdout)
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/frett'

metadata

@@ -0,0 +1,233 @@
+--- !ruby/object:Gem::Specification 
+name: frett
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- johannes-kostas goetzinger
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-07-03 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: daemons
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 1
+        - 1
+        - 8
+        version: 1.1.8
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: ferret
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 63
+        segments: 
+        - 0
+        - 11
+        - 8
+        - 4
+        version: 0.11.8.4
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: listen
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 0
+        - 4
+        - 7
+        version: 0.4.7
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: ptools
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 1
+        - 2
+        - 2
+        version: 1.2.2
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: colorize
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 0
+        - 5
+        - 8
+        version: 0.5.8
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: mime-types
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 41
+        segments: 
+        - 1
+        - 19
+        version: "1.19"
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: rdoc
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 19
+        segments: 
+        - 3
+        - 10
+        version: "3.10"
+  type: :development
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: newgem
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 1
+        - 5
+        - 3
+        version: 1.5.3
+  type: :development
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 3
+        - 0
+        version: "3.0"
+  type: :development
+  version_requirements: *id009
+description: |-
+  the goal of frett is to provide a much quicker search functionality than ack on large projects.
+  it's built on top of the 'ferret' and 'listen' gems.
+email: 
+- dbldots@gmail.com
+executables: 
+- frett
+- frett_service
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.rdoc
+files: 
+- History.txt
+- Manifest.txt
+- README.rdoc
+- Rakefile
+- bin/frett
+- bin/frett_service
+- lib/frett.rb
+- lib/frett/adapter.rb
+- lib/frett/cli.rb
+- lib/frett/config.rb
+- lib/frett/indexer.rb
+- lib/frett/search.rb
+- script/console
+- test/test_frett.rb
+- test/test_frett_cli.rb
+- test/test_helper.rb
+- .gemtest
+has_rdoc: true
+homepage: http://github.com/dbldots/frett
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: frett
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: the goal of frett is to provide a much quicker search functionality than ack on large projects
+test_files: 
+- test/test_frett.rb
+- test/test_frett_cli.rb
+- test/test_helper.rb