rsgrep 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+spec/data
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rsgrep.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Sam Rose
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# Rsgrep
+
+This is a pure Ruby implementation with the same goal as the small but amazing
+[sorted grep](http://sourceforge.net/projects/sgrep/) program written by
+Stephen C. Losen.
+
+It is designed for use on large, lexicographically sorted files. It allows you
+to search for lines that *begin* with a certain pattern (searching for anything
+at a position anywhere other than the start of a line isn't possible using a
+binary search).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rsgrep'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rsgrep
+
+## Usage
+
+The gem monkey patches into the File class. It can be used in the following two
+ways:
+
+``` ruby
+require 'rsgrep'
+
+puts File.sgrep("key pattern", "path/to/file.txt")
+#=> array of all lines that start with "key pattern", empty array for no
+#   matches.
+
+# or ...
+
+f = File.open("path/to/file.txt")
+puts f.sgrep("key pattern")
+#=> array of all lines that start with "key pattern", empty array for no
+#   matches.
+
+f.close
+```
+
+You can pass both of these functions an options hash. Here are some examples of
+the options you can pass:
+
+``` ruby
+require 'rsgrep'
+
+f = File.open("path/to/file.txt")
+
+# Case insensitive search
+f.sgrep("PaTTern", :insensitive => true)
+
+f.close
+```
+
+**NOTE**: There are a lot of caveat involved in getting this to work properly.
+For example, you **cannot** do a case insensitive search on a file that is not
+sorted in a case insensitive fashion. The results will not be what you expect.
+
+This will be true of almost all options you pass to rsgrep. You will get the
+best results on a file that uses alphanumeric characters and only uses one
+casing (upper or lower, doesn't matter which).
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## A note on the specs...
+
+Because writing specs for this required having a very large file to scan, I had
+to choose a very large file that was freely available. For obvious reasons, I
+cannot put the file into this repository but you can download it from
+[here](http://books.google.com/ngrams/datasets). It's the 0th file of the 3grams
+dataset in English.
+
+Direct link:
+[http://commondatastorage.googleapis.com/books/ngrams/books/googlebooks-eng-all-3gram-20090715-0.csv.zip](http://commondatastorage.googleapis.com/books/ngrams/books/googlebooks-eng-all-3gram-20090715-0.csv.zip)
+
+It's about 440mb compressed, 3gb uncompressed. You will need to uncompress it
+into the `spec/data` directory in order to run the specs successfully.
+
+This file is a bit of a bad example though, to be honest. I'm only using it at
+the moment so that the specs give a good idea of how long it takes to scan
+through such a large file. The reason that this is not a good file to use is
+because it isn't sorted in a way the rsgrep knows how to process yet. Its
+handling of capital letters and punctuation are a bit confusing and I haven't
+yet been able to find a consistent and clean way of scanning it.

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+desc 'Default: run specs.'
+task :default => :spec
+
+desc "Run specs"
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = '-cfs'
+end

data/bin/rsgrep

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '..', 'lib', 'rsgrep')
+require 'optparse'
+
+options = {}
+
+OptionParser.new do |opts|
+  opts.on('-i', '--insensitive', 'Case insensitive search.') do
+    options[:insensitive] = true
+  end
+end.parse!
+
+key  = ARGV.shift
+file = ARGV.shift
+
+begin
+  puts File.sgrep(key, file, options)
+rescue Errno::EPIPE
+  # If piping to another program, then that program closes STDOUT (by exiting
+  # for example), you get an EPIPE excetion. We don't really care about it. This
+  # is just to stop error output.
+end

data/lib/rsgrep.rb

@@ -0,0 +1,8 @@
+libdir = File.dirname(__FILE__)
+$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
+
+require "rsgrep/version"
+require "rsgrep/file"
+
+module Rsgrep
+end

data/lib/rsgrep/file.rb

@@ -0,0 +1,77 @@
+class File
+  def self.sgrep key, filename, options = {}
+    File.open(filename) do |f|
+      f.sgrep key, options
+    end
+  end
+
+  def sgrep key, options = {}
+    # initialise the variables that the binary search algorithm needs.
+    hi  = size
+    lo  = 0
+    mid = (hi + lo) / 2
+    ret = []
+
+    if options[:insensitive]
+      comparator = Proc.new do |key, line|
+        key  = key.downcase
+        line = line.downcase
+        if line.start_with? key
+          0
+        else
+          key <=> line
+        end
+      end
+    else
+      comparator = Proc.new do |key, line|
+        if line.start_with? key
+          0
+        else
+          key <=> line
+        end
+      end
+    end
+
+    while lo < mid and mid < hi
+      seek(mid)
+      seek(-2, IO::SEEK_CUR) while (c = getc) != "\n" and c != nil
+
+      case comparator.call(key, (line = gets))
+      when 0
+        # So we've found a line that matches the key, but this may not be the
+        # first line that does (due to the nature of binary searching). This
+        # begin/end block scans backwards in the file to find the first line
+        # that matches the key.
+        begin
+          # Seek back 2 lines because "gets" will advance the file pointer
+          # forward.
+          2.times do
+            seek(-2, IO::SEEK_CUR) # First read will always be a newline, skip it
+            seek(-2, IO::SEEK_CUR) while (c = getc) != "\n" and c != nil
+          end
+        end while comparator.call(key, (line = gets)) == 0
+
+        # Then scan forward line by line until all of the lines that match have
+        # been added to the return array.
+        ret << line.rstrip while (line = gets) != nil and comparator.call(key, line) == 0
+
+        # and we're done!
+        return ret
+      when -1
+        # Key is less than the line. Shift the hi value down and recalculate the
+        # mid.
+        hi = mid
+        mid = (hi + lo) / 2
+      when 1
+        # Key is greater than the line. Shift the lo value up and recalculate
+        # the mid.
+        lo = mid
+        mid = (hi + lo) / 2
+      else
+        raise "Should not be raised, ever."
+      end
+    end
+
+    return ret
+  end
+end

data/lib/rsgrep/version.rb

@@ -0,0 +1,3 @@
+module Rsgrep
+  VERSION = "0.0.1"
+end

data/rsgrep.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/rsgrep/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Sam Rose"]
+  gem.email         = ["samwho@lbak.co.uk"]
+  gem.description   = %q{Pure Ruby implementation of the sorted grep command.}
+  gem.summary       = %q{sgrep for Ruby!}
+  gem.homepage      = "http://github.com/samwho/rsgrep"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "rsgrep"
+  gem.require_paths = ["lib"]
+  gem.version       = Rsgrep::VERSION
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rspec-core'
+end

data/spec/file_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+describe "File#sgrep" do
+  let(:data_file1) { File.open(DATA_FILE1) }
+
+  after :each do
+    data_file1.close
+  end
+
+  context "when searching for 'search for '" do
+    key = "search for "
+
+    subject      { data_file1.sgrep key }
+    it           { should_not be_empty }
+
+    specify "all elements start with '#{key}'" do
+      subject.all? { |elem| elem.start_with? key }.should be_true
+    end
+  end
+
+  context "when searching for 'a'" do
+    key = 'a'
+
+    subject      { data_file1.sgrep key }
+    it           { should_not be_empty }
+
+    specify "all elements start with '#{key}'" do
+      subject.all? { |elem| elem.start_with? key }.should be_true
+    end
+  end
+
+  context "when searching for '!'" do
+    key = '!'
+
+    subject      { data_file1.sgrep key }
+    it           { should_not be_empty }
+
+    specify "all elements start with '#{key}'" do
+      subject.all? { |elem| elem.start_with? key }.should be_true
+    end
+  end
+
+  context "no results (search 'this is not in this file anywhere')" do
+    key = 'this is not in this file anywhere'
+
+    subject      { data_file1.sgrep key }
+    it           { should be_empty }
+  end
+
+  if DICT_FILE
+    let(:dict_file) { File.open(DICT_FILE) }
+
+    after :each do
+      dict_file.close
+    end
+
+    context "Dictionary data tests" do
+      context "case insensitive searches" do
+        context "search for 'zyzzogeton'" do
+          key = "zyzzogeton"
+
+          subject { dict_file.sgrep key, :insensitive => true }
+          it      { should include "Zyzzogeton" }
+        end
+      end
+    end
+  else
+    STDERR.puts "It doesn't seem like you have a dictionary file in a standard location. Skipping dictionary file tests."
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require File.join(File.dirname(__FILE__), '..', 'lib', 'rsgrep')
+
+DATA_ROOT  = File.join(File.dirname(__FILE__), 'data')
+DATA_FILE1 = File.join(DATA_ROOT, 'googlebooks-eng-all-3gram-20090715-0.csv')
+
+DICT_FILE  = if File.exists?("/usr/share/dict/words")
+               "/usr/share/dict/words"
+             else
+               nil
+             end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: rsgrep
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Sam Rose
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Pure Ruby implementation of the sorted grep command.
+email:
+- samwho@lbak.co.uk
+executables:
+- rsgrep
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/rsgrep
+- lib/rsgrep.rb
+- lib/rsgrep/file.rb
+- lib/rsgrep/version.rb
+- rsgrep.gemspec
+- spec/file_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/samwho/rsgrep
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1288555168689700844
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1288555168689700844
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: sgrep for Ruby!
+test_files:
+- spec/file_spec.rb
+- spec/spec_helper.rb