log_slice 0.1

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+

data/Gemfile

@@ -0,0 +1,2 @@
+gemspec
+

data/README.md

@@ -0,0 +1,51 @@
+LogSlice
+========
+
+Uses binary search to find a line quickly in a large log file. O(log2(n))
+
+Can only search sorted data, which means it's probably only useful for searching by timestamp.
+
+## Example
+
+something-interesting.log:
+
+    [2012-08-29 18:41:12] (9640) something something something else 1
+    [2012-08-29 18:41:14] (9640) something something something else 2
+    [2012-08-29 18:41:14] (9640) something something something else 3
+    [2012-08-29 18:41:14] (9640) something something something else 4
+    [2012-08-29 18:42:18] (9640) something something something else 5
+    [2012-08-29 18:42:18] (9640) something something something else 6
+    [2012-08-29 18:42:18] (9640) something something something else 7
+    [2012-08-29 18:42:20] (9640) something something something else 8
+    [2012-08-29 18:42:20] (9640) something something something else 9
+    [2012-08-29 18:42:20] (9640) something something something else 10
+
+extract everything that happened at or after 18:42:18:
+
+    find_date = DateTime.parse("2012-08-29 18:42:18")
+    file = LogSlice.new("something-interesting.log").find do |line|
+        date_string = line.match(/^\[([^\]]+)\]/)[1]
+        find_date <=> DateTime.parse(date_string)
+    end
+
+    # this will yield an instance of File
+    # the position is the file is the first byte of the found line
+
+    file.readline
+    #=> "[2012-08-29 18:42:18] (9640) something something something else 5"
+
+    file.readline
+    #=> "[2012-08-29 18:42:18] (9640) something something something else 6"
+
+LogSlice.new takes a File or file path, and a block. When passed a line,
+the block must return -1 if the value represented by the line is too high,
+1 if it's too low, or 0 if it's just right.
+
+## Limitations
+
+* Can only search sorted data. At the moment, if the data isn't sorting, it won't detect it and it will loop forever.
+* Can only search for a known value. For example, searching for 18:42:19 in the example above will yield nothing.
+
+## Disclaimer
+
+Use this at your own risk. Better yet, don't use it, it probably doesn't work.

data/lib/log_slice.rb

@@ -0,0 +1,129 @@
+class LogSlice
+
+  # @param log_file [File, String]
+  def initialize log_file
+    @file = log_file.respond_to?(:seek) ? log_file : File.open(log_file, 'r')
+    @size = @file.stat.size
+    @lower = 0
+    @upper = @size
+    @char_cursor = nil
+    @line_cursor = nil
+  end
+
+  # Depends on lines being sorted
+  # @return [File] file after seeking to start of line
+  def find &compare
+    direction = :forward
+    line_cursor = nil
+    loop do
+      line = next_line direction
+      if line_cursor == @line_cursor
+        return nil
+      end
+      line_cursor = @line_cursor
+      case compare.call(line)
+        when 0 # found
+          walk_up_to_first_match compare
+          return @file
+        when -1
+          direction = :back
+        when 1
+          direction = :forward
+        else
+          raise ArgumentError
+      end
+    end
+  end
+
+  private
+
+  # @param direction [Symbol] direction in file to move, :forward or :back
+  # @return [String] line
+  def next_line direction
+    move_char_cursor direction
+    find_next_newline
+  end
+
+  # once the line has been found, we must check the lines above it -
+  # if a line above also matches, we should seek to it.
+  # (this make search on some files O(n/2) instead of O(log2(n))) )
+  def walk_up_to_first_match compare
+    move_to_previous_line compare
+    @file.seek(@line_cursor)
+  end
+
+  def move_to_previous_line compare
+    last_cursor_position = @line_cursor
+    each_line_reverse do |line|
+      if compare.call(line) != 0
+        @line_cursor = last_cursor_position
+        break
+      end
+      last_cursor_position = @line_cursor
+    end
+  end
+
+  def each_line_reverse
+    chunk_size = 512
+    left_over = ""
+    cursor = @line_cursor
+    loop do
+      cursor = cursor - chunk_size
+      if cursor < 0
+        chunk_size = chunk_size + cursor
+        cursor = 0
+      end
+      break if chunk_size == 0
+      #puts "seeking to #{cursor}, chunk size #{chunk_size}, left over #{left_over.length}"
+      @file.seek(cursor)
+      chunk = @file.read(chunk_size) + left_over
+      lines = chunk.split("\n")
+      while lines.length > 1
+        line = lines.pop || ""
+        @line_cursor = @line_cursor - (line.length + 1)
+        yield(line)
+      end
+      left_over = lines[0] || ""
+      lines = []
+    end
+    yield left_over unless left_over == ''
+  end
+
+  def find_next_newline
+    newline_char = "\n"[0]
+    @line_cursor = @char_cursor
+    @file.seek(@line_cursor)
+    current_char = nil
+    while (current_char = @file.getc) != newline_char && !current_char.nil?
+      @line_cursor = @line_cursor + 1
+    end
+    if current_char.nil?
+      # eof
+      ""
+    else
+      @line_cursor = @line_cursor + 1
+      @file.seek(@line_cursor)
+      @file.readline
+    end
+  end
+
+  # @param direction [Symbol] direction in file to move the cursor, :forward or :back
+  def move_char_cursor direction
+    if @char_cursor
+      if direction == :forward
+        distance = (@upper - @char_cursor) / 2
+        old_cursor = @char_cursor
+        @char_cursor = @char_cursor + distance
+        @lower = old_cursor
+      else
+        distance = (@char_cursor - @lower) / 2
+        old_cursor = @char_cursor
+        @char_cursor = @char_cursor - distance
+        @upper = old_cursor
+      end
+    else
+      @char_cursor = @size / 2
+    end
+  end
+
+end

data/log_slice.gemspec

@@ -0,0 +1,15 @@
+Gem::Specification.new do |s|
+  s.name        = 'log_slice'
+  s.version     = '0.1'
+  s.authors     = ["Joel Plane"]
+  s.email       = ["joel.plane@gmail.com"]
+  s.homepage    = 'https://github.com/joelplane/log_slice'
+  s.date        = '2012-08-29'
+  s.summary     = "Find a line in a log file"
+  s.description = "Find a line in a log file. Uses binary search to find the line quickly in a large log file. Can only search sorted data - which in the case of log file is the timestamp, and probably not much else."
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {spec}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_development_dependency 'rspec'
+end

data/spec/helper.rb

@@ -0,0 +1,27 @@
+require 'tempfile'
+
+RSpec.configure do |c|
+  c.include(Module.new do
+
+    def range_to_file range
+      file = Tempfile.new("test-#{range}")
+      file.write(range.to_a.join("\n"))
+      file.flush
+      file.seek(0)
+      file
+    end
+
+    def string_to_file string
+      file = Tempfile.new("test-string")
+      file.write(string)
+      file.flush
+      file.seek(0)
+      file
+    end
+
+    def log2 n
+      Math.log(n) / Math.log(2)
+    end
+
+  end)
+end

data/spec/log_slice_spec.rb

@@ -0,0 +1,90 @@
+require File.expand_path('../lib/log_slice', File.dirname(__FILE__))
+require 'helper'
+
+describe LogSlice do
+
+  it "finds the line" do
+    log_slice = LogSlice.new(range_to_file 1..100)
+    file = log_slice.find do |line|
+      42 <=> line.to_i
+    end
+    file.readline.should == "42\n"
+  end
+
+  it "finds the first line when there are many matching lines" do
+    log_slice = LogSlice.new(string_to_file (["1", "2"] + ["3"]*20).join("\n"))
+    file = log_slice.find do |line|
+      3 <=> line.to_i
+    end
+    file.pos.should == "1\n2\n".length
+    file.readline.should == "3\n"
+  end
+
+  it "finds a matching line with log2(lines)+1 calls to comparison function" do
+    [100, 10_000, 100_000].each do |total_lines|
+      log_slice = LogSlice.new(range_to_file 1..total_lines)
+      comparisons_count = 0
+      log_slice.find do |line|
+        comparisons_count = comparisons_count + 1
+        42 <=> line.to_i
+      end
+      comparisons_count.should <= log2(total_lines).ceil + 1
+    end
+  end
+
+  it "nil when no matching line is found (1)" do
+    log_slice = LogSlice.new(range_to_file 1..100)
+    file = log_slice.find do |line|
+      1
+    end
+    file.should be_nil
+  end
+
+  it "nil when no matching line is found (2)" do
+    log_slice = LogSlice.new(range_to_file 1..100)
+    file = log_slice.find do |line|
+      -1
+    end
+    file.should be_nil
+  end
+
+  it "nil when acting on an empty file" do
+    log_slice = LogSlice.new(string_to_file "")
+    file = log_slice.find do |line|
+      42 <=> line.to_i
+    end
+    file.should be_nil
+  end
+
+  it "#each_line_reverse" do
+    log_slice = LogSlice.new(range_to_file 1..10000)
+    log_slice.instance_eval { @line_cursor = @size }
+    lines = []
+    file = log_slice.send(:each_line_reverse) do |line|
+      lines << line.strip.to_i
+    end
+    lines.should == Array(1..10000).reverse
+  end
+
+  it "#each_line_reverse when file is empty" do
+    log_slice = LogSlice.new(string_to_file "")
+    log_slice.instance_eval { @line_cursor = @size }
+    lines = []
+    file = log_slice.send(:each_line_reverse) do |line|
+      lines << line.strip.to_i
+    end
+    lines.should == []
+  end
+
+  it "#each_line_reverse when file has single newline char" do
+    log_slice = LogSlice.new(string_to_file "\n")
+    log_slice.instance_eval { @line_cursor = @size }
+    lines = []
+    file = log_slice.send(:each_line_reverse) do |line|
+      lines << line.strip.to_i
+    end
+    lines.should == []
+  end
+
+
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification 
+name: log_slice
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Joel Plane
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-08-29 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: Find a line in a log file. Uses binary search to find the line quickly in a large log file. Can only search sorted data - which in the case of log file is the timestamp, and probably not much else.
+email: 
+- joel.plane@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.md
+- lib/log_slice.rb
+- log_slice.gemspec
+- spec/helper.rb
+- spec/log_slice_spec.rb
+homepage: https://github.com/joelplane/log_slice
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Find a line in a log file
+test_files: []
+