log_slice 0.1 → 0.2

data/README.md

@@ -5,46 +5,59 @@ Uses binary search to find a line quickly in a large log file. O(log2(n))
 
 Can only search sorted data, which means it's probably only useful for searching by timestamp.
 
+## Installation
+
+    gem install log_slice
+
 ## Example
 
 something-interesting.log:
 
-    [2012-08-29 18:41:12] (9640) something something something else 1
-    [2012-08-29 18:41:14] (9640) something something something else 2
-    [2012-08-29 18:41:14] (9640) something something something else 3
-    [2012-08-29 18:41:14] (9640) something something something else 4
-    [2012-08-29 18:42:18] (9640) something something something else 5
-    [2012-08-29 18:42:18] (9640) something something something else 6
-    [2012-08-29 18:42:18] (9640) something something something else 7
-    [2012-08-29 18:42:20] (9640) something something something else 8
-    [2012-08-29 18:42:20] (9640) something something something else 9
-    [2012-08-29 18:42:20] (9640) something something something else 10
+    [2012-08-29 18:41:12] (9640) 1  something something something else 1
+    [2012-08-29 18:41:14] (9640) 2  something something something else 2
+    [2012-08-29 18:41:14] (9640) 3  something something something else 3
+    [2012-08-29 18:41:14] (9640) 4  something something something else 4
+    [2012-08-29 18:42:18] (9640) 5  something something something else 5
+    [2012-08-29 18:42:18] (9640) 6  something something something else 6
+    [2012-08-29 18:42:18] (9640) 7  something something something else 7
+    [2012-08-29 18:42:20] (9640) 8  something something something else 8
+    [2012-08-29 18:42:20] (9640) 9  something something something else 9
+    [2012-08-29 18:42:20] (9640) 10 something something something else 10
+
+extract everything that happened at or after 18:42:00:
+```ruby
+find_date = DateTime.parse("2012-08-29 18:42:00")
+file = LogSlice.new("something-interesting.log").find do |line|
+    date_string = line.match(/^\[([^\]]+)\]/)[1]
+    find_date <=> DateTime.parse(date_string)
+end
 
-extract everything that happened at or after 18:42:18:
+# this will yield an instance of File
+# the position in the file is the first byte of the found line
 
-    find_date = DateTime.parse("2012-08-29 18:42:18")
-    file = LogSlice.new("something-interesting.log").find do |line|
-        date_string = line.match(/^\[([^\]]+)\]/)[1]
-        find_date <=> DateTime.parse(date_string)
-    end
+file.readline
+#=> "[2012-08-29 18:42:18] (9640) 5  something something something else 5"
 
-    # this will yield an instance of File
-    # the position is the file is the first byte of the found line
+# Once you found the line you were after,
+# you can continue to read subsequent lines:
 
-    file.readline
-    #=> "[2012-08-29 18:42:18] (9640) something something something else 5"
+file.readline
+#=> "[2012-08-29 18:42:18] (9640) 6  something something something else 6"
+```
 
-    file.readline
-    #=> "[2012-08-29 18:42:18] (9640) something something something else 6"
+LogSlice.new takes a File or file path, and a comparison function, passed as a block.
+When passed a line, the block must return -1 if the value represented by the line
+is too high, 1 if it's too low, or 0 if it's just right.
 
-LogSlice.new takes a File or file path, and a block. When passed a line,
-the block must return -1 if the value represented by the line is too high,
-1 if it's too low, or 0 if it's just right.
+```ruby
+LogSlice.new(file_or_file_path).find(&comparison_function) #=> File or nil
+```
 
 ## Limitations
 
-* Can only search sorted data. At the moment, if the data isn't sorting, it won't detect it and it will loop forever.
-* Can only search for a known value. For example, searching for 18:42:19 in the example above will yield nothing.
+* Can only search sorted data. If the data isn't sorted, it will most likely not find anything (ie return nil).
+  In very rare cases it may find value the anyway by chance, so it's not guaranteed that unsorted
+  input will always yield nil.
 
 ## Disclaimer
 

data/lib/log_slice/search_boundary.rb

@@ -0,0 +1,34 @@
+class LogSlice
+  class SearchBoundary
+
+    attr_reader :cursor
+
+    def initialize file_size
+      @file_size = file_size
+    end
+
+    # reset the search boundary to cover the entire file
+    def reset
+      @lower = 0
+      @upper = @file_size
+      @cursor = 0
+      cursor_forward
+      self
+    end
+
+    # Move cursor forward.
+    # The cursor is moved half way between it's start location and the upper boundary.
+    def cursor_forward
+      @lower = @cursor
+      @cursor = @cursor + (@upper - @cursor) / 2
+    end
+
+    # Move cursor backward.
+    # The cursor is moved half way between it's start location and the lower boundary.
+    def cursor_back
+      @upper = @cursor
+      @cursor = @cursor - (@cursor - @lower) / 2
+    end
+
+  end
+end

data/lib/log_slice.rb

@@ -1,129 +1,142 @@
+require File.expand_path("log_slice/search_boundary", File.dirname(__FILE__))
+
 class LogSlice
 
+  NEWLINE = "\n"
+  NEWLINE_CHAR = "\n"[0]
+
   # @param log_file [File, String]
-  def initialize log_file
+  # @param options [Hash] :exact_match default false
+  def initialize log_file, options={}
     @file = log_file.respond_to?(:seek) ? log_file : File.open(log_file, 'r')
-    @size = @file.stat.size
-    @lower = 0
-    @upper = @size
-    @char_cursor = nil
+    @exact_match = options[:exact_match] || false
+    @search_boundary = SearchBoundary.new(@file.stat.size)
     @line_cursor = nil
   end
 
-  # Depends on lines being sorted
-  # @return [File] file after seeking to start of line
+  # Find line in the file using the comparison function.
+  # Depends on lines being sorted.
+  # The comparison function will be passed lines from the file. It must
+  # return -1 if the line is later than the one it's looking for, 1 if
+  # the line is earlier than the one it's looking for, and 0 if it is
+  # the line it's looking for.
+  # @param compare [Proc] comparison function
+  # @return [File, nil] file after seeking to start of line or nil if line not found
   def find &compare
-    direction = :forward
-    line_cursor = nil
-    loop do
-      line = next_line direction
-      if line_cursor == @line_cursor
-        return nil
-      end
-      line_cursor = @line_cursor
-      case compare.call(line)
-        when 0 # found
-          walk_up_to_first_match compare
-          return @file
-        when -1
-          direction = :back
-        when 1
-          direction = :forward
-        else
-          raise ArgumentError
+    reset_progress_check
+    @search_boundary.reset
+    line = find_next_newline
+    while making_progress?
+      comp_value = compare.call(line)
+      if comp_value == 0 # found matching line
+        backtrack_to_first_line_match compare
+        return @file
+      else
+        @search_boundary.send(comp_value < 0 ? :cursor_back : :cursor_forward)
+        line = find_next_newline
       end
     end
+    if @exact_match
+      nil
+    else
+      backtrack_to_gap compare
+      return @file.eof? ? nil : @file
+    end
   end
 
   private
 
-  # @param direction [Symbol] direction in file to move, :forward or :back
-  # @return [String] line
-  def next_line direction
-    move_char_cursor direction
-    find_next_newline
+  # whether the cursor has moved since previous call
+  def making_progress?
+    return false if @previous_cursor_position == @line_cursor
+    @previous_cursor_position = @line_cursor
+    true
   end
 
+  def reset_progress_check
+    @previous_cursor_position = nil
+  end
+
+
   # once the line has been found, we must check the lines above it -
   # if a line above also matches, we should seek to it.
   # (this make search on some files O(n/2) instead of O(log2(n))) )
-  def walk_up_to_first_match compare
-    move_to_previous_line compare
+  # @param compare [Proc] comparison function
+  def backtrack_to_first_line_match compare
+    previous_cursor_position = @line_cursor
+    each_line_reverse do |line|
+      if compare.call(line) != 0
+        # we've found a non-matching line,
+        # so we set @line_cursor back to the previous matching line
+        @line_cursor = previous_cursor_position
+        break
+      end
+      previous_cursor_position = @line_cursor
+    end
     @file.seek(@line_cursor)
   end
 
-  def move_to_previous_line compare
-    last_cursor_position = @line_cursor
+  # if no match was found, we're sitting at too-high.
+  # backtrack up to the first too-high
+  def backtrack_to_gap compare
+    @line_cursor = @file.pos
+    previous_cursor_position = @line_cursor
     each_line_reverse do |line|
-      if compare.call(line) != 0
-        @line_cursor = last_cursor_position
+      if compare.call(line) == 1
+        @line_cursor = previous_cursor_position
         break
       end
-      last_cursor_position = @line_cursor
+      previous_cursor_position = @line_cursor
     end
+    @file.seek(@line_cursor)
   end
 
+  # iterate over each line from the current cursor position, in reverse.
   def each_line_reverse
     chunk_size = 512
-    left_over = ""
     cursor = @line_cursor
+    left_over = ""
     loop do
-      cursor = cursor - chunk_size
-      if cursor < 0
-        chunk_size = chunk_size + cursor
+      if chunk_size > cursor
+        chunk_size = cursor
         cursor = 0
+      else
+        cursor -= chunk_size
       end
       break if chunk_size == 0
-      #puts "seeking to #{cursor}, chunk size #{chunk_size}, left over #{left_over.length}"
       @file.seek(cursor)
       chunk = @file.read(chunk_size) + left_over
-      lines = chunk.split("\n")
+      lines = chunk.split(NEWLINE)
       while lines.length > 1
         line = lines.pop || ""
-        @line_cursor = @line_cursor - (line.length + 1)
+        @line_cursor -= (line.length + NEWLINE.length)
         yield(line)
       end
       left_over = lines[0] || ""
       lines = []
     end
+    @line_cursor -= (left_over.length + NEWLINE.length)
     yield left_over unless left_over == ''
   end
 
+  # After the search is moved by cursor search_boundary.cursor_*, it's position
+  # is probably not at the start of a line, but somewhere within a line.
+  # find_next_newline advances the cursor until we're at the start of the
+  # next line.
   def find_next_newline
-    newline_char = "\n"[0]
-    @line_cursor = @char_cursor
+    @line_cursor = @search_boundary.cursor
     @file.seek(@line_cursor)
-    current_char = nil
-    while (current_char = @file.getc) != newline_char && !current_char.nil?
-      @line_cursor = @line_cursor + 1
+    while (current_char = @file.getc) != NEWLINE_CHAR && !current_char.nil?
+      @line_cursor += 1
     end
-    if current_char.nil?
-      # eof
+    if @file.eof?
       ""
     else
-      @line_cursor = @line_cursor + 1
+      @line_cursor += 1
       @file.seek(@line_cursor)
       @file.readline
     end
   end
 
-  # @param direction [Symbol] direction in file to move the cursor, :forward or :back
-  def move_char_cursor direction
-    if @char_cursor
-      if direction == :forward
-        distance = (@upper - @char_cursor) / 2
-        old_cursor = @char_cursor
-        @char_cursor = @char_cursor + distance
-        @lower = old_cursor
-      else
-        distance = (@char_cursor - @lower) / 2
-        old_cursor = @char_cursor
-        @char_cursor = @char_cursor - distance
-        @upper = old_cursor
-      end
-    else
-      @char_cursor = @size / 2
-    end
-  end
 
 end

data/log_slice.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log_slice'
-  s.version     = '0.1'
+  s.version     = '0.2'
   s.authors     = ["Joel Plane"]
   s.email       = ["joel.plane@gmail.com"]
   s.homepage    = 'https://github.com/joelplane/log_slice'

data/spec/helper.rb

@@ -3,7 +3,7 @@ require 'tempfile'
 RSpec.configure do |c|
   c.include(Module.new do
 
-    def range_to_file range
+    def enumerable_to_file range
       file = Tempfile.new("test-#{range}")
       file.write(range.to_a.join("\n"))
       file.flush

data/spec/log_slice_spec.rb

@@ -4,7 +4,7 @@ require 'helper'
 describe LogSlice do
 
   it "finds the line" do
-    log_slice = LogSlice.new(range_to_file 1..100)
+    log_slice = LogSlice.new enumerable_to_file(1..100), :exact_match => true
     file = log_slice.find do |line|
       42 <=> line.to_i
     end
@@ -12,7 +12,7 @@ describe LogSlice do
   end
 
   it "finds the first line when there are many matching lines" do
-    log_slice = LogSlice.new(string_to_file (["1", "2"] + ["3"]*20).join("\n"))
+    log_slice = LogSlice.new string_to_file((["1", "2"] + ["3"]*20).join("\n")), :exact_match => true
     file = log_slice.find do |line|
       3 <=> line.to_i
     end
@@ -22,7 +22,7 @@ describe LogSlice do
 
   it "finds a matching line with log2(lines)+1 calls to comparison function" do
     [100, 10_000, 100_000].each do |total_lines|
-      log_slice = LogSlice.new(range_to_file 1..total_lines)
+      log_slice = LogSlice.new enumerable_to_file(1..total_lines), :exact_match => true
       comparisons_count = 0
       log_slice.find do |line|
         comparisons_count = comparisons_count + 1
@@ -32,24 +32,69 @@ describe LogSlice do
     end
   end
 
-  it "nil when no matching line is found (1)" do
-    log_slice = LogSlice.new(range_to_file 1..100)
+  it "nil when no matching line is found (all values lower)" do
+    log_slice = LogSlice.new enumerable_to_file(1..100), :exact_match => true
     file = log_slice.find do |line|
       1
     end
     file.should be_nil
   end
 
-  it "nil when no matching line is found (2)" do
-    log_slice = LogSlice.new(range_to_file 1..100)
+  it "nil when no matching line is found (all values higher)" do
+    log_slice = LogSlice.new enumerable_to_file(1..100), :exact_match => true
     file = log_slice.find do |line|
       -1
     end
     file.should be_nil
   end
 
+  it "exact_match nil when no matching line is found (higher and lower values)" do
+    log_slice = LogSlice.new enumerable_to_file((1..100).to_a-[42]), :exact_match => true
+    file = log_slice.find do |line|
+      42 <=> line.to_i
+    end
+    file.should be_nil
+  end
+
+  it "non-exact_match find when no matching line is found (higher and lower values)" do
+    log_slice = LogSlice.new enumerable_to_file((1..100).to_a-[42])
+    file = log_slice.find do |line|
+      42 <=> line.to_i
+    end
+    file.readline.should == "43\n"
+  end
+
+  it "non-exact_match find when no matching line is found, many dups" do
+    [0,2,4,6].each do |n|
+      log_slice = LogSlice.new enumerable_to_file([1,1,1,3,3,3,5,5,5,7,7,7])
+      file = log_slice.find do |line|
+        n <=> line.to_i
+      end
+      3.times do
+        file.readline.strip.should == "#{n+1}"
+      end
+    end
+  end
+
+  it "non-exact_match find when no matching line is found, beyond" do
+    log_slice = LogSlice.new enumerable_to_file([1,1,1,3,3,3,5,5,5,7,7,7])
+    file = log_slice.find do |line|
+      8 <=> line.to_i
+    end
+    file.should be_nil
+  end
+
+  it "nil when lines are not sorted" do
+    unsorted = [1,99,4,96,7,70,15,67,24,45,30,40]
+    log_slice = LogSlice.new enumerable_to_file(unsorted), :exact_match => true
+    file = log_slice.find do |line|
+      42 <=> line.to_i
+    end
+    file.should be_nil
+  end
+
   it "nil when acting on an empty file" do
-    log_slice = LogSlice.new(string_to_file "")
+    log_slice = LogSlice.new string_to_file(""), :exact_match => true
     file = log_slice.find do |line|
       42 <=> line.to_i
     end
@@ -57,8 +102,8 @@ describe LogSlice do
   end
 
   it "#each_line_reverse" do
-    log_slice = LogSlice.new(range_to_file 1..10000)
-    log_slice.instance_eval { @line_cursor = @size }
+    log_slice = LogSlice.new enumerable_to_file(1..10000), :exact_match => true
+    log_slice.instance_eval { @line_cursor = @file.stat.size }
     lines = []
     file = log_slice.send(:each_line_reverse) do |line|
       lines << line.strip.to_i
@@ -67,8 +112,8 @@ describe LogSlice do
   end
 
   it "#each_line_reverse when file is empty" do
-    log_slice = LogSlice.new(string_to_file "")
-    log_slice.instance_eval { @line_cursor = @size }
+    log_slice = LogSlice.new string_to_file(""), :exact_match => true
+    log_slice.instance_eval { @line_cursor = @file.stat.size }
     lines = []
     file = log_slice.send(:each_line_reverse) do |line|
       lines << line.strip.to_i
@@ -77,8 +122,8 @@ describe LogSlice do
   end
 
   it "#each_line_reverse when file has single newline char" do
-    log_slice = LogSlice.new(string_to_file "\n")
-    log_slice.instance_eval { @line_cursor = @size }
+    log_slice = LogSlice.new string_to_file("\n"), :exact_match => true
+    log_slice.instance_eval { @line_cursor = @file.stat.size }
     lines = []
     file = log_slice.send(:each_line_reverse) do |line|
       lines << line.strip.to_i

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: log_slice
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 15
   prerelease: 
   segments: 
   - 0
-  - 1
-  version: "0.1"
+  - 2
+  version: "0.2"
 platform: ruby
 authors: 
 - Joel Plane
@@ -44,6 +44,7 @@ files:
 - Gemfile
 - README.md
 - lib/log_slice.rb
+- lib/log_slice/search_boundary.rb
 - log_slice.gemspec
 - spec/helper.rb
 - spec/log_slice_spec.rb