net-http-follow_tail 0.0.1

data/.rspec

@@ -0,0 +1 @@
+-f doc

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gem 'exponential-backoff'
+gem 'reindeer'
+
+group :test do
+  gem 'rake'
+  gem 'rspec'
+  gem 'webmock'
+end
+
+gem 'pry', group: 'development'

data/Gemfile.lock

@@ -0,0 +1,38 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.4)
+    coderay (1.0.7)
+    crack (0.3.2)
+    diff-lcs (1.2.3)
+    exponential-backoff (0.0.2)
+    method_source (0.8)
+    pry (0.9.12.1)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rake (10.0.4)
+    reindeer (0.0.1)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    slop (3.4.4)
+    webmock (1.11.0)
+      addressable (>= 2.2.7)
+      crack (>= 0.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  exponential-backoff
+  pry
+  rake
+  reindeer
+  rspec
+  webmock

data/README.md

@@ -0,0 +1,45 @@
+# Net::HTTP::FollowTail
+
+Fulfils the same role as `tail -f` but for files over HTTP. That is to
+say if you have log files (e.g IRC logs) available at a URL you could
+follow them with this module.
+
+# Usage
+
+```ruby
+require 'net/http/follow_tail'
+
+Net::HTTP::FollowTail.follow(uri: 'http://example.com/irc.log') do |result, tailer|
+  puts "Someone on IRC said: ", result.content
+end
+```
+
+# Interface
+
+If you're desiring of a URI's tail then the simplest way of using this
+module is to use the `follow` class method on `Net::HTTP::FollowTail`.
+It's first argument should be a hash, or array of hashes, containing
+at least `uri` key with a value that's either a `URI::HTTP` instance
+or something that would `URI.parse` to one. It also expects a block
+which gets executed whenever new data appears at the tail at any of
+the URIs. That's demonstrated in the *Usage* example above.
+
+Other data that can be passed in the hash argument(s) are:
+
+- `wait_in_seconds`: How long to wait in seconds between polls.
+- `offset`: An offset in `Fixnum` bytes to start at.
+- `max_retries`: The number of times to retry in the face of failure
+  before giving up.
+- `always_callback`: A boolean indicating that the callback should be
+  called even the tail request wasn't successful.
+
+The callback is called with two arguments - a
+`Net::HTTP::FollowTail::Result` instance and a
+`Net::HTTP::FollowTail::Tailer` instance respectively. The former
+exposing the result of the most recent tail request at the latter the
+current tailing state. By default it is only called when the tail
+request was successful.
+
+# Author
+
+Dan Brook `<dan@broquaint.com>`

data/Rakefile

@@ -0,0 +1,7 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |config|
+#  config.rcov = true
+end
+
+task :default => :spec

data/lib/net/http/follow_tail.rb

@@ -0,0 +1,156 @@
+require 'net/http'
+require 'uri'
+require 'exponential_backoff'
+require 'reindeer'
+
+class Net::HTTP::FollowTail
+  class Result < Reindeer
+    has :state,    is_a: Symbol
+    has :method,   is_a: Symbol
+    has :response, is_a: Net::HTTPSuccess
+
+    def has_response?
+      not @response.nil?
+    end
+
+    def is_success?
+      @state == :success
+    end
+    def is_error?
+      @state == :error
+    end
+
+    def content
+      response.body
+    end
+  end
+
+  class Tailer < Reindeer
+    has :uri,                 required: true
+    has :offset,              is_a: Fixnum, default: -> { 0 }
+    has :wait_in_seconds,     is_a: Fixnum, default: -> { 60 }
+    has :exponential_backoff, lazy_build: true
+    has :max_retries,         is_a: Fixnum, default: -> { 5 }
+    has :retries_so_far,      is_a: Fixnum, default: -> { 0 }
+    has :still_following,     is: :rw,      default: -> { true }
+
+    def build(opts)
+      @uri = opts[:uri].kind_of?(URI::HTTP) ? opts[:uri] : URI.parse(opts[:uri])
+    end
+
+    def still_following?
+      @still_following
+    end
+
+    # This and regular_wait need new names!
+    def error_wait
+      if exponential_backoff.length > 1
+        exponential_backoff.shift
+      else
+        exponential_backoff.first
+      end
+    end
+    def regular_wait
+      @exponential_backoff = get_backoff_list
+      @retries_so_far      = 0
+      wait_in_seconds
+    end
+
+    def update_offset(offset_increment)
+      @offset += offset_increment.to_i
+    end
+
+    def head_request
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.request( Net::HTTP::Head.new(uri.to_s) )
+    end
+
+    def get_request(size_now)
+      http = Net::HTTP.new(uri.host, uri.port)
+      req  = Net::HTTP::Get.new(uri.to_s)
+      req.initialize_http_header('Range' => "bytes=#{offset}-#{size_now}")
+      http.request(req)
+    end
+
+    def tail
+      @retries_so_far += 1
+
+      begin
+        head_response = head_request
+      rescue Timeout::Error, SocketError, EOFError, Errno::ETIMEDOUT
+        return Result.new(state: :error, method: :head)
+      end
+
+      # TODO invoke head_response.value to check for non 200s.
+
+      size_now = head_response.content_length
+      if size_now == offset
+        return Result.new(state: :no_change, method: :head, response: head_response)
+      end
+
+      begin
+        get_response = get_request(size_now)
+      rescue Timeout::Error, SocketError, EOFError
+        return Result.new(state: :error, method: :get)
+      end
+
+      update_offset get_response.content_length
+
+      # yield get_response, offset_for(uri)
+      return Result.new(state: :success, method: :get, response: get_response)
+    end
+
+    private
+
+    def get_backoff_list
+      ExponentialBackoff.new(
+        wait_in_seconds, wait_in_seconds ** 2
+      ).intervals_for(0 .. max_retries)
+    end
+
+    def build_exponential_backoff
+      get_backoff_list
+    end
+  end
+
+  def self.follow(opts, &block)
+    tailers = normalize_options(opts).collect do |o|
+      {t: Tailer.new(o), ac: o[:always_callback]}
+    end
+
+    while tailers.any? {|h| h[:t].still_following?}
+      for tailer in tailers.select{|h| h[:t].still_following?}
+        get_tail tailer[:t], tailer[:ac], block
+      end
+    end
+  end
+
+  def self.normalize_options(opts)
+    return [opts] if opts.is_a?(Hash)
+
+    raise ArgumentError, "Expected a Hash or Array not a #{opts}" unless opts.is_a? Array
+
+    opts
+  end
+
+  def self.get_tail(tailer, always_callback, block)
+    result = tailer.tail
+
+    while result.is_error?
+      block.call(result, tailer) if always_callback
+      return unless tailer.still_following?
+
+      if tailer.retries_so_far >= tailer.max_retries
+        # Would throw an exception but that breaks out of the #follow loop too.
+        tailer.still_following = false
+        return
+      end
+      sleep tailer.error_wait
+      result = tailer.tail
+    end
+
+    block.call(result, tailer) if result.is_success? or always_callback
+
+    sleep tailer.regular_wait
+  end
+end

data/spec/follow_tail_spec.rb

@@ -0,0 +1,95 @@
+require 'net/http/follow_tail'
+require 'spec_helper'
+
+describe Net::HTTP::FollowTail do
+  # Not sure if this is appropriate but it does the job.
+  before(:each, simple_stub_request: true) do
+    stub_request(:head, 'example.com')
+      .to_return(headers: { 'Content-Length' => 321 })
+    stub_request(:get, 'example.com')
+      .with(headers: {'Range' => 'bytes=0-321'})
+      .to_return(headers: { 'Content-Length' => 321 })
+    Net::HTTP::FollowTail.should_receive(:sleep).with(60)
+  end
+  
+  describe '#follow' do
+    it 'calls a block with a result and a tailer', simple_stub_request: true do
+      Net::HTTP::FollowTail.follow(uri: 'http://example.com/') do |result, tailer|
+        expect(result).to be_an_instance_of(Net::HTTP::FollowTail::Result)
+        expect(result.is_success?).to be_true
+        tailer.still_following = false
+      end
+    end
+
+    it 'should accept an Array of options', simple_stub_request: true do
+      # Would use multiple items but failing to stub sleep correctly :/
+      opts = [{uri: 'http://example.com/'}]
+      Net::HTTP::FollowTail.follow(opts) do |result, tailer|
+        expect(result).to be_an_instance_of(Net::HTTP::FollowTail::Result)
+        expect(result.is_success?).to be_true
+        tailer.still_following = false
+      end
+    end
+
+    it 'raises an error for weird input' do
+      expect {
+        Net::HTTP::FollowTail.follow(:boom)
+      }.to raise_error(ArgumentError)
+    end
+  end
+
+  describe '#get_tail' do
+    it 'should make a request and call a block', simple_stub_request: true do
+      a_tailer = Net::HTTP::FollowTail::Tailer.new(uri: 'http://example.com')
+      Net::HTTP::FollowTail.get_tail(a_tailer, false, Proc.new{ |result, tailer|
+        expect(tailer).to eql(a_tailer)
+        expect(result.is_success?).to be_true
+      })
+    end
+
+    # Incidentally tests always_callback which was introduced to allow
+    # this testing to work.
+    it 'should retry when an error is received' do
+      stub_request(:head, 'example.com').to_timeout
+      Net::HTTP::FollowTail.stub(:sleep) { }
+
+      # A bit gross but effective.
+      call_count = 0
+
+      a_tailer = Net::HTTP::FollowTail::Tailer.new(uri: 'http://example.com')
+      Net::HTTP::FollowTail.get_tail(a_tailer, true, Proc.new{ |result, tailer|
+          if call_count == 1
+            expect(result.is_success?).to be_true
+            call_count += 1
+          else
+            expect(result.is_success?).to be_false
+            call_count += 1
+
+            stub_request(:head, 'example.com')
+              .to_return(headers: { 'Content-Length' => 321 })
+            stub_request(:get, 'example.com')
+              .with(headers: {'Range' => 'bytes=0-321'})
+              .to_return(headers: { 'Content-Length' => 321 })
+          end
+        })
+      expect(call_count).to eq(2)
+    end
+
+    it 'exit loop when max_retries is hit' do
+      stub_request(:head, 'example.com').to_timeout
+      Net::HTTP::FollowTail.stub(:sleep) { }
+
+      a_tailer = Net::HTTP::FollowTail::Tailer.new(
+        uri: 'http://example.com',
+        max_retries: 2
+      )
+
+      Net::HTTP::FollowTail.get_tail(a_tailer, true, Proc.new{ |result, tailer|
+          expect(result.is_success?).to be_false
+        })
+
+      expect(a_tailer.still_following?).to be_false
+      expect(a_tailer.retries_so_far).to eq(2)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'webmock/rspec'
+
+RSpec.configure do |config|
+  # No monkey patching please.
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

data/spec/tailer_spec.rb

@@ -0,0 +1,182 @@
+require 'net/http/follow_tail'
+require 'spec_helper'
+
+describe Net::HTTP::FollowTail::Tailer do
+  let(:example_uri)   { 'http://example.com/' }
+  let(:example_host)  { 'example.com' }
+  let(:simple_tailer) { Net::HTTP::FollowTail::Tailer.new(uri: example_uri) }
+  let(:default_wait)  { 60 }
+
+  describe '#new' do
+    it 'has sensible defaults' do
+      tailer = simple_tailer
+      expect(tailer.uri).to be_an_instance_of(URI::HTTP)
+      expect(tailer.offset).to eq(0)
+      expect(tailer.wait_in_seconds).to eq(default_wait)
+      expect(tailer.max_retries).to eq(5)
+      expect(tailer.retries_so_far).to eq(0)
+    end
+
+    it 'requires a uri to be specified' do
+      expect {
+        Net::HTTP::FollowTail::Tailer.new
+      }.to raise_error(Reindeer::Meta::Attribute::AttributeError)
+    end
+
+    it 'accepts a URI instance' do
+      uri = URI.parse(example_uri)
+      tailer = Net::HTTP::FollowTail::Tailer.new(uri: uri)
+      expect(tailer.uri).to eql(uri)
+    end
+
+    it 'accepts offset, wait & max_retries options' do
+      tailer = Net::HTTP::FollowTail::Tailer.new(
+        uri: example_uri,
+        offset: 1234,
+        wait_in_seconds: 20,
+        max_retries: 2,
+      )
+      expect(tailer.uri).to be_an_instance_of(URI::HTTP)
+      expect(tailer.offset).to eq(1234)
+      expect(tailer.wait_in_seconds).to eq(20)
+      expect(tailer.max_retries).to eq(2)
+    end
+  end
+
+  describe '.regular_wait' do
+    it 'always returns wait_in_seconds' do
+      tailer = Net::HTTP::FollowTail::Tailer.new(
+        uri: example_uri,
+        wait_in_seconds: 66,
+      )
+
+      expect(tailer.wait_in_seconds).to eq(66)
+    end
+  end
+
+  describe '.error_wait' do
+    it 'munges state appropriately' do
+      tailer = simple_tailer
+
+      expect(tailer.wait_in_seconds).to eq(default_wait)
+      expect(tailer.error_wait).to eq(default_wait)
+      expect(tailer.error_wait).to be > default_wait
+    end
+  end
+
+  describe '.head_request' do
+    it 'provides a response' do
+      stub_request :head, "example.com"
+      expect(simple_tailer.head_request).to be_an_instance_of(Net::HTTPOK)
+    end
+  end
+
+  describe '.get_request' do
+    it 'should make ranged requests' do
+      stub_request(:get, example_host).with(headers: {'Range' => 'bytes=0-200'})
+      expect(simple_tailer.get_request(200)).to be_an_instance_of(Net::HTTPOK)
+    end
+    
+    it 'makes range requests against an offset' do
+      tailer = simple_tailer
+      tailer.update_offset 200
+      stub_request(:get, example_host).with(headers: {'Range' => 'bytes=200-400'})
+      expect(tailer.get_request(400)).to be_an_instance_of(Net::HTTPOK)
+    end
+  end
+
+  describe '.tail' do
+    it 'returns a result object' do
+      stub_request(:head, example_host)
+        .to_return(headers: { 'Content-Length' => 321 })
+      stub_request(:get, example_host)
+        .with(headers: {'Range' => 'bytes=0-321'})
+        .to_return(headers: { 'Content-Length' => 321 })
+
+      result = simple_tailer.tail
+      expect(result).to be_an_instance_of(Net::HTTP::FollowTail::Result)
+      expect(result.is_success?).to be_true
+    end
+
+    it 'updates offset state with multiple tail calls' do
+      stub_request(:head, example_host)
+        .to_return(headers: { 'Content-Length' => 5 })
+      stub_request(:get, example_host)
+        .with(headers: {'Range' => 'bytes=0-5'})
+        .to_return(headers: { 'Content-Length' => 5 })
+
+      tailer = simple_tailer
+      tailer.tail
+
+      stub_request(:head, example_host)
+        .to_return(headers: { 'Content-Length' => 10 })
+      stub_request(:get, example_host)
+        .with(headers: {'Range' => 'bytes=5-10'})
+        .to_return(headers: { 'Content-Length' => 5 })
+
+      result = tailer.tail
+      expect(result).to be_an_instance_of(Net::HTTP::FollowTail::Result)
+      expect(result.is_success?).to be_true
+      expect(result.method).to be(:get)
+      expect(tailer.offset).to eq(10)
+
+      stub_request(:head, example_host)
+        .to_return(headers: { 'Content-Length' => 15 })
+      stub_request(:get, example_host)
+        .with(headers: {'Range' => 'bytes=10-15'})
+        .to_return(headers: { 'Content-Length' => 5 })
+
+      result = tailer.tail
+      expect(tailer.offset).to eq(15)
+    end
+
+    it 'correctly updates against existing offset' do
+      stub_request(:head, example_host)
+        .to_return(headers: { 'Content-Length' => 66 })
+      stub_request(:get, example_host)
+        .with(headers: {'Range' => 'bytes=50-66'})
+        .to_return(headers: { 'Content-Length' => 16 })
+
+      tailer = Net::HTTP::FollowTail::Tailer.new(uri: example_uri, offset: 50)
+      tailer.tail
+      expect(tailer.offset).to eq(66)
+    end
+
+    it 'to handle HEAD errors' do
+      stub_request(:head, example_host).to_timeout
+
+      result = simple_tailer.tail
+      expect(result.is_error?).to be_true
+      expect(result.method).to be(:head)
+      expect(result.has_response?).to be_false
+    end
+
+    it 'to handle GET errors' do
+      stub_request(:head, example_host)
+        .to_return(headers: { 'Content-Length' => 10 })
+      stub_request(:get, example_host).to_timeout
+
+      result = simple_tailer.tail
+      expect(result.is_error?).to be_true
+      expect(result.method).to be(:get)
+      expect(result.has_response?).to be_false
+    end
+
+    it 'returns a no change Result when appropriate' do
+      stub_request(:head, example_host)
+        .to_return(headers: { 'Content-Length' => 25 })
+
+      tailer = Net::HTTP::FollowTail::Tailer.new(
+        uri: example_uri,
+        offset: 25
+      )
+
+      result = tailer.tail
+      expect(result.state).to eq(:no_change)
+      expect(result.method).to eq(:head)
+      expect(result.has_response?).to be_true
+      expect(result.is_success?).to be_false
+      expect(result.is_error?).to be_false
+    end
+  end
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: net-http-follow_tail
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Dan Brook
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: exponential-backoff
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Watch multiple URIs for appended content e.g log files
+email: dan@broquaint.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .rspec
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- lib/net/http/follow_tail.rb
+- spec/follow_tail_spec.rb
+- spec/spec_helper.rb
+- spec/tailer_spec.rb
+homepage: http://github.com/broquaint/net-http-follow_tail
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Like tail -f for the web
+test_files:
+- spec/follow_tail_spec.rb
+- spec/spec_helper.rb
+- spec/tailer_spec.rb