papermill-agent 0.0.1

data/Gemfile

@@ -0,0 +1,11 @@
+source 'http://rubygems.org'
+
+gem 'rest-client', '~>1.6.1'
+
+group :test do
+  gem 'fakeweb',  '>=1.3.0'
+  gem 'autotest', '>=4.2.2'
+  gem 'sinatra',  '1.1.0'
+  gem 'rails',    '~> 3.0.0'
+  gem 'rspec',    '>=2.0.1'
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Alex Sharp
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,18 @@
+
+WARNING: This is pre-alpha software. Use at your own risk until 1.0.
+
+The papermill agent parses responses from your web application.
+
+## Usage Instructions
+
+Papermill works via a middleware called the Collector. 
+
+#### In Rails
+    # in config/environment.rb
+    config.middleware.use 'Papermill::Collector'
+
+#### In Sinatra/rack
+    # in config.ru
+    use Papermill::Collector
+
+

data/Rakefile

@@ -0,0 +1,5 @@
+require 'rspec/core/rake_task'
+task :default => :spec
+
+Rspec::Core::RakeTask.new do |t|
+end

data/config/papermill.sample.yml

@@ -0,0 +1 @@
+token: api-key

data/doc/ideas.md

@@ -0,0 +1,5 @@
+It gathers the status and headers from each and every response rendered by 
+your app. Additionally, you can configure the agent to save other types
+of arbitrary data:
+
+Papermill::Collector.new :fields => {:email => lambda { User.find('session_key') }}

data/lib/papermill-agent.rb

@@ -0,0 +1,17 @@
+
+$:.unshift File.dirname(File.expand_path(__FILE__))
+
+module Papermill
+  autoload :Agent,          'papermill-agent/agent'
+  autoload :Collector,      'papermill-agent/rack/collector'
+  autoload :ResponseParser, 'papermill-agent/response_parser'
+  autoload :Storage,        'papermill-agent/storage'
+
+  module ResponseAdapters
+    autoload :Base,    'papermill-agent/response_adapters/base'
+    autoload :Rails,   'papermill-agent/response_adapters/rails'
+    autoload :Sinatra, 'papermill-agent/response_adapters/sinatra'
+  end
+end
+
+Papermill::Agent.instance.start

data/lib/papermill-agent/agent.rb

@@ -0,0 +1,76 @@
+require 'singleton'
+require 'timeout'
+require 'json'
+require 'restclient'
+require 'yaml'
+
+module Papermill
+
+  class Agent
+    include Singleton
+
+    # papermill endpoint which will receive client requests
+    API_ENDPOINT = 'http://api.papermillapp.com'
+    
+    # send new request data every 10 seconds
+    UPDATE_INTERVAL = 10
+
+    attr_reader :last_sent, :mutex, :config
+
+    def start
+      @last_sent = Time.now
+      @mutex     = Mutex.new
+      @config    = YAML.load_file('config/papermill.yml')
+      Thread.abort_on_exception = true
+
+      @worker_thread = Thread.new do
+        loop do
+          if time_since_last_sent > UPDATE_INTERVAL
+            p "sending #{Storage.store.count} requests to papermill..."
+            send_data_to_papermill
+            @last_sent = Time.now
+          end
+          sleep_time = seconds_until_next_run
+          sleep sleep_time if sleep_time > 0
+        end
+      end
+    end
+
+    def time_since_last_sent
+      Time.now - last_sent
+    end
+
+    def seconds_until_next_run
+      UPDATE_INTERVAL - time_since_last_sent
+    end
+
+    def send_data_to_papermill
+      begin
+        Timeout.timeout(9) { do_request unless Storage.store.empty? }
+      rescue Timeout::Error
+        p 'timeout error'
+      end
+    end
+
+    def do_request
+      begin
+        RestClient.post API_ENDPOINT, { :api_key => config['token'], :payload => jsonify_payload }
+      rescue RestClient::Exception, Errno::ECONNREFUSED
+        p 'transmission error ocurred...'
+      end
+    end
+
+    # Return a json version of the storage array.
+    # Also, we'll clear out the storage here
+    #
+    # TODO: move this to the Storage class.
+    def jsonify_payload
+      mutex.synchronize do
+        json_data = JSON.generate(Storage.store.flatten)
+        Storage.clear
+        return json_data
+      end
+    end
+  end
+
+end

data/lib/papermill-agent/rack/collector.rb

@@ -0,0 +1,20 @@
+
+module Papermill
+
+  # The Collector class is the middleware component of papermill.
+  # To use it with a rails app, you must add the following to
+  # your environment.rb file:
+  # config.middleware.use Papermill::Collector
+  class Collector
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      status, headers, response = @app.call(env)
+      ResponseParser.parse(status, headers, response, env)
+      [status, headers, response]
+    end
+  end
+
+end

data/lib/papermill-agent/response_adapters/base.rb

@@ -0,0 +1,27 @@
+
+module Papermill
+  module ResponseAdapters
+
+    class Base
+      attr_reader :status, :headers, :response, :env
+      def initialize(*args)
+        @status, @headers, @response, @env = args.flatten
+      end
+
+      def parse
+        parsed_response = { :headers => headers.merge(env), :status => status }
+        # if @status != 304 && @response #&& !@response.body.is_a?(Proc)
+        #   parsed_response.merge!(prepare_extra_fields)
+        # end
+        parsed_response.merge!(additional_response_data)
+        Papermill::Storage.store << parsed_response
+      end
+
+      private
+      def additional_response_data
+        {:request_time => Time.now}
+      end
+    end
+
+  end
+end

data/lib/papermill-agent/response_adapters/rails.rb

@@ -0,0 +1,15 @@
+
+module Papermill
+  module ResponseAdapters
+
+    class Rails < Base
+
+    #   def format_controller_action
+    #     params = @response.request.path_parameters
+    #     "#{params['controller']}##{params['action']}"
+    #   end
+    # end
+    end
+
+  end
+end

data/lib/papermill-agent/response_adapters/sinatra.rb

@@ -0,0 +1,9 @@
+
+module Papermill
+  module ResponseAdapters
+
+    class Sinatra < Base
+    end
+
+  end
+end

data/lib/papermill-agent/response_parser.rb

@@ -0,0 +1,24 @@
+
+module Papermill
+
+  class ResponseParser
+    # ENV_CAPTURE_FIELDS = [
+    #   'PATH_INFO', 'QUERY_STRING', 'REMOTE_ADDR', 'REMOTE_HOST', 'REQUEST_METHOD', 
+    #   'REQUEST_URI', 'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_SOFTWARE', 'HTTP_HOST', 
+    #   'HTTP_ACCEPT', 'HTTP_USER_AGENT', 'REQUEST_PATH'
+    # ]
+
+    def self.parse(status, headers, response, env = {})
+      klass = if defined?(Rails)
+        ResponseAdapters::Rails
+      elsif defined?(Sinatra)
+        ResponseAdapters::Sinatra
+      else
+        ResponseAdapters::Base
+      end
+
+      klass.new(status, headers, response, env).parse
+    end
+  end
+
+end

data/lib/papermill-agent/storage.rb

@@ -0,0 +1,26 @@
+require 'singleton'
+
+module Papermill
+
+  # The Storage class inherits from Array, and thus can be used in a very simple
+  # and predictable way. This is used by the Collector to save requests until
+  # they are ready to be sent to the remote server, which is handled by the
+  # Agent class.
+  class Storage < Array
+    include Singleton
+
+    class << self
+      # TODO: We need a mutex sync around adding items to storage
+
+      def store
+        instance
+      end
+
+      def clear
+        store.clear
+      end
+    end
+  end
+
+end
+

data/spec/agent_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+module Papermill
+
+  describe 'the agent' do
+    it 'has a mutex' do
+      Agent.instance.mutex.should be_instance_of Mutex
+    end
+
+    context 'when a new process is started' do
+      context 'when an agent process does not already exist' do
+        it 'should create a new agent process' do
+          Papermill::Agent.instance.should be_instance_of Papermill::Agent
+        end
+      end
+
+      context 'when an agent process already exists' do
+        it 'should only allow one agent instance to exist' do
+          Papermill::Agent.instance.should eql Papermill::Agent.instance
+        end
+      end
+    end
+
+    context 'when attempting to initialize the agent directly' do
+      it 'raises a NoMethodError for attempting to initialize a singleton' do
+        lambda {
+          Papermill::Agent.initialize
+        }.should raise_error(NoMethodError, /private method `initialize' called/)
+      end
+    end
+
+  end
+
+  describe 'the api endpoint' do
+    subject { Agent::API_ENDPOINT }
+    it { should == 'http://api.papermillapp.com' }
+  end
+
+  describe 'determining the time since the last time data was sent to papermill' do
+    before do
+      Agent.instance.stub!(:last_sent => Time.mktime(2010, 11, 11, 0, 0, 0))
+      Time.stub!(:now => Time.mktime(2010, 11, 11, 0, 0, 9))
+    end
+
+    context 'time since the last run' do
+      it 'subtracts the current time from @last_sent' do
+        Agent.instance.time_since_last_sent.should == 9
+      end
+    end
+
+    context 'the time until the next run' do
+      it 'time until next = interval - time elapsed since last' do
+        Agent.instance.seconds_until_next_run.should == 1
+      end
+    end
+
+  end
+
+  describe 'configuration' do
+    it 'provides access to the api token' do
+      Agent.instance.config['token'].should == 'api-key'
+    end
+  end
+
+  describe 'sending data to papermill' do
+    before do
+      Storage.store << [{:headers => {'Content-Type' => 'text/html'}, :status => 200}]
+    end
+
+    it 'jsonifies the payload data' do
+      Agent.instance.jsonify_payload.should == '[{"headers":{"Content-Type":"text/html"},"status":200}]'
+    end
+
+    it 'empties the store' do
+      Agent.instance.jsonify_payload
+      Storage.store.should be_empty
+    end
+
+    it 'sends a request to the papermill api endpoint' do
+      RestClient.should_receive(:post).with(Agent::API_ENDPOINT, :api_key => 'api-key', :payload => '[{"headers":{"Content-Type":"text/html"},"status":200}]')
+      Agent.instance.send_data_to_papermill
+    end
+
+    it 'should not send anything if no requests have been stored' do
+      Storage.clear
+      Agent.should_not_receive(:do_request)
+      Agent.instance.send_data_to_papermill
+    end
+  end
+
+end
+

data/spec/rack/collector_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+require 'sinatra'
+
+
+module Papermill
+
+  describe 'collecting statistics via the middleware layer' do
+    before do
+      @app = Sinatra.new Sinatra::Application do
+        get '/' do
+          '<html><body>Hello, world!</body></html>'
+        end
+      end
+      @app.use Papermill::Collector
+
+      request_headers = {'HTTP_USER_AGENT' => 'agent', 'REMOTE_ADDR' => '0.0.0.0', 
+        'QUERY_STRING' => 'q=search',
+        'SCRIPT_NAME'  => 'my-script',
+        'HTTP_ACCEPT'  => 'text/html, application/json',
+        'HTTP_HOST'    => 'localhost',
+        'REQUEST_URI'  => 'i-requested-this',
+        'REMOTE_HOST'  => '123.345.34.2',
+        'SERVER_SOFTWARE' => 'apache'
+      }
+
+      Time.stub(:now => Time.utc(2010, 01, 01))
+      @status, @headers, @body = @app.call(Rack::MockRequest.env_for('/', request_headers))
+    end
+    
+    def last_store
+      Storage.store.last
+    end
+
+    def headers
+      last_store[:headers]
+    end
+
+    it 'records the query string' do
+      last_store[:status].should == 200
+    end
+
+    it 'records request duration'
+
+    it 'records the request time' do
+      last_store[:request_time].should == Time.utc(2010, 01, 01)
+    end
+
+    it 'records the path info' do
+      headers['PATH_INFO'].should == '/'
+    end
+
+    it 'records the request uri' do
+      headers['REQUEST_URI'].should == 'i-requested-this'
+    end
+
+    it 'records the query remote addr' do
+      headers['REMOTE_ADDR'].should == '0.0.0.0'
+    end
+
+    it 'records the server name' do
+      headers['SERVER_NAME'].should == 'example.org'
+    end
+
+    it 'records the query string' do
+      headers['QUERY_STRING'].should == 'q=search'
+    end
+
+    it 'records the request method' do
+      headers['REQUEST_METHOD'].should == 'GET'
+    end
+
+    it 'records the script name' do
+      headers['SCRIPT_NAME'].should == 'my-script'
+    end
+
+    it 'records the user agent' do
+      headers['HTTP_USER_AGENT'].should == 'agent'
+    end
+
+    it 'records the http accept types' do
+      headers['HTTP_ACCEPT'].should == 'text/html, application/json'
+    end
+
+    it 'records the http host' do
+      headers['HTTP_HOST'].should == 'localhost'
+    end
+
+    it 'records the server software' do
+      headers['SERVER_SOFTWARE'].should == 'apache'
+    end
+
+    it 'records the remote host' do
+      headers['REMOTE_HOST'].should == '123.345.34.2'
+    end
+
+    it 'records the url scheme' do
+      headers['rack.url_scheme'].should == 'http'
+    end
+
+    context 'for a 304 response' do
+      it 'only records certain things'
+    end
+
+    context 'for a streamed file response' do
+      it 'only records certain things'
+    end
+  end
+
+end

data/spec/response_parser_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+module Papermill
+
+  describe '.parse' do
+    before do
+      ResponseAdapters::Base.stub(:new).and_return(mock('object', :parse => nil))
+    end
+    context 'in a normal rack app' do
+      it 'uses the base adapter' do
+        ResponseAdapters::Base.should_receive(:new).and_return(mock('object', :parse => nil))
+        ResponseParser.parse(200, {}, [], {})
+      end
+    end
+
+    context 'in a rails app' do
+      class Rails
+      end
+
+      it 'uses the rails adapter' do
+        ResponseAdapters::Rails.should_receive(:new).and_return(mock('object', :parse => nil))
+        ResponseParser.parse(200, {}, [], {})
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+ENV['RACK_ENV'] = 'test'
+
+require 'rubygems'
+require 'bundler'
+
+Bundler.require :default, :test
+
+$:.unshift File.dirname(File.expand_path(__FILE__))
+$:.unshift File.dirname(File.expand_path(__FILE__) + '/../lib')
+
+require 'papermill-agent'
+require 'fakeweb'
+
+FakeWeb.allow_net_connect = false
+
+RSpec.configure do |config|
+  config.before(:each) do
+    # clear the storage before each example
+    Papermill::Storage.clear
+  end
+end

data/spec/storage_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+module Papermill
+
+  describe 'adding an item to the local data storage' do
+    it 'acts like an array' do
+      Storage.store.should respond_to(:<<)
+    end
+
+    it 'stores data' do
+      Storage.store << 'some data'
+      Storage.store.should include 'some data'
+    end
+  end
+
+  describe 'the store' do
+    it 'wraps the singleton instance' do
+      Storage.store == Storage.instance
+    end
+  end
+
+  describe 'emptying the cache' do
+    before { Storage.clear }
+
+    it 'clears out the storage' do
+      Storage.store << 'stuff'
+      Storage.store.should == ['stuff']
+
+      Storage.clear
+      Storage.store.should == []
+    end
+  end
+
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification 
+name: papermill-agent
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Alex Sharp
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-19 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 6
+        - 1
+        version: 1.6.1
+  type: :runtime
+  version_requirements: *id001
+description: The client agent for papermillapp.com
+email: 
+- ajsharp@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+- doc/ideas.md
+files: 
+- lib/papermill-agent/agent.rb
+- lib/papermill-agent/rack/collector.rb
+- lib/papermill-agent/response_adapters/base.rb
+- lib/papermill-agent/response_adapters/rails.rb
+- lib/papermill-agent/response_adapters/sinatra.rb
+- lib/papermill-agent/response_parser.rb
+- lib/papermill-agent/storage.rb
+- lib/papermill-agent.rb
+- LICENSE
+- README.md
+- config/papermill.sample.yml
+- Gemfile
+- Rakefile
+- doc/ideas.md
+- spec/agent_spec.rb
+- spec/integration/rails2_spec.rb
+- spec/integration/rails3_spec.rb
+- spec/rack/collector_spec.rb
+- spec/response_parser_spec.rb
+- spec/spec_helper.rb
+- spec/storage_spec.rb
+has_rdoc: true
+homepage: http://github.com/ajsharp/papermill-agent
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: The client agent for papermillapp.com
+test_files: 
+- spec/agent_spec.rb
+- spec/integration/rails2_spec.rb
+- spec/integration/rails3_spec.rb
+- spec/rack/collector_spec.rb
+- spec/response_parser_spec.rb
+- spec/spec_helper.rb
+- spec/storage_spec.rb