papermill-agent 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/Gemfile ADDED
@@ -0,0 +1,11 @@
1
+ source 'http://rubygems.org'
2
+
3
+ gem 'rest-client', '~>1.6.1'
4
+
5
+ group :test do
6
+ gem 'fakeweb', '>=1.3.0'
7
+ gem 'autotest', '>=4.2.2'
8
+ gem 'sinatra', '1.1.0'
9
+ gem 'rails', '~> 3.0.0'
10
+ gem 'rspec', '>=2.0.1'
11
+ end
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2010 Alex Sharp
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,18 @@
1
+
2
+ WARNING: This is pre-alpha software. Use at your own risk until 1.0.
3
+
4
+ The papermill agent parses responses from your web application.
5
+
6
+ ## Usage Instructions
7
+
8
+ Papermill works via a middleware called the Collector.
9
+
10
+ #### In Rails
11
+ # in config/environment.rb
12
+ config.middleware.use 'Papermill::Collector'
13
+
14
+ #### In Sinatra/rack
15
+ # in config.ru
16
+ use Papermill::Collector
17
+
18
+
data/Rakefile ADDED
@@ -0,0 +1,5 @@
1
+ require 'rspec/core/rake_task'
2
+ task :default => :spec
3
+
4
+ Rspec::Core::RakeTask.new do |t|
5
+ end
@@ -0,0 +1 @@
1
+ token: api-key
data/doc/ideas.md ADDED
@@ -0,0 +1,5 @@
1
+ It gathers the status and headers from each and every response rendered by
2
+ your app. Additionally, you can configure the agent to save other types
3
+ of arbitrary data:
4
+
5
+ Papermill::Collector.new :fields => {:email => lambda { User.find('session_key') }}
@@ -0,0 +1,17 @@
1
+
2
+ $:.unshift File.dirname(File.expand_path(__FILE__))
3
+
4
+ module Papermill
5
+ autoload :Agent, 'papermill-agent/agent'
6
+ autoload :Collector, 'papermill-agent/rack/collector'
7
+ autoload :ResponseParser, 'papermill-agent/response_parser'
8
+ autoload :Storage, 'papermill-agent/storage'
9
+
10
+ module ResponseAdapters
11
+ autoload :Base, 'papermill-agent/response_adapters/base'
12
+ autoload :Rails, 'papermill-agent/response_adapters/rails'
13
+ autoload :Sinatra, 'papermill-agent/response_adapters/sinatra'
14
+ end
15
+ end
16
+
17
+ Papermill::Agent.instance.start
@@ -0,0 +1,76 @@
1
+ require 'singleton'
2
+ require 'timeout'
3
+ require 'json'
4
+ require 'restclient'
5
+ require 'yaml'
6
+
7
+ module Papermill
8
+
9
+ class Agent
10
+ include Singleton
11
+
12
+ # papermill endpoint which will receive client requests
13
+ API_ENDPOINT = 'http://api.papermillapp.com'
14
+
15
+ # send new request data every 10 seconds
16
+ UPDATE_INTERVAL = 10
17
+
18
+ attr_reader :last_sent, :mutex, :config
19
+
20
+ def start
21
+ @last_sent = Time.now
22
+ @mutex = Mutex.new
23
+ @config = YAML.load_file('config/papermill.yml')
24
+ Thread.abort_on_exception = true
25
+
26
+ @worker_thread = Thread.new do
27
+ loop do
28
+ if time_since_last_sent > UPDATE_INTERVAL
29
+ p "sending #{Storage.store.count} requests to papermill..."
30
+ send_data_to_papermill
31
+ @last_sent = Time.now
32
+ end
33
+ sleep_time = seconds_until_next_run
34
+ sleep sleep_time if sleep_time > 0
35
+ end
36
+ end
37
+ end
38
+
39
+ def time_since_last_sent
40
+ Time.now - last_sent
41
+ end
42
+
43
+ def seconds_until_next_run
44
+ UPDATE_INTERVAL - time_since_last_sent
45
+ end
46
+
47
+ def send_data_to_papermill
48
+ begin
49
+ Timeout.timeout(9) { do_request unless Storage.store.empty? }
50
+ rescue Timeout::Error
51
+ p 'timeout error'
52
+ end
53
+ end
54
+
55
+ def do_request
56
+ begin
57
+ RestClient.post API_ENDPOINT, { :api_key => config['token'], :payload => jsonify_payload }
58
+ rescue RestClient::Exception, Errno::ECONNREFUSED
59
+ p 'transmission error ocurred...'
60
+ end
61
+ end
62
+
63
+ # Return a json version of the storage array.
64
+ # Also, we'll clear out the storage here
65
+ #
66
+ # TODO: move this to the Storage class.
67
+ def jsonify_payload
68
+ mutex.synchronize do
69
+ json_data = JSON.generate(Storage.store.flatten)
70
+ Storage.clear
71
+ return json_data
72
+ end
73
+ end
74
+ end
75
+
76
+ end
@@ -0,0 +1,20 @@
1
+
2
+ module Papermill
3
+
4
+ # The Collector class is the middleware component of papermill.
5
+ # To use it with a rails app, you must add the following to
6
+ # your environment.rb file:
7
+ # config.middleware.use Papermill::Collector
8
+ class Collector
9
+ def initialize(app)
10
+ @app = app
11
+ end
12
+
13
+ def call(env)
14
+ status, headers, response = @app.call(env)
15
+ ResponseParser.parse(status, headers, response, env)
16
+ [status, headers, response]
17
+ end
18
+ end
19
+
20
+ end
@@ -0,0 +1,27 @@
1
+
2
+ module Papermill
3
+ module ResponseAdapters
4
+
5
+ class Base
6
+ attr_reader :status, :headers, :response, :env
7
+ def initialize(*args)
8
+ @status, @headers, @response, @env = args.flatten
9
+ end
10
+
11
+ def parse
12
+ parsed_response = { :headers => headers.merge(env), :status => status }
13
+ # if @status != 304 && @response #&& !@response.body.is_a?(Proc)
14
+ # parsed_response.merge!(prepare_extra_fields)
15
+ # end
16
+ parsed_response.merge!(additional_response_data)
17
+ Papermill::Storage.store << parsed_response
18
+ end
19
+
20
+ private
21
+ def additional_response_data
22
+ {:request_time => Time.now}
23
+ end
24
+ end
25
+
26
+ end
27
+ end
@@ -0,0 +1,15 @@
1
+
2
+ module Papermill
3
+ module ResponseAdapters
4
+
5
+ class Rails < Base
6
+
7
+ # def format_controller_action
8
+ # params = @response.request.path_parameters
9
+ # "#{params['controller']}##{params['action']}"
10
+ # end
11
+ # end
12
+ end
13
+
14
+ end
15
+ end
@@ -0,0 +1,9 @@
1
+
2
+ module Papermill
3
+ module ResponseAdapters
4
+
5
+ class Sinatra < Base
6
+ end
7
+
8
+ end
9
+ end
@@ -0,0 +1,24 @@
1
+
2
+ module Papermill
3
+
4
+ class ResponseParser
5
+ # ENV_CAPTURE_FIELDS = [
6
+ # 'PATH_INFO', 'QUERY_STRING', 'REMOTE_ADDR', 'REMOTE_HOST', 'REQUEST_METHOD',
7
+ # 'REQUEST_URI', 'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_SOFTWARE', 'HTTP_HOST',
8
+ # 'HTTP_ACCEPT', 'HTTP_USER_AGENT', 'REQUEST_PATH'
9
+ # ]
10
+
11
+ def self.parse(status, headers, response, env = {})
12
+ klass = if defined?(Rails)
13
+ ResponseAdapters::Rails
14
+ elsif defined?(Sinatra)
15
+ ResponseAdapters::Sinatra
16
+ else
17
+ ResponseAdapters::Base
18
+ end
19
+
20
+ klass.new(status, headers, response, env).parse
21
+ end
22
+ end
23
+
24
+ end
@@ -0,0 +1,26 @@
1
+ require 'singleton'
2
+
3
+ module Papermill
4
+
5
+ # The Storage class inherits from Array, and thus can be used in a very simple
6
+ # and predictable way. This is used by the Collector to save requests until
7
+ # they are ready to be sent to the remote server, which is handled by the
8
+ # Agent class.
9
+ class Storage < Array
10
+ include Singleton
11
+
12
+ class << self
13
+ # TODO: We need a mutex sync around adding items to storage
14
+
15
+ def store
16
+ instance
17
+ end
18
+
19
+ def clear
20
+ store.clear
21
+ end
22
+ end
23
+ end
24
+
25
+ end
26
+
@@ -0,0 +1,92 @@
1
+ require 'spec_helper'
2
+
3
+ module Papermill
4
+
5
+ describe 'the agent' do
6
+ it 'has a mutex' do
7
+ Agent.instance.mutex.should be_instance_of Mutex
8
+ end
9
+
10
+ context 'when a new process is started' do
11
+ context 'when an agent process does not already exist' do
12
+ it 'should create a new agent process' do
13
+ Papermill::Agent.instance.should be_instance_of Papermill::Agent
14
+ end
15
+ end
16
+
17
+ context 'when an agent process already exists' do
18
+ it 'should only allow one agent instance to exist' do
19
+ Papermill::Agent.instance.should eql Papermill::Agent.instance
20
+ end
21
+ end
22
+ end
23
+
24
+ context 'when attempting to initialize the agent directly' do
25
+ it 'raises a NoMethodError for attempting to initialize a singleton' do
26
+ lambda {
27
+ Papermill::Agent.initialize
28
+ }.should raise_error(NoMethodError, /private method `initialize' called/)
29
+ end
30
+ end
31
+
32
+ end
33
+
34
+ describe 'the api endpoint' do
35
+ subject { Agent::API_ENDPOINT }
36
+ it { should == 'http://api.papermillapp.com' }
37
+ end
38
+
39
+ describe 'determining the time since the last time data was sent to papermill' do
40
+ before do
41
+ Agent.instance.stub!(:last_sent => Time.mktime(2010, 11, 11, 0, 0, 0))
42
+ Time.stub!(:now => Time.mktime(2010, 11, 11, 0, 0, 9))
43
+ end
44
+
45
+ context 'time since the last run' do
46
+ it 'subtracts the current time from @last_sent' do
47
+ Agent.instance.time_since_last_sent.should == 9
48
+ end
49
+ end
50
+
51
+ context 'the time until the next run' do
52
+ it 'time until next = interval - time elapsed since last' do
53
+ Agent.instance.seconds_until_next_run.should == 1
54
+ end
55
+ end
56
+
57
+ end
58
+
59
+ describe 'configuration' do
60
+ it 'provides access to the api token' do
61
+ Agent.instance.config['token'].should == 'api-key'
62
+ end
63
+ end
64
+
65
+ describe 'sending data to papermill' do
66
+ before do
67
+ Storage.store << [{:headers => {'Content-Type' => 'text/html'}, :status => 200}]
68
+ end
69
+
70
+ it 'jsonifies the payload data' do
71
+ Agent.instance.jsonify_payload.should == '[{"headers":{"Content-Type":"text/html"},"status":200}]'
72
+ end
73
+
74
+ it 'empties the store' do
75
+ Agent.instance.jsonify_payload
76
+ Storage.store.should be_empty
77
+ end
78
+
79
+ it 'sends a request to the papermill api endpoint' do
80
+ RestClient.should_receive(:post).with(Agent::API_ENDPOINT, :api_key => 'api-key', :payload => '[{"headers":{"Content-Type":"text/html"},"status":200}]')
81
+ Agent.instance.send_data_to_papermill
82
+ end
83
+
84
+ it 'should not send anything if no requests have been stored' do
85
+ Storage.clear
86
+ Agent.should_not_receive(:do_request)
87
+ Agent.instance.send_data_to_papermill
88
+ end
89
+ end
90
+
91
+ end
92
+
File without changes
File without changes
@@ -0,0 +1,109 @@
1
+ require 'spec_helper'
2
+ require 'sinatra'
3
+
4
+
5
+ module Papermill
6
+
7
+ describe 'collecting statistics via the middleware layer' do
8
+ before do
9
+ @app = Sinatra.new Sinatra::Application do
10
+ get '/' do
11
+ '<html><body>Hello, world!</body></html>'
12
+ end
13
+ end
14
+ @app.use Papermill::Collector
15
+
16
+ request_headers = {'HTTP_USER_AGENT' => 'agent', 'REMOTE_ADDR' => '0.0.0.0',
17
+ 'QUERY_STRING' => 'q=search',
18
+ 'SCRIPT_NAME' => 'my-script',
19
+ 'HTTP_ACCEPT' => 'text/html, application/json',
20
+ 'HTTP_HOST' => 'localhost',
21
+ 'REQUEST_URI' => 'i-requested-this',
22
+ 'REMOTE_HOST' => '123.345.34.2',
23
+ 'SERVER_SOFTWARE' => 'apache'
24
+ }
25
+
26
+ Time.stub(:now => Time.utc(2010, 01, 01))
27
+ @status, @headers, @body = @app.call(Rack::MockRequest.env_for('/', request_headers))
28
+ end
29
+
30
+ def last_store
31
+ Storage.store.last
32
+ end
33
+
34
+ def headers
35
+ last_store[:headers]
36
+ end
37
+
38
+ it 'records the query string' do
39
+ last_store[:status].should == 200
40
+ end
41
+
42
+ it 'records request duration'
43
+
44
+ it 'records the request time' do
45
+ last_store[:request_time].should == Time.utc(2010, 01, 01)
46
+ end
47
+
48
+ it 'records the path info' do
49
+ headers['PATH_INFO'].should == '/'
50
+ end
51
+
52
+ it 'records the request uri' do
53
+ headers['REQUEST_URI'].should == 'i-requested-this'
54
+ end
55
+
56
+ it 'records the query remote addr' do
57
+ headers['REMOTE_ADDR'].should == '0.0.0.0'
58
+ end
59
+
60
+ it 'records the server name' do
61
+ headers['SERVER_NAME'].should == 'example.org'
62
+ end
63
+
64
+ it 'records the query string' do
65
+ headers['QUERY_STRING'].should == 'q=search'
66
+ end
67
+
68
+ it 'records the request method' do
69
+ headers['REQUEST_METHOD'].should == 'GET'
70
+ end
71
+
72
+ it 'records the script name' do
73
+ headers['SCRIPT_NAME'].should == 'my-script'
74
+ end
75
+
76
+ it 'records the user agent' do
77
+ headers['HTTP_USER_AGENT'].should == 'agent'
78
+ end
79
+
80
+ it 'records the http accept types' do
81
+ headers['HTTP_ACCEPT'].should == 'text/html, application/json'
82
+ end
83
+
84
+ it 'records the http host' do
85
+ headers['HTTP_HOST'].should == 'localhost'
86
+ end
87
+
88
+ it 'records the server software' do
89
+ headers['SERVER_SOFTWARE'].should == 'apache'
90
+ end
91
+
92
+ it 'records the remote host' do
93
+ headers['REMOTE_HOST'].should == '123.345.34.2'
94
+ end
95
+
96
+ it 'records the url scheme' do
97
+ headers['rack.url_scheme'].should == 'http'
98
+ end
99
+
100
+ context 'for a 304 response' do
101
+ it 'only records certain things'
102
+ end
103
+
104
+ context 'for a streamed file response' do
105
+ it 'only records certain things'
106
+ end
107
+ end
108
+
109
+ end
@@ -0,0 +1,27 @@
1
+ require 'spec_helper'
2
+
3
+ module Papermill
4
+
5
+ describe '.parse' do
6
+ before do
7
+ ResponseAdapters::Base.stub(:new).and_return(mock('object', :parse => nil))
8
+ end
9
+ context 'in a normal rack app' do
10
+ it 'uses the base adapter' do
11
+ ResponseAdapters::Base.should_receive(:new).and_return(mock('object', :parse => nil))
12
+ ResponseParser.parse(200, {}, [], {})
13
+ end
14
+ end
15
+
16
+ context 'in a rails app' do
17
+ class Rails
18
+ end
19
+
20
+ it 'uses the rails adapter' do
21
+ ResponseAdapters::Rails.should_receive(:new).and_return(mock('object', :parse => nil))
22
+ ResponseParser.parse(200, {}, [], {})
23
+ end
24
+ end
25
+ end
26
+
27
+ end
@@ -0,0 +1,21 @@
1
+ ENV['RACK_ENV'] = 'test'
2
+
3
+ require 'rubygems'
4
+ require 'bundler'
5
+
6
+ Bundler.require :default, :test
7
+
8
+ $:.unshift File.dirname(File.expand_path(__FILE__))
9
+ $:.unshift File.dirname(File.expand_path(__FILE__) + '/../lib')
10
+
11
+ require 'papermill-agent'
12
+ require 'fakeweb'
13
+
14
+ FakeWeb.allow_net_connect = false
15
+
16
+ RSpec.configure do |config|
17
+ config.before(:each) do
18
+ # clear the storage before each example
19
+ Papermill::Storage.clear
20
+ end
21
+ end
@@ -0,0 +1,34 @@
1
+ require 'spec_helper'
2
+
3
+ module Papermill
4
+
5
+ describe 'adding an item to the local data storage' do
6
+ it 'acts like an array' do
7
+ Storage.store.should respond_to(:<<)
8
+ end
9
+
10
+ it 'stores data' do
11
+ Storage.store << 'some data'
12
+ Storage.store.should include 'some data'
13
+ end
14
+ end
15
+
16
+ describe 'the store' do
17
+ it 'wraps the singleton instance' do
18
+ Storage.store == Storage.instance
19
+ end
20
+ end
21
+
22
+ describe 'emptying the cache' do
23
+ before { Storage.clear }
24
+
25
+ it 'clears out the storage' do
26
+ Storage.store << 'stuff'
27
+ Storage.store.should == ['stuff']
28
+
29
+ Storage.clear
30
+ Storage.store.should == []
31
+ end
32
+ end
33
+
34
+ end
metadata ADDED
@@ -0,0 +1,106 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: papermill-agent
3
+ version: !ruby/object:Gem::Version
4
+ prerelease: false
5
+ segments:
6
+ - 0
7
+ - 0
8
+ - 1
9
+ version: 0.0.1
10
+ platform: ruby
11
+ authors:
12
+ - Alex Sharp
13
+ autorequire:
14
+ bindir: bin
15
+ cert_chain: []
16
+
17
+ date: 2010-11-19 00:00:00 -08:00
18
+ default_executable:
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
21
+ name: rest-client
22
+ prerelease: false
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ none: false
25
+ requirements:
26
+ - - ~>
27
+ - !ruby/object:Gem::Version
28
+ segments:
29
+ - 1
30
+ - 6
31
+ - 1
32
+ version: 1.6.1
33
+ type: :runtime
34
+ version_requirements: *id001
35
+ description: The client agent for papermillapp.com
36
+ email:
37
+ - ajsharp@gmail.com
38
+ executables: []
39
+
40
+ extensions: []
41
+
42
+ extra_rdoc_files:
43
+ - README.md
44
+ - doc/ideas.md
45
+ files:
46
+ - lib/papermill-agent/agent.rb
47
+ - lib/papermill-agent/rack/collector.rb
48
+ - lib/papermill-agent/response_adapters/base.rb
49
+ - lib/papermill-agent/response_adapters/rails.rb
50
+ - lib/papermill-agent/response_adapters/sinatra.rb
51
+ - lib/papermill-agent/response_parser.rb
52
+ - lib/papermill-agent/storage.rb
53
+ - lib/papermill-agent.rb
54
+ - LICENSE
55
+ - README.md
56
+ - config/papermill.sample.yml
57
+ - Gemfile
58
+ - Rakefile
59
+ - doc/ideas.md
60
+ - spec/agent_spec.rb
61
+ - spec/integration/rails2_spec.rb
62
+ - spec/integration/rails3_spec.rb
63
+ - spec/rack/collector_spec.rb
64
+ - spec/response_parser_spec.rb
65
+ - spec/spec_helper.rb
66
+ - spec/storage_spec.rb
67
+ has_rdoc: true
68
+ homepage: http://github.com/ajsharp/papermill-agent
69
+ licenses: []
70
+
71
+ post_install_message:
72
+ rdoc_options: []
73
+
74
+ require_paths:
75
+ - lib
76
+ required_ruby_version: !ruby/object:Gem::Requirement
77
+ none: false
78
+ requirements:
79
+ - - ">="
80
+ - !ruby/object:Gem::Version
81
+ segments:
82
+ - 0
83
+ version: "0"
84
+ required_rubygems_version: !ruby/object:Gem::Requirement
85
+ none: false
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ segments:
90
+ - 0
91
+ version: "0"
92
+ requirements: []
93
+
94
+ rubyforge_project:
95
+ rubygems_version: 1.3.7
96
+ signing_key:
97
+ specification_version: 3
98
+ summary: The client agent for papermillapp.com
99
+ test_files:
100
+ - spec/agent_spec.rb
101
+ - spec/integration/rails2_spec.rb
102
+ - spec/integration/rails3_spec.rb
103
+ - spec/rack/collector_spec.rb
104
+ - spec/response_parser_spec.rb
105
+ - spec/spec_helper.rb
106
+ - spec/storage_spec.rb