fluoride-collector 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cf7c03aa82878bb35ce0803cf3fafc97821809aa
-  data.tar.gz: 6011f540fff63c4babdbf8e449c20f634c227113
+  metadata.gz: 2c5a859fb19f96aa3640e06ba654c5fbe887ea06
+  data.tar.gz: 0a295ed1d22617b187be2a446fcf58042edadb06
 SHA512:
-  metadata.gz: d296dd8d97c1d6d85750cb366588bbd9cb6f29fc701a7a04c9e12c99c9b707e7b8eb9565287b394d16d2ef9a485948c3194e7b8f86a20579da69a7a42b730934
-  data.tar.gz: 54fb62bc45b2300147f71920cc804860e7f99ad399d9913bd1efae8237ce7692201ff95b1597b1edbf0bc4fa2234d36a9a924f05ceaa16321fb536e1fe061eff
+  metadata.gz: da0889ab97c77b64a3ccdc2489a8a85c2262397fb5c57aa250b90e7c27ba05116de989da726bd2e5083a159aaa2ecf316971555ba33cec5cab46caee7882c7fb
+  data.tar.gz: 795e63f92f12646eb040715c87d1338bd86309dabd95def0982c153f864345dd720efe8cdd8f5b3d29d573a4e3bf8baa665a9afedcd3fb0672697d03cfbd47f8

data/lib/fluoride-collector/config.rb

@@ -0,0 +1,37 @@
+require 'fluoride-collector'
+module Fluoride
+  module Collector
+    class Config
+      attr_accessor :tags
+
+      def persister(type, record)
+        persister_class.new(self, type, record)
+      end
+
+      class FS < Config
+        attr_accessor :directory, :storage_limit
+
+        def initialize
+          require 'fluoride-collector/storage/fs'
+          @storage_limit = 250_000_000
+        end
+
+        def persister_class
+          Storage::FS
+        end
+      end
+
+      class S3 < Config
+        attr_accessor :bucket, :key_id, :access_secret
+
+        def initialize
+          require 'fluoride-collector/storage/s3'
+        end
+
+        def persister_class
+          Storage::S3
+        end
+      end
+    end
+  end
+end

data/lib/fluoride-collector/middleware/collect-exceptions.rb

@@ -0,0 +1,34 @@
+require 'fluoride-collector/middleware'
+module Fluoride
+  module Collector
+    class Middleware
+      class CollectExceptions < Middleware
+        def call(env)
+          @app.call(env)
+        rescue Object => ex
+          store( clean_hash(
+              "type" => "exception_raised",
+              "tags" => @tagging,
+              "request" => request_hash(env),
+              "response" => exception_hash(ex)
+          ))
+          raise
+        end
+
+        private
+
+        def collection_type
+          :exception
+        end
+
+        def exception_hash(ex)
+          {
+            "type" => ex.class.name,
+            "message" => ex.message,
+            "backtrace" => ex.backtrace[0..10]
+          }
+        end
+      end
+    end
+  end
+end

data/lib/fluoride-collector/middleware/collect-exchanges.rb

@@ -0,0 +1,44 @@
+require 'fluoride-collector/middleware'
+module Fluoride
+  module Collector
+    class Middleware
+      class CollectExchanges < Middleware
+        def call(env)
+          @app.call(env).tap do |response|
+            store( clean_hash(
+              "type" => "normal_exchange",
+              "tags" => @tagging,
+              "request" => request_hash(env),
+              "response" => response_hash(response)
+            ))
+          end
+        end
+
+        private
+
+        def collection_type
+          :exchange
+        end
+
+        def extract_body(body)
+          array = []
+          body.each do |chunk|
+            array << chunk
+          end
+          body.rewind if body.respond_to?(:rewind)
+          array
+        end
+
+        def response_hash(response)
+          status, headers, body = *response
+
+          {
+            "status" => status,
+            "headers" => headers.to_hash,
+            "body" => extract_body(body)
+          }
+        end
+      end
+    end
+  end
+end

data/lib/fluoride-collector/middleware.rb

@@ -1,48 +1,24 @@
 require 'uri'
-
+require 'fluoride-collector'
 module Fluoride
   module Collector
     class Middleware
-      def initialize(app, directory, storage_limit = 250_000_000, tagging = nil)
+      attr_reader :config
+
+      def initialize(app, config)
         @app = app
-        @directory = directory
-        @storage_limit = storage_limit
-        @tagging = tagging
+        @config = config
+        @tagging = config.tags
       end
 
       private
 
-      def thread_locals
-        Thread.current[:fluoride_collector] ||= {}
-      end
-
-      def storage_path
-        thread_locals[collection_type] ||= File::join(@directory, "#{collection_type}-#{Process.pid}-#{Thread.current.object_id}.yml")
-      end
-
-      def storage_file
-        FileUtils.mkdir_p(File::dirname(storage_path))
-        return if storage_used > @storage_limit
-        File::open(storage_path, "a") do |file|
-          yield file
-        end
-      end
-
-      def storage_used
-        dir = Dir.new(@directory)
-        dir.inject(0) do |sum, file|
-          if file =~ %r{\A\.}
-            sum
-          else
-            sum + File.size(File::join(@directory, file))
-          end
-        end
-      end
-
       def store(record)
-        storage_file do |file|
-          file.write(YAML::dump(record))
-        end
+        #take only pictures
+        @config.persister(collection_type, record).write
+      rescue Exception => ex
+        #leave only footprints
+        $stderr.puts "#{ex.inspect}" if $stderr.respond_to? :puts
       end
 
       def clean_hash(hash)
@@ -50,7 +26,7 @@ module Fluoride
           value = hash[key]
           case value
           when String
-            if value.ascii_only?
+            if value.respond_to?(:ascii_only?) and value.ascii_only? and value.respond_to?(:force_encoding)
               value = value.dup
               value.force_encoding("US-ASCII")
               hash[key] = value
@@ -78,75 +54,9 @@ module Fluoride
           "host" => env['HTTP_HOST'] || "#{env['SERVER_NAME'] || env['SERVER_ADDR']}:#{env['SERVER_PORT']}",
           "path" => URI.unescape(env["SCRIPT_NAME"].to_s + env["PATH_INFO"].to_s),
           "query_string" => env["QUERY_STRING"].to_s,
-          "body" => body,
+          "body" => body
         )
       end
-
-      class CollectExceptions < Middleware
-        def call(env)
-          @app.call(env)
-        rescue Object => ex
-          store( clean_hash(
-              "type" => "exception_raised",
-              "tags" => @tagging,
-              "request" => request_hash(env),
-              "response" => exception_hash(ex)
-          ))
-          raise
-        end
-
-        private
-
-        def collection_type
-          :exception
-        end
-
-        def exception_hash(ex)
-          {
-            "type" => ex.class.name,
-            "message" => ex.message,
-            "backtrace" => ex.backtrace[0..10]
-          }
-        end
-      end
-
-      class CollectExchanges < Middleware
-        def call(env)
-          @app.call(env).tap do |response|
-            store( clean_hash(
-              "type" => "normal_exchange",
-              "tags" => @tagging,
-              "request" => request_hash(env),
-              "response" => response_hash(response)
-            ))
-          end
-        end
-
-        private
-
-        def collection_type
-          :exchange
-        end
-
-        def extract_body(body)
-          array = []
-          body.each do |chunk|
-            array << chunk
-          end
-          body.rewind if body.respond_to?(:rewind)
-          array
-        end
-
-        def response_hash(response)
-          status, headers, body = *response
-
-          {
-            "status" => status,
-            "headers" => headers.to_hash,
-            "body" => extract_body(body)
-          }
-        end
-      end
     end
   end
 end

data/lib/fluoride-collector/rails/railtie.rb

@@ -1,3 +1,5 @@
+require 'fluoride-collector'
+
 module Fluoride
   module Collector
     class Railtie < ::Rails::Railtie
@@ -5,17 +7,27 @@ module Fluoride
       config.fluoride.storage_limit = 250_000_000
       config.fluoride.tags = nil
       config.fluoride.directory = "fluoride-collector"
+      config.fluoride.store_to = :file
 
       initializer "fluoride-collector.add_middleware" do |app|
-        app.middleware.use(   Fluoride::Collector::Middleware::CollectExceptions,
-                              config.fluoride.directory,
-                              config.fluoride.storage_limit,
-                              config.fluoride.tags)
-        app.middleware.insert("Rack::Sendfile",
-                              Fluoride::Collector::Middleware::CollectExchanges,
-                              config.fluoride.directory,
-                              config.fluoride.storage_limit,
-                              config.fluoride.tags)
+        cfg = nil
+        case config.fluoride.store_to
+        when :file
+          cfg = Fluoride::Collector::Config::FS.new
+          cfg.directory = config.fluoride.directory
+          cfg.storage_limit = config.fluoride.storage_limit
+        when :s3
+          cfg = Fluoride::Collector::Config::S3.new
+          cfg.bucket = config.fluoride.bucket
+          cfg.key_id = config.fluoride.key_id
+          cfg.access_secret = config.fluoride.access_secret
+        else
+          raise Fluoride::Collector::ConfigurationError, "Don't know how to store to #{config.fluoride.store_to}!"
+        end
+        cfg.tags = config.fluoride.tags
+
+        app.middleware.use(Fluoride::Collector::Middleware::CollectExceptions, cfg)
+        app.middleware.insert(0, Fluoride::Collector::Middleware::CollectExchanges, cfg)
       end
     end
   end

data/lib/fluoride-collector/storage/fs.rb

@@ -0,0 +1,46 @@
+require 'fluoride-collector'
+require 'fluoride-collector/storage'
+module Fluoride
+  module Collector
+    class Storage
+      class FS < Storage
+        def directory
+          @config.directory
+        end
+
+        def storage_limit
+          @config.storage_limit
+        end
+
+        def write
+          storage_file do |file|
+            file.write(record_yaml)
+          end
+        end
+
+        def storage_used
+          dir = Dir.new(directory)
+          dir.inject(0) do |sum, file|
+            if file =~ %r{\A\.}
+              sum
+            else
+              sum + File.size(File::join(directory, file))
+            end
+          end
+        end
+
+        def storage_path
+          thread_locals[collection_type] ||= File::join(directory, "#{collection_type}-#{Process.pid}-#{Thread.current.object_id}.yml")
+        end
+
+        def storage_file
+          FileUtils.mkdir_p(File::dirname(storage_path))
+          return if storage_used > storage_limit
+          File::open(storage_path, "a") do |file|
+            yield file
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluoride-collector/storage/s3.rb

@@ -0,0 +1,87 @@
+require 'fluoride-collector'
+require 'fluoride-collector/storage'
+require 'net/http'
+require 'openssl'
+require 'base64'
+module Fluoride
+  module Collector
+    class Storage
+      class S3 < Storage
+        def write
+          Net::HTTP.start(host, port) do |http|
+            res = http.request(put_request)
+          end
+        end
+
+        def bucket
+          @config.bucket
+        end
+
+        def key_id
+          @config.key_id
+        end
+
+        def access_secret
+          @config.access_secret
+        end
+
+        def host
+          "#{bucket}.s3.amazonaws.com"
+        end
+
+        def port
+          443
+        end
+
+        def request_index
+          @request_index ||=
+            begin
+              thread_locals[:request_index] ||= 0
+              thread_locals[:request_index] += 1
+            end
+        end
+
+        def remote_path
+          @remote_path ||= "#{collection_type}-#{Process.pid}-#{Thread.current.object_id}-#{request_index}.yml"
+        end
+
+        def uri
+          "https://#{host}/#{remote_path}"
+        end
+
+        def authorization
+          hmac = OpenSSL::HMAC.digest(OpenSSL::Digest::SHA1.new, string_to_sign, access_secret)
+          signature = Base64.strict_encode64(hmac)
+
+          "AWS #{key_id}:#{signature}"
+        end
+
+        def string_to_sign
+          "PUT\n#{content_md5}\n#{content_type}\n#{date}\n/#{bucket}/#{remote_path}"
+        end
+
+        def content_md5
+          @context_md5 ||= Base64.strict_encode64(OpenSSL::Digest::MD5.digest(record_yaml))
+        end
+
+        def content_type
+          "text/yaml"
+        end
+
+        def date
+          @date ||= Time.now.strftime("+%a, %d %h %Y %T %z")
+        end
+
+        def put_request
+          req = Net::HTTP::Put.new(uri)
+          req["Authorization"] = authorization
+          req["Date"] = date
+          req["Content-MD5"] = content_md5
+          req["Content-Type"] = content_type
+          req.body = record_yaml
+          return req
+        end
+      end
+    end
+  end
+end

data/lib/fluoride-collector/storage.rb

@@ -0,0 +1,22 @@
+require 'fluoride-collector'
+module Fluoride
+  module Collector
+    class Storage
+      attr_reader :collection_type, :record
+
+      def initialize(config, type, record)
+        @record = record
+        @collection_type = type
+        @config = config
+      end
+
+      def thread_locals
+        Thread.current[:fluoride_collector] ||= {}
+      end
+
+      def record_yaml
+        @record_yaml ||= YAML::dump(record)
+      end
+    end
+  end
+end

data/lib/fluoride-collector.rb

@@ -1,2 +1,11 @@
-require 'fluoride-collector/middleware'
+module Fluoride
+  module Collector
+    class ConfigurationError < ::StandardError
+    end
+  end
+end
+
+require 'fluoride-collector/config'
+require 'fluoride-collector/middleware/collect-exceptions'
+require 'fluoride-collector/middleware/collect-exchanges'
 require 'fluoride-collector/rails' if defined?(Rails)

data/spec/middleware.rb

@@ -9,11 +9,20 @@ describe Fluoride::Collector::Middleware do
     Dir.new(dir.path)
   end
 
+  let :config do
+    Fluoride::Collector::Config::FS.new.tap do |cfg|
+      cfg.directory = "collections"
+      cfg.storage_limit = 1500
+      cfg.tags = "TEST"
+    end
+  end
+
   let :app do
     run_app = test_app
     klass = middleware_class
+    cfg = config
     Rack::Builder.app do
-      use klass, "collections", 1500, "TEST"
+      use klass, cfg
       run run_app
     end
   end
@@ -71,7 +80,8 @@ describe Fluoride::Collector::Middleware do
         rescue test_ex_class
         end
         path = File::join(collection_directory.path, collection_directory.each.to_a.grep(/[a-z].*/).first)
-        YAML.load_stream(File.read(path)).first
+        stream = YAML.load_stream(File.read(path))
+        stream[0]
       end
 
       it "should have tags" do
@@ -114,7 +124,7 @@ describe Fluoride::Collector::Middleware do
       let :yaml do
         app.call(env)
         path = File::join(collection_directory.path, collection_directory.each.to_a.grep(/[a-z].*/).first)
-        YAML.load_stream(File.read(path)).first
+        YAML.load_stream(File.read(path))[0]
       end
 
       it "should have tags" do

data/spec/railtie.rb

@@ -1,15 +1,58 @@
-require 'railtie-help'
+require "action_controller/railtie"
+require "rails/test_unit/railtie"
+require 'fluoride-collector/rails'
 
 describe Fluoride::Collector::Railtie do
+
+  ENV["RAILS_ENV"] ||= 'test'
+
+  def config(app)
+
+  end
+
+  let :rails_application do
+    Class.new(::Rails::Application) do
+      config.active_support.deprecation = :stderr
+      config.eager_load = false
+    end.tap do |app|
+      config(app)
+      app.initialize!
+    end
+  end
+
+  after :each do
+    Rails.application = nil #because Rails has ideas of it's own, silly thing
+  end
+
   it "should add exception collection to the middleware stack" do
-    expect(Rails.application.middleware.middlewares).to include(Fluoride::Collector::Middleware::CollectExceptions)
+    expect(rails_application.middleware.middlewares).to include(Fluoride::Collector::Middleware::CollectExceptions)
   end
 
   it "should add exchange collection to the middleware stack" do
-    expect(Rails.application.middleware.middlewares).to include(Fluoride::Collector::Middleware::CollectExchanges)
+    expect(rails_application.middleware.middlewares).to include(Fluoride::Collector::Middleware::CollectExchanges)
   end
 
   it "should put exchange collection at top of stack" do
-    expect(Rails.application.middleware.middlewares.first).to eq(Fluoride::Collector::Middleware::CollectExchanges)
+    expect(rails_application.middleware.middlewares.first).to eq(Fluoride::Collector::Middleware::CollectExchanges)
+  end
+
+  describe "configured to use S3" do
+    def config(app)
+      app.configure do
+        config.fluoride.store_to = :s3
+        config.fluoride.bucket = "nowhere-really"
+        config.fluoride.key_id = "testtest"
+        config.fluoride.access_secrety = "donttellnobody"
+      end
+    end
+
+    it "should put exchange collection at top of stack" do
+      expect(rails_application.middleware.middlewares.first).to eq(Fluoride::Collector::Middleware::CollectExchanges)
+    end
+
+    it "should configured collector with S3Storage" do
+      collector = rails_application.middleware.middlewares.first.build(proc{})
+      expect(collector.config.persister("exchange", {})).to be_a(Fluoride::Collector::Storage::S3)
+    end
   end
 end

metadata

@@ -1,25 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: fluoride-collector
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Judson Lester
+- Evan Down
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-16 00:00:00.000000000 Z
+date: 2014-06-19 00:00:00.000000000 Z
 dependencies: []
 description: |2
     Part of the Fluoride suite - tools for making your black box a bit whiter
 email:
 - nyarly@gmail.com
+- evan@lrdesign.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/fluoride-collector/rails/railtie.rb
+- lib/fluoride-collector/config.rb
+- lib/fluoride-collector/middleware/collect-exchanges.rb
+- lib/fluoride-collector/middleware/collect-exceptions.rb
 - lib/fluoride-collector/rails.rb
+- lib/fluoride-collector/storage/fs.rb
+- lib/fluoride-collector/storage/s3.rb
+- lib/fluoride-collector/storage.rb
 - lib/fluoride-collector/middleware.rb
 - lib/fluoride-collector.rb
 - spec/railtie.rb
@@ -38,7 +46,7 @@ rdoc_options:
 - --main
 - doc/README
 - --title
-- fluoride-collector-0.0.3 Documentation
+- fluoride-collector-0.0.4 Documentation
 require_paths:
 - lib/
 required_ruby_version: !ruby/object:Gem::Requirement
@@ -52,7 +60,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: fluoride-collector
+rubyforge_project: 
 rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4