maxaf-fluke 0.0.3 → 0.0.4

data/History.txt

@@ -1,3 +1,8 @@
+=== 0.0.4 2009-09-09
+
+* HTTP proxy support.
+* Added MIME type recording.
+
 === 0.0.3 2009-09-08
 
 * Added Watcher#url_strftime to simplify time-based URL generation.

data/db/migrate/0003_add_meth_to_watchers.rb

@@ -0,0 +1,13 @@
+class AddMethToWatchers < ActiveRecord::Migration
+  def self.up
+    change_table :watchers do |t|
+      t.string :meth, :null => false, :default => 'GET'
+    end
+  end
+
+  def self.down
+    change_table :watchers do |t|
+      t.remove :meth
+    end
+  end
+end

data/db/migrate/0004_add_url_to_results.rb

@@ -0,0 +1,13 @@
+class AddUrlToResults < ActiveRecord::Migration
+  def self.up
+    change_table :results do |t|
+      t.string :url, :null => true
+    end
+  end
+
+  def self.down
+    change_table :results do |t|
+      t.remove :url
+    end
+  end
+end

data/examples/dot-fluke.yml

@@ -0,0 +1,19 @@
+:db:
+  adapter: jdbcmysql
+  encoding: utf8
+  reconnect: true
+  database: fluke
+  pool: 5
+  username: fluke
+  password: fluke
+  host: localhost
+  port: 3306
+:s3:
+  :connection:
+    :access_key_id: booya
+    :secret_access_key: kasha
+  :bucket:
+    :result: my-result-bucket
+:proxy:
+  :host: localhost
+  :port: 8080

data/examples/gen_url.rb

@@ -0,0 +1,3 @@
+watch :foo do
+  self.url_generator { |raw| "#{raw}?_=#{Time.now.to_i}" }
+end

data/examples/strftime_url.rb

@@ -0,0 +1,3 @@
+watch :foo_two do
+  self.url_strftime(-3.weeks)
+end

data/fluke.gemspec

@@ -0,0 +1,48 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{fluke}
+  s.version = "0.0.4"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Max Afonov"]
+  s.date = %q{2009-09-09}
+  s.default_executable = %q{fluke}
+  s.description = %q{A simple resource observer designed to detect change over time.}
+  s.email = ["max@bumnetworks.com"]
+  s.executables = ["fluke"]
+  s.extra_rdoc_files = ["History.txt", "Manifest.txt"]
+  s.files = ["bin/fluke", "db/migrate/0001_create_watchers.rb", "db/migrate/0002_create_results.rb", "db/migrate/0003_add_meth_to_watchers.rb", "db/migrate/0004_add_url_to_results.rb", "examples/dot-fluke.yml", "examples/gen_url.rb", "examples/strftime_url.rb", "fluke.gemspec", "History.txt", "lib/fluke/cli.rb", "lib/fluke/monkey_patch.rb", "lib/fluke.rb", "lib/fluke/result.rb", "lib/fluke/watcher.rb", "Manifest.txt", "Rakefile", "README.rdoc", "script/console", "script/destroy", "script/generate", "tasks/ar.rake", "test/test_fluke_cli.rb", "test/test_fluke.rb", "test/test_helper.rb"]
+  s.homepage = %q{http://github.com/maxaf/fluke}
+  s.rdoc_options = ["--main", "README.rdoc"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{fluke}
+  s.rubygems_version = %q{1.3.3}
+  s.summary = %q{A simple resource observer designed to detect change over time.}
+  s.test_files = ["test/test_fluke_cli.rb", "test/test_helper.rb", "test/test_fluke.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<activerecord>, [">= 2.3"])
+      s.add_runtime_dependency(%q<aws-s3>, [">= 0.6.2"])
+      s.add_runtime_dependency(%q<httpclient>, [">= 2.1.5.2"])
+      s.add_runtime_dependency(%q<pvande-differ>, [">= 0.1.1"])
+      s.add_development_dependency(%q<hoe>, [">= 2.3.3"])
+    else
+      s.add_dependency(%q<activerecord>, [">= 2.3"])
+      s.add_dependency(%q<aws-s3>, [">= 0.6.2"])
+      s.add_dependency(%q<httpclient>, [">= 2.1.5.2"])
+      s.add_dependency(%q<pvande-differ>, [">= 0.1.1"])
+      s.add_dependency(%q<hoe>, [">= 2.3.3"])
+    end
+  else
+    s.add_dependency(%q<activerecord>, [">= 2.3"])
+    s.add_dependency(%q<aws-s3>, [">= 0.6.2"])
+    s.add_dependency(%q<httpclient>, [">= 2.1.5.2"])
+    s.add_dependency(%q<pvande-differ>, [">= 0.1.1"])
+    s.add_dependency(%q<hoe>, [">= 2.3.3"])
+  end
+end

data/lib/fluke.rb

@@ -9,7 +9,7 @@ require 'activerecord'
 require 'aws/s3'
 
 module Fluke
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
   ROOT = Dir.new(File.expand_path(File.dirname(__FILE__) + "/../"))
   DEFAULT_CONF_PATH = "~/.fluke.yml"
   @@verbose = false
@@ -55,12 +55,17 @@ module Fluke
     }
     p.merge! opts
     @@log_here = p[p[:log_here]] || p[:stderr]
+
     begin
       @@conf = YAML::load(File.open(p[:conf_path]))
+      if !@@conf[:proxy].nil?
+        @@conf[:s3][:connection][:proxy] = @@conf[:proxy]
+        @@conf[:proxy_string] = "http://#{@@conf[:proxy][:host]}:#{@@conf[:proxy][:port]}"
+      end
       ActiveRecord::Base.colorize_logging = false
       ActiveRecord::Base.logger = Logger.new(@@log_here) if verbose?
       ActiveRecord::Base.establish_connection(@@conf[:db])
-      AWS::S3::Base.establish_connection!(@@conf[:s3][:auth])
+      AWS::S3::Base.establish_connection!(@@conf[:s3][:connection])
       Result::Body.set_current_bucket_to @@conf[:s3][:bucket][:result]
     rescue => e
       STDERR.puts e.inspect
@@ -73,8 +78,12 @@ module Fluke
     watchers.each do |n,watcher|
       watcher.thread = Thread.new do
         while true
-          watcher.run
-          watcher.wait
+          begin
+            watcher.run
+            watcher.wait
+          rescue => e
+            Fluke.log { "#{watcher}: failed to run: #{e.inspect}; backtrace: #{e.backtrace.join("; ")}" }
+          end
         end
       end
     end

data/lib/fluke/result.rb

@@ -25,12 +25,17 @@ module Fluke
       Body.exists? self.body_key
     end
 
-    def self.from_response(watcher, content, generated_url = nil)
-      checksum = Digest::SHA1.hexdigest(content)
+    def self.from_response(args)
+      opts = {
+        :mime_type => 'binary/octet-stream',
+        :generated_url => nil
+      }.merge(args)
+
+      checksum = Digest::SHA1.hexdigest(opts[:content])
       result = Result.new :checksum => checksum
-      result.watcher = watcher
-      if generated_url and generated_url != watcher.url
-        result.url = generated_url
+      result.watcher = opts[:watcher]
+      if opts[:generated_url] and opts[:generated_url] != opts[:watcher].url
+        result.url = opts[:generated_url]
       end
       result.save
 
@@ -39,7 +44,7 @@ module Fluke
           Fluke.log { "#{result.watcher} exists: #{result.body_key}" }
         else
           Fluke.log { "#{result.watcher} store: #{result.body_key}" }
-          Body.store(result.body_key, content.dup)
+          Body.store(result.body_key, opts[:content].dup, :content_type => opts[:mime_type])
         end
       end
 

data/lib/fluke/watcher.rb

@@ -20,18 +20,18 @@ module Fluke
     end
 
     def run_file(path)
-      result = Result.from_response(self, File.read(path), "file://#{path}")
+      result = Result.from_response :watcher => self, :content => File.read(path), :generated_url => "file://#{path}"
     end
 
     def run_http(generated_url)
-      hc = HTTPClient.new
+      hc = HTTPClient.new :proxy => Fluke.conf[:proxy_string], :user_agent => "Fluke/#{Fluke::VERSION}"
       http_method = self.meth.downcase.to_sym
       unless hc.respond_to?(http_method)
         Fluke::log { "#{self}: invalid method '#{http_method}', converting to :get" }
         http_method = :get
       end
       res = hc.send(http_method, generated_url, { 'Cache-Control' => 'no-cache', 'Pragma' => 'no-cache' })
-      result = Result.from_response(self, res.body.dump.dup, generated_url)
+      result = Result.from_response :watcher => self, :content => res.body.dump.dup, :generated_url => generated_url, :mime_type => res.header['Content-Type']
     end
 
     def url_generator(&block)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: maxaf-fluke
 version: !ruby/object:Gem::Version 
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors: 
 - Max Afonov
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-09-08 00:00:00 -07:00
+date: 2009-09-09 00:00:00 -07:00
 default_executable: fluke
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -76,6 +76,12 @@ files:
 - bin/fluke
 - db/migrate/0001_create_watchers.rb
 - db/migrate/0002_create_results.rb
+- db/migrate/0003_add_meth_to_watchers.rb
+- db/migrate/0004_add_url_to_results.rb
+- examples/dot-fluke.yml
+- examples/gen_url.rb
+- examples/strftime_url.rb
+- fluke.gemspec
 - History.txt
 - lib/fluke/cli.rb
 - lib/fluke/monkey_patch.rb