benschwarz-smoke 0.3.6 → 0.3.7

data/README.markdown

@@ -70,13 +70,15 @@ Integrity [is running for smoke](http://integrity.ffolio.net/smoke)
 
 ### TODO (working on, just mental notes)
 
-* Implement a disaptch / request method that will actually thread property (event-machine)
+* Secrets
 * Checkout experimental fakeweb version to stub out the yql specs
 
 #### Later / maybe
-* Allow for sources to explicitly set the content type being returned for those stupid content providers
 * YQL w/oAuth
 * YQL Subqueries?
+* Implement basic auth for sources
+* Allow for sources to explicitly set the content type being returned for those stupid content providers
+
 
 #### For wiki pages (docs, later)
 * How to use `path`

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
+:patch: 7
 :major: 0
 :minor: 3
-:patch: 6

data/lib/smoke.rb

@@ -1,9 +1,9 @@
-require 'rubygems'
+require 'zlib'
 require 'open-uri'
 require 'logger'
-require 'json'
 require 'crack'
 require 'simple-rss'
+require 'json'
 
 $:<< File.join(File.dirname(__FILE__))
 
@@ -13,7 +13,10 @@ require 'core_ext/hash.rb'
 module Smoke  
   class << self
     @@active_sources = {}
-    attr_reader :active_sources
+    @@config = {
+      :enable_logging => true,
+      :user_agent     => "Ruby/#{RUBY_VERSION}/Smoke"
+    }
     
     # Access registered smoke source instances
     #
@@ -58,7 +61,25 @@ module Smoke
     #   Smoke.log.error "message"
     #   Smoke.log.warn "message"
     def log
-      @@log ||= Logger.new($stdout)
+      @@log ||= Logger.new(config[:enable_logging] ? $stdout : "/dev/null")
+    end
+    
+    # Set any configurable options
+    #
+    #     Smoke.configure do |c|
+    #       c[:user_agent] = "Some other site"
+    #     end
+    #
+    def configure(&block)
+      yield @@config
+    end
+    
+    # Access configuration options
+    #
+    #   Smoke.config[:option_name]
+    #   => true
+    def config
+      @@config 
     end
     
     def yql(name, &block); Smoke::Source::YQL.new(name, &block); end

data/lib/smoke/origin.rb

@@ -34,8 +34,9 @@ module Smoke
       end      
     end
     
-    def items=(items) # :nodoc:
-      @items = items.flatten.map{|i| i.symbolize_keys! }
+    def items=(response) # :nodoc:
+      @items = [(@path.nil?) ? response : drill(response, *@path)]
+      symbolize_keys!
       transform!
     end
     
@@ -73,6 +74,24 @@ module Smoke
       @transformation << block
     end
     
+    # Transform must be used inside an `emit` block.
+    # It can be used to alter the a single field at a time
+    #
+    # Usage:
+    #     emit do
+    #       transform :name do |name|
+    #         name.gsub(/\302/, "")
+    #       end
+    #     end
+    #
+    # In quasi-english: The result of the block is returned and set to each 
+    # of the :name attributes in the result set.
+    def transform(name, &block)
+      items.each do |item|
+        item[name] = yield(item[name]) || item[name]
+      end
+    end
+    
     # Prepare is used when you'd like to provision for 
     # arguments / variables to be set after the source definition.
     # Eg, create a source definition for twitter, omitting the "username".
@@ -188,6 +207,10 @@ module Smoke
       @transformation.each{|t| t.call } unless @transformation.nil?
     end
     
+    def symbolize_keys!
+      @items = items.flatten.map{|i| i.symbolize_keys! } if items.respond_to? :flatten
+    end
+    
     def drill(*path)
       path.inject(nil) do |obj, pointer|
         obj = obj.nil? ? pointer : obj[pointer]

data/lib/smoke/request.rb

@@ -20,9 +20,14 @@ module Smoke
     
     private
     def dispatch
+      opts = {
+        "User-Agent"      => Smoke.config[:user_agent],
+        "Accept-Encoding" => "gzip"
+      }
       Thread.new {
-        open(@uri, "User-Agent" => "Ruby/#{RUBY_VERSION}/Smoke") do |request|
+        open(@uri, opts) do |request|
           @content_type = request.content_type
+          request = Zlib::GzipReader.new(request) if request.content_encoding.include? "gzip"
           @body = request.read
         end
       }.join

data/lib/smoke/source/data.rb

@@ -28,7 +28,7 @@ module Smoke
       protected
       def dispatch
         @request = Smoke::Request.new(@url)
-        self.items = (@path.nil?) ? @request.body : drill(@request.body, *@path)
+        self.items = @request.body
       end
     end
   end

data/lib/smoke/source/join.rb

@@ -6,7 +6,6 @@ module Smoke
     #
     # Usage:
     #   Smoke.join(:delicious, :twitter, :flickr) do
-    #     name :stream
     #     path :photos, :photo
     #   end
     class Join < Origin # :nodoc: 

data/lib/smoke/source/yql.rb

@@ -63,7 +63,7 @@ module Smoke
       
       def dispatch
         @request = Smoke::Request.new(build_uri)
-        self.items = [(@path.nil?) ? @request.body : drill(@request.body, *@path)]
+        self.items = @request.body
       end
       
       private

data/spec/smoke/origin_spec.rb

@@ -6,6 +6,10 @@ describe Smoke::Origin do
       emit do
         rename(:head => :title)
         sort(:title)
+        
+        transform :title do |title|
+          title.gsub(/Animal: /, '')
+        end
       end
     end
   end
@@ -43,8 +47,21 @@ describe Smoke::Origin do
     
     describe "filtering" do
       before do
-        TestSource.source(:keep)
-        TestSource.source(:discard)
+        TestSource.source(:keep) do
+          emit do
+            transform :head do |head|
+              head.gsub(/Animal: /, '')
+            end
+          end
+        end
+        
+        TestSource.source(:discard) do
+          emit do
+            transform :head do |head|
+              head.gsub(/Animal: /, '')
+            end
+          end
+        end
       end
       
       it "should keep items" do
@@ -93,12 +110,18 @@ describe Smoke::Origin do
   end
   
   it "method chaining" do
-    @source = TestSource.source(:chain)
-    @source.rename(:head => :header).sort(:header).output.should == [{:header => "Kangaroo"}, {:header => "Platypus"}]
+    TestSource.source(:chain) do
+      emit do
+        transform :head do |head|
+          head.gsub(/Animal: /, '')
+        end
+      end
+    end
+    Smoke[:chain].rename(:head => :header).sort(:header).output.should == [{:header => "Kangaroo"}, {:header => "Platypus"}]
   end
   
   it "should softly error when attempting to sort on a key that doesn't exist" do
-    TestSource.source(:chain).sort(:header).should_not raise_error("NoMethodError")
+    Smoke[:chain].sort(:header).should_not raise_error("NoMethodError")
   end
   
   describe "preperation" do
@@ -114,51 +137,65 @@ describe Smoke::Origin do
       lambda { @source.prepare }.should raise_error
       lambda { @source.prepare {} }.should_not raise_error
     end
-    
-    describe "call order" do
-      before :all do
-        @url = "http://domain.tld/benschwarz/feed"
-        FakeWeb.register_uri(@url, :response => File.join(SPEC_DIR, 'supports', 'flickr-photo.json'))
-        
-        Smoke.data :feed_preperation_call_order do
-          prepare do 
-            url "http://domain.tld/#{username}/feed"
-          end
-          
-          path :photos, :photo
+  end
+  
+  describe "call order" do
+    before :all do
+      @url = "http://domain.tld/benschwarz/feed"
+      FakeWeb.register_uri(@url, :response => File.join(SPEC_DIR, 'supports', 'flickr-photo.json'))
+      
+      Smoke.data :feed_preperation_call_order do
+        prepare do 
+          url "http://domain.tld/#{username}/feed"
         end
+        
+        path :photos, :photo
+      end
+    end
+    
+    describe "before setting variables" do
+      it "should fail" do
+        lambda { Smoke[:feed_preperation_call_order].output }.should raise_error(NameError)
+      end
+    end
+    
+    describe "setting abstract properties" do
+      it "should not respond to a property that hasn't been set" do
+        lambda { Smoke[:feed_preperation_call_order].abstract }.should raise_error(NoMethodError)
       end
       
-      describe "before setting variables" do
-        it "should fail" do
-          lambda { Smoke[:feed_preperation_call_order].output }.should raise_error(NameError)
-        end
+      it "should allow setting a property" do
+        lambda { Smoke[:feed_preperation_call_order].abstract(:value) }.should_not raise_error(NoMethodError)
+        Smoke[:feed_preperation_call_order].abstract.should == :value
       end
       
-      describe "setting abstract properties" do
-        it "should not respond to a property that hasn't been set" do
-          lambda { Smoke[:feed_preperation_call_order].abstract }.should raise_error(NoMethodError)
-        end
-        
-        it "should allow setting a property" do
-          lambda { Smoke[:feed_preperation_call_order].abstract(:value) }.should_not raise_error(NoMethodError)
-          Smoke[:feed_preperation_call_order].abstract.should == :value
-        end
-        
-        it "should chain the source when setting a property" do
-          Smoke[:feed_preperation_call_order].abstract(:value).should be_an_instance_of(Smoke::Source::Data)
-        end
+      it "should chain the source when setting a property" do
+        Smoke[:feed_preperation_call_order].abstract(:value).should be_an_instance_of(Smoke::Source::Data)
+      end
+    end
+    
+    describe "after setting variables" do
+      it "should output successfully" do
+        lambda { Smoke[:feed_preperation_call_order].username("benschwarz").output }.should_not raise_error
       end
       
-      describe "after setting variables" do
-        it "should output successfully" do
-          lambda { Smoke[:feed_preperation_call_order].username("benschwarz").output }.should_not raise_error
-        end
-        
-        it "should have used the correct url" do
-          Smoke[:feed_preperation_call_order].request.uri.should == @url
-        end
+      it "should have used the correct url" do
+        Smoke[:feed_preperation_call_order].request.uri.should == @url
       end
     end
   end
+
+  describe "trasformations" do
+    it "should respond to emit" do
+      Smoke[:test].should respond_to(:emit)
+    end
+    
+    it "should respond to transform" do
+      Smoke[:test].should respond_to(:transform)
+    end
+    
+    it "should have at least one transformation" do
+      Smoke[:test].transformation.size.should == 1
+    end
+  end
 end

data/spec/smoke/request_spec.rb

@@ -22,13 +22,33 @@ describe Smoke::Request do
   
   it "should be a pure ruby array response" do
     # Temporary real request, fakeweb isn't allowing content_type setting as of writing
-    @request = Smoke::Request.new("http://query.yahooapis.com/v1/public/yql?q=SELECT%20*%20FROM%20search.web%20WHERE%20query%20=%20'ruby'&format=xml")
-    @request.body.should be_an_instance_of(Hash)
+    request = Smoke::Request.new("http://query.yahooapis.com/v1/public/yql?q=SELECT%20*%20FROM%20search.web%20WHERE%20query%20=%20'ruby'&format=xml")
+    request.body.should be_an_instance_of(Hash)
   end
   
   it "should be a raw string response" do
-    @request = Smoke::Request.new(@url, :raw_response)
-    @request.body.should be_an_instance_of(String)
+    request = Smoke::Request.new(@url, :raw_response)
+    request.body.should be_an_instance_of(String)
+  end
+  
+  describe "gzipped responses" do
+    before do
+      # class Stream < StringIO
+      #   def close; rewind; end
+      # end
+      # output = Stream.new
+      # gz = Zlib::GzipWriter.new(output) 
+      # gz.write(File.read(@gzip_response)) 
+      # gz.close 
+      #@gzip_response = File.join(SPEC_DIR, 'supports', 'gzip.response')
+      #FakeWeb.register_uri(@url, :file => @gzip_response)
+    end
+    
+    it "should transparently handle a gzipped response" do
+      pending
+      request = Smoke::Request.new(@url)
+      request.body.should == "gzip_response"
+    end
   end
   
   describe "http redirects" do

data/spec/smoke_spec.rb

@@ -17,4 +17,31 @@ describe Smoke do
       Smoke[:b].should be_an_instance_of(Smoke::Origin)
     end
   end
+  
+  describe "configuration" do
+    it "should be configurable" do
+      Smoke.should respond_to(:configure)
+    end
+    
+    it "should be accessible" do
+      Smoke.should respond_to(:config)
+    end
+    
+    it "should take a block and be retrievable" do
+      Smoke.configure {|c| c[:spec] = true }
+      Smoke.config[:spec].should be_true
+    end
+    
+    describe "default configurations" do
+      it "should have at least one default configuration" do
+        Smoke.config.keys.should_not be_empty
+      end
+
+      it "should allow overwriting default configurations" do
+        key = Smoke.config.keys.first
+        Smoke.configure {|c| c[key] = true }
+        Smoke.config[key].should be_true
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -13,4 +13,8 @@ $:<< SPEC_DIR
 require 'smoke'
 
 require 'supports/mayo.rb'
-require 'supports/test_source.rb'
+require 'supports/test_source.rb'
+
+Smoke.configure do |c|
+  c[:enable_logging] = false
+end

data/spec/supports/test_source.rb

@@ -2,8 +2,8 @@ module TestSource
   def self.source(name, &block)
     source = Smoke::Origin.new(name, &block || Proc.new {})
     source.items = [
-      {:head => "Platypus"},
-      {:head => "Kangaroo"}
+      {:head => "Animal: Platypus"},
+      {:head => "Animal: Kangaroo"}
     ]
     return source
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: benschwarz-smoke
 version: !ruby/object:Gem::Version 
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors: 
 - Ben Schwarz
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-09 00:00:00 -07:00
+date: 2009-06-15 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -100,7 +100,7 @@ requirements: []
 rubyforge_project: 
 rubygems_version: 1.2.0
 signing_key: 
-specification_version: 3
+specification_version: 2
 summary: smoke is a DSL that allows you to take data from YQL, RSS / Atom
 test_files: []