tripod 0.4.2 → 0.5.0

data/lib/tripod/errors/sparql_response_too_large.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+module Tripod::Errors
+
+  # sparql response too large.
+  class SparqlResponseTooLarge < StandardError
+  end
+
+end

data/lib/tripod/errors.rb

@@ -7,5 +7,6 @@ require 'tripod/errors/rdf_type_not_set'
 require 'tripod/errors/validations'
 require 'tripod/errors/rdf_parse_failed'
 
+require 'tripod/errors/sparql_response_too_large'
 require 'tripod/errors/bad_sparql_request'
 require 'tripod/errors/bad_data_request'

data/lib/tripod/sparql_client.rb

@@ -21,26 +21,24 @@ module Tripod::SparqlClient
           Rails.logger.debug "TRIPOD: About to run query:"
           Rails.logger.debug sparql
         end
+
         params = {:query => sparql}.merge(extra_params)
-        hdrs = {:accept => accept_header, :params => params}
-
-        make_the_call = -> {
-          response = RestClient::Request.execute(
-            :method => :get,
-            :url => Tripod.query_endpoint,
-            :headers => hdrs,
-            :timeout => Tripod.timeout_seconds
-          )
-          response.body
+        request_url = Tripod.query_endpoint + '?' + params.to_query
+        streaming_opts = {:accept => accept_header, :timeout_seconds => Tripod.timeout_seconds}
+        streaming_opts.merge!(:response_limit_bytes => Tripod.response_limit_bytes) if Tripod.response_limit_bytes
+
+        # Hash.to_query from active support core extensions
+        stream_data = -> {
+          Tripod::Streaming.get_data(request_url, streaming_opts)
         }
 
         if Tripod.cache_store # if a cache store is configured
           # SHA-2 the key to keep the it within the small limit for many cache stores (e.g. Memcached is 250bytes)
           # Note: SHA2's are pretty certain to be unique http://en.wikipedia.org/wiki/SHA-2.
           key = 'SPARQL-QUERY-' + Digest::SHA2.hexdigest([extra_params, accept_header, sparql].join(" "))
-          Tripod.cache_store.fetch(key, &make_the_call)
+          Tripod.cache_store.fetch(key, &stream_data)
         else
-          make_the_call.call()
+          stream_data.call()
         end
 
       rescue RestClient::BadRequest => e

data/lib/tripod/streaming.rb

@@ -0,0 +1,39 @@
+require 'net/http'
+
+module Tripod
+  module Streaming
+
+    # stream data from a url
+    # opts
+    #  :accept => "*/*"
+    #  :timeout_seconds = 10
+    #  :limit_bytes = nil
+    def self.get_data(request_url, opts={})
+
+      accept = opts[:accept] || "*/*"
+      timeout_in_seconds = opts[:timeout_seconds] || 10
+      limit_in_bytes = opts[:response_limit_bytes]
+
+      uri = URI(request_url)
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.read_timeout = timeout_in_seconds
+
+      total_bytes = 0
+      response_string = ""
+
+      http.request_get(uri.request_uri, 'Accept' => accept) do |res|
+        res.read_body do |seg|
+          total_bytes += seg.size
+          response_string += seg.to_s
+          # if there's a limit, stop when we reach it
+          raise Tripod::Errors::SparqlResponseTooLarge.new if limit_in_bytes && (total_bytes > limit_in_bytes)
+        end
+      end
+
+      response_string
+
+    end
+
+  end
+end

data/lib/tripod/version.rb

@@ -1,3 +1,3 @@
 module Tripod
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/lib/tripod.rb

@@ -51,6 +51,9 @@ module Tripod
   mattr_accessor :timeout_seconds
   @@timeout_seconds = 30
 
+  mattr_accessor :response_limit_bytes
+  @@response_limit_bytes = 4.megabytes
+
   mattr_accessor :cache_store
 
   # Use +configure+ to override configuration in an app, (defaults shown)
@@ -59,6 +62,7 @@ module Tripod
   #     config.update_endpoint = 'http://127.0.0.1:3030/tripod/update'
   #     config.query_endpoint = 'http://127.0.0.1:3030/tripod/sparql'
   #     config.timeout_seconds = 30#
+  #     config.response_limit_bytes = 4.megabytes # omit for no limit
   #     config.cache_store = nil #e.g Tripod::CacheStores::MemcachedCacheStore.new('localhost:11211')
   #       # note: if using memcached, make sure you set the -I (slab size) to big enough to store each result
   #       # and set the -m (total size) to something quite big (or the cache will recycle too often).
@@ -73,6 +77,7 @@ end
 require 'tripod/cache_stores/memcached_cache_store'
 
 require "tripod/extensions"
+require "tripod/streaming"
 require "tripod/sparql_client"
 require "tripod/sparql_query"
 require "tripod/resource_collection"

data/spec/tripod/sparql_client_spec.rb

@@ -4,6 +4,21 @@ describe Tripod::SparqlClient do
   describe Data do
     let(:data) { 'some-data-goes-here' }
 
+    describe "Query#query" do
+      it "should use Tripod::Streaming to get the data" do
+        query = "SELECT * WHERE {?s ?p ?o}"
+        Tripod::Streaming.should_receive(:get_data).with(
+          Tripod.query_endpoint + "?query=#{CGI.escape(query)}",
+          {
+            accept: "application/sparql-results+json",
+            timeout_seconds: Tripod.timeout_seconds,
+            response_limit_bytes: Tripod.response_limit_bytes
+          }
+        ).and_return("some data")
+        Tripod::SparqlClient::Query.query(query, "application/sparql-results+json")
+      end
+    end
+
     describe "Data#append" do
       it "should add the graph uri to the configured data endpoint" do
         RestClient::Request.should_receive(:execute).with(hash_including(url: 'http://127.0.0.1:3030/tripod-test/data?graph=http://example.com/foo'))

data/spec/tripod/streaming_spec.rb

@@ -0,0 +1,70 @@
+require "spec_helper"
+
+describe Tripod::Streaming do
+
+  let(:url) { 'http://example.com' }
+  let(:response_length) { 64.kilobytes }
+
+  before do
+    WebMock.enable!
+    stub_http_request(:get, url).with(:headers => {'Accept' => "*/*"}).to_return(:body => ("0" * response_length))
+  end
+
+  describe ".get_data" do
+    it "should make a request to the url passed in" do
+      Tripod::Streaming.get_data(url)
+    end
+
+    context "with timeout option" do
+      it "should set the read_timeout to that value" do
+        Net::HTTP.any_instance.should_receive(:read_timeout=).with(28)
+        Tripod::Streaming.get_data(url, :timeout_seconds => 28)
+      end
+    end
+
+    context "with no timeout option" do
+      it "should set the read_timeout to the default (10s)" do
+        Net::HTTP.any_instance.should_receive(:read_timeout=).with(10)
+        Tripod::Streaming.get_data(url)
+      end
+    end
+
+    context "with an accept header option" do
+      it "should use that header for the request " do
+        stub_http_request(:get, url).with(:headers => {'Accept' => "application/json"})
+        Tripod::Streaming.get_data(url, :accept => 'application/json')
+      end
+    end
+
+    # these tests actually download remote resources (from jQuery's CDN) to test the streaming bits
+    # TODO: move this out so it doesn't run with the normal rake task??
+    context "streaming" do
+      before { WebMock.disable! }
+
+      context "with no limit" do
+
+        it "should return the full body" do
+          full_response_length = RestClient.get('http://code.jquery.com/jquery-1.9.1.js').body.length
+          response = Tripod::Streaming.get_data('http://code.jquery.com/jquery-1.9.1.js')
+          response.length.should == full_response_length
+        end
+      end
+
+      context "with a limit" do
+        it "should raise an exception if it's bigger than the limit" do
+          unlimited_response = Tripod::Streaming.get_data('http://code.jquery.com/jquery-1.9.1.js')
+          unlimited_response.length.should > 48.kilobytes
+
+          lambda {
+            Tripod::Streaming.get_data('http://code.jquery.com/jquery-1.9.1.js', :response_limit_bytes => 32.kilobytes)
+          }.should raise_error(Tripod::Errors::SparqlResponseTooLarge)
+        end
+      end
+    end
+  end
+
+  after do
+    WebMock.disable!
+  end
+
+end

data/tripod.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
   gem.rubyforge_project         = "tripod"
 
   gem.add_dependency "rest-client"
-  gem.add_dependency "activemodel", "~> 3.1"
+  gem.add_dependency "activemodel", "~> 3.2"
   gem.add_dependency "equivalent-xml"
   gem.add_dependency "rdf", "~> 1.0"
   gem.add_dependency "rdf-rdfxml"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tripod
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-21 00:00:00.000000000 Z
+date: 2013-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &70208187066620 !ruby/object:Gem::Requirement
+  requirement: &70308451617140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -23,21 +23,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70208187066620
+  version_requirements: *70308451617140
 - !ruby/object:Gem::Dependency
   name: activemodel
-  requirement: &70208187061260 !ruby/object:Gem::Requirement
+  requirement: &70308451616420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '3.2'
   type: :runtime
   prerelease: false
-  version_requirements: *70208187061260
+  version_requirements: *70308451616420
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
-  requirement: &70208187091120 !ruby/object:Gem::Requirement
+  requirement: &70308451632020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -45,10 +45,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70208187091120
+  version_requirements: *70308451632020
 - !ruby/object:Gem::Dependency
   name: rdf
-  requirement: &70208187100360 !ruby/object:Gem::Requirement
+  requirement: &70308451631020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -56,10 +56,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70208187100360
+  version_requirements: *70308451631020
 - !ruby/object:Gem::Dependency
   name: rdf-rdfxml
-  requirement: &70208187094320 !ruby/object:Gem::Requirement
+  requirement: &70308451627060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -67,10 +67,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70208187094320
+  version_requirements: *70308451627060
 - !ruby/object:Gem::Dependency
   name: rdf-n3
-  requirement: &70208186978440 !ruby/object:Gem::Requirement
+  requirement: &70308451625140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -78,10 +78,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70208186978440
+  version_requirements: *70308451625140
 - !ruby/object:Gem::Dependency
   name: rdf-json
-  requirement: &70208186977380 !ruby/object:Gem::Requirement
+  requirement: &70308451624460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -89,10 +89,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70208186977380
+  version_requirements: *70308451624460
 - !ruby/object:Gem::Dependency
   name: json-ld
-  requirement: &70208186975660 !ruby/object:Gem::Requirement
+  requirement: &70308451639760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -100,10 +100,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70208186975660
+  version_requirements: *70308451639760
 - !ruby/object:Gem::Dependency
   name: guid
-  requirement: &70208186974980 !ruby/object:Gem::Requirement
+  requirement: &70308451637040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -111,10 +111,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70208186974980
+  version_requirements: *70308451637040
 - !ruby/object:Gem::Dependency
   name: dalli
-  requirement: &70208186973620 !ruby/object:Gem::Requirement
+  requirement: &70308451634220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -122,7 +122,7 @@ dependencies:
         version: '2.6'
   type: :runtime
   prerelease: false
-  version_requirements: *70208186973620
+  version_requirements: *70308451634220
 description: RDF ruby ORM
 email:
 - ric@swirrl.com
@@ -152,6 +152,7 @@ files:
 - lib/tripod/errors/rdf_parse_failed.rb
 - lib/tripod/errors/rdf_type_not_set.rb
 - lib/tripod/errors/resource_not_found.rb
+- lib/tripod/errors/sparql_response_too_large.rb
 - lib/tripod/errors/uri_not_set.rb
 - lib/tripod/errors/validations.rb
 - lib/tripod/extensions.rb
@@ -168,6 +169,7 @@ files:
 - lib/tripod/sparql_client.rb
 - lib/tripod/sparql_query.rb
 - lib/tripod/state.rb
+- lib/tripod/streaming.rb
 - lib/tripod/version.rb
 - spec/app/models/person.rb
 - spec/app/models/resource.rb
@@ -186,6 +188,7 @@ files:
 - spec/tripod/sparql_client_spec.rb
 - spec/tripod/sparql_query_spec.rb
 - spec/tripod/state_spec.rb
+- spec/tripod/streaming_spec.rb
 - tripod.gemspec
 homepage: http://github.com/Swirrl/tripod
 licenses: []
@@ -229,4 +232,5 @@ test_files:
 - spec/tripod/sparql_client_spec.rb
 - spec/tripod/sparql_query_spec.rb
 - spec/tripod/state_spec.rb
+- spec/tripod/streaming_spec.rb
 has_rdoc: