datacatalog 0.3.8 → 0.4.0

data/Rakefile

@@ -5,7 +5,7 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
     gem.name = "datacatalog"
-    gem.version = '0.3.8'
+    gem.version = '0.4.0'
     gem.rubyforge_project = "datacatalog"
     gem.summary = %Q{Client for the National Data Catalog API}
     gem.description = %Q{A Ruby client library for the National Data Catalog API}

data/datacatalog.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog}
-  s.version = "0.3.8"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Luigi Montanez", "David James"]
-  s.date = %q{2009-12-02}
+  s.date = %q{2009-12-03}
   s.description = %q{A Ruby client library for the National Data Catalog API}
   s.email = %q{luigi@sunlightfoundation.com}
   s.extra_rdoc_files = [
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
      "Rakefile",
      "datacatalog.gemspec",
      "lib/base.rb",
+     "lib/cursor.rb",
      "lib/datacatalog.rb",
      "lib/main.rb",
      "lib/resources/about.rb",

data/lib/base.rb

@@ -51,6 +51,15 @@ module DataCatalog
       response
     end
     
+    def self.cursor(uri, query_hash)
+      Cursor.new({
+        :klass      => self,
+        :query_hash => query_hash,
+        :response   => http_get(uri, :query => query_hash),
+        :uri        => uri,
+      })
+    end
+    
     def self.error(response)
       parsed_body = JSON.parse(response.body)
       if parsed_body.empty?
@@ -64,8 +73,9 @@ module DataCatalog
       "Unable to parse: #{response.body.inspect}"
     end
     
-    def self.many(response)
-      response.map { |item| new(item) }
+    def self._first(response)
+      item = response['members'][0]
+      item.blank? ? nil : new(item)
     end
 
     def self.one(response)
@@ -78,13 +88,14 @@ module DataCatalog
     end
     
     def self.filterize(conditions)
-      conditions.map do |k, v|
+      filtered_conditions = conditions.map do |k, v|
         "#{k}" + if v.is_a?(Regexp)
           %(:"#{v.source}")
         else
           %(="#{v}")
         end
-      end.join(" ")
+      end
+      filtered_conditions.join(" ")
     end
     
     def method_missing(method_name, *args)

data/lib/cursor.rb

@@ -0,0 +1,98 @@
+module DataCatalog
+
+  class Cursor
+    
+    attr_reader :document_count
+    attr_reader :page_count
+    attr_reader :page_size
+    
+    include Enumerable
+    
+    def initialize(options)
+      @klass      = options[:klass]
+      @query_hash = options[:query_hash]
+      @response   = options[:response]
+      @uri        = options[:uri]
+
+      @page_size      = @response['page_size']
+      @document_count = @response['document_count']
+      @members        = members(@response)
+      @page_count     = @response['page_count']
+    end
+    
+    def [](i)
+      member_at(i)
+    end
+
+    def each
+      @document_count.times { |i| yield member_at(i) }
+    end
+    
+    def first
+      member_at(0)
+    end
+    
+    def last
+      member_at(@document_count - 1)
+    end
+
+    def length
+      @document_count
+    end
+
+    def page(page_number)
+      page_indices(page_number).map { |i| member_at(i) }
+    end
+    
+    def size
+      @members.size
+    end
+    
+    protected
+    
+    def fetch_members_near_index(index)
+      page_number = page_for_index(index)
+      response = response_for_page(page_number)
+      fetched = members(response)
+      range = page_indices(page_number)
+      range.each_with_index do |i, j|
+        @members[i] = fetched[j]
+      end
+      fetched.length
+    end
+    
+    def member_at(index)
+      if index < 0
+        raise "index (#{index}) must be >= 0"
+      end
+      cached = @members[index]
+      return nil if @document_count == 0
+      return cached if cached
+      fetch_members_near_index(index)
+      fetched = @members[index]
+      raise "cannot find find member at index #{index}" unless fetched
+      fetched
+    end
+
+    def members(response)
+      response['members'].map { |x| @klass.new(x) }
+    end
+    
+    def page_for_index(index)
+      (index / @page_size) + 1
+    end
+
+    def page_indices(page_number)
+      min = (page_number - 1) * @page_size
+      max = [min + @page_size - 1,  @document_count - 1].min
+      (min .. max)
+    end
+
+    def response_for_page(page_number)
+      query_hash = @query_hash.merge(:page => page_number)
+      @klass.http_get(@uri, :query => query_hash)
+    end
+
+  end
+  
+end

data/lib/datacatalog.rb

@@ -6,4 +6,5 @@ require 'mash'
 
 require File.dirname(__FILE__) + '/main'
 require File.dirname(__FILE__) + '/base'
+require File.dirname(__FILE__) + '/cursor'
 Dir.glob(File.dirname(__FILE__) + '/resources/*.rb').each { |f| require f }

data/lib/resources/api_key.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class ApiKey < Base
     
     def self.all(user_id, conditions={})
-      many(http_get(uri(user_id), :query => query_hash(conditions)))
+      cursor(uri(user_id), query_hash(conditions))
     end
 
     def self.create(user_id, params={})
@@ -15,7 +15,7 @@ module DataCatalog
     end
 
     def self.first(user_id, conditions={})
-      one(http_get(uri(user_id), :query => query_hash(conditions)).first)
+      _first(http_get(uri(user_id), :query => query_hash(conditions)))
     end
 
     def self.get(user_id, id)

data/lib/resources/comment.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class Comment < Base
     
     def self.all(conditions={})
-      many(http_get(uri, :query => query_hash(conditions)))
+      cursor(uri, query_hash(conditions))
     end
 
     def self.create(params={})

data/lib/resources/document.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class Document < Base
     
     def self.all(conditions={})
-      many(http_get(uri, :query => query_hash(conditions)))
+      cursor(uri, query_hash(conditions))
     end
 
     def self.get(id)

data/lib/resources/favorite.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class Favorite < Base
     
     def self.all(conditions={})
-      many(http_get(uri, :query => query_hash(conditions)))
+      cursor(uri, query_hash(conditions))
     end
 
     def self.create(params={})

data/lib/resources/note.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class Note < Base
     
     def self.all(conditions={})
-      many(http_get(uri, :query => query_hash(conditions)))
+      cursor(uri, query_hash(conditions))
     end
 
     def self.get(id)

data/lib/resources/organization.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class Organization < Base
     
     def self.all(conditions={})
-      many(http_get(uri, :query => query_hash(conditions)))
+      cursor(uri, query_hash(conditions))
     end
 
     def self.create(params={})
@@ -15,7 +15,7 @@ module DataCatalog
     end
 
     def self.first(conditions={})
-      one(http_get(uri, :query => query_hash(conditions)).first)
+      _first(http_get(uri, :query => query_hash(conditions)))
     end
 
     def self.get(id)
@@ -23,7 +23,7 @@ module DataCatalog
     end
 
     def self.search(term)
-      many(http_get(uri, :query => { :search => term }))
+      cursor(uri, { :search => term })
     end
 
     def self.update(id, params={})

data/lib/resources/rating.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class Rating < Base
     
     def self.all(conditions={})
-      many(http_get(uri, :query => query_hash(conditions)))
+      cursor(uri, query_hash(conditions))
     end
 
     def self.create(params={})

data/lib/resources/source.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class Source < Base
     
     def self.all(conditions={})
-      many(http_get(uri, :query => query_hash(conditions)))
+      cursor(uri, query_hash(conditions))
     end
 
     def self.create(params={})
@@ -15,7 +15,7 @@ module DataCatalog
     end
 
     def self.first(conditions={})
-      one(http_get(uri, :query => query_hash(conditions)).first)
+      _first(http_get(uri, :query => query_hash(conditions)))
     end
 
     def self.get(id)
@@ -23,7 +23,7 @@ module DataCatalog
     end
 
     def self.search(term)
-      many(http_get(uri, :query => { :search => term }))
+      cursor(uri, { :search => term })
     end
 
     def self.update(id, params={})

data/lib/resources/user.rb

@@ -3,7 +3,7 @@ module DataCatalog
   class User < Base
     
     def self.all(conditions={})
-      many(http_get(uri, :query => query_hash(conditions)))
+      cursor(uri, query_hash(conditions))
     end
 
     def self.create(params={})
@@ -15,7 +15,7 @@ module DataCatalog
     end
 
     def self.first(conditions={})
-      one(http_get(uri, :query => query_hash(conditions)).first)
+      _first(http_get(uri, :query => query_hash(conditions)))
     end
     
     def self.get(id)

data/spec/base_spec.rb

@@ -85,25 +85,6 @@ describe Base do
     end
   end
 
-  describe ".many" do
-    it "should create an object from a filled array" do
-      array = Base.many([
-        {
-          :name  => "Carl Malamud",
-          :email => "no-spam-carl@media.org"
-        },
-        {
-          :name  => "Ellen Miller",
-          :email => "no-spam-ellen@sunlightfoundation.com"
-        }
-      ])
-      array.map do |item|
-        item.should be_an_instance_of(Base)
-      end
-      array.map(&:name).should == ["Carl Malamud", "Ellen Miller"]
-    end    
-  end
-
   describe ".one" do
     it "should create an object from a filled hash" do
       hash = Base.one({

data/spec/comment_spec.rb

@@ -6,13 +6,15 @@ describe Comment do
   before do
     setup_api
     clean_slate
-    
-    @user = User.create(:name  => "Ted Smith",
-                        :email => "ted@email.com")
-    
-    @source = Source.create(:title       => "Some FCC Data",
-                            :url         => "http://fcc.gov/somedata.csv",
-                            :source_type => "dataset")
+    @user = User.create(
+      :name  => "Ted Smith",
+      :email => "ted@email.com"
+    )
+    @source = Source.create(
+      :title       => "Some FCC Data",
+      :url         => "http://fcc.gov/somedata.csv",
+      :source_type => "dataset"
+    )
   end
   
   describe ".create" do

data/spec/document_spec.rb

@@ -6,18 +6,21 @@ describe Document do
   before do
     setup_api
     clean_slate
-    
-    @user = User.create(:name  => "Ted Smith",
-                        :email => "ted@email.com")
-    
-    @source = Source.create(:title       => "Some FCC Data",
-                            :url         => "http://fcc.gov/somedata.csv",
-                            :source_type => "dataset")
-    
+    @user = User.create(
+      :name  => "Ted Smith",
+      :email => "ted@email.com"
+    )
+    @source = Source.create(
+      :title       => "Some FCC Data",
+      :url         => "http://fcc.gov/somedata.csv",
+      :source_type => "dataset"
+    )
     DataCatalog.with_key(@user.primary_api_key) do
-      @document = Document.create(:source_id => @source.id, :text => "This is community documentation.")
+      @document = Document.create(
+        :source_id => @source.id,
+        :text      => "This is community documentation."
+      )
     end
-    
   end
   
   describe ".create" do
@@ -49,11 +52,11 @@ describe Document do
   
   describe ".all" do
   
-    it "should return a collection of all the source's documents" do
+    it "should return an enumeration of documents" do
       documents = Document.all(:source_id => @source.id)
-  
-      documents.should be_an_instance_of(Array)
-      documents[0].should be_an_instance_of(Document)
+      documents.each do |o|
+        o.should be_an_instance_of(Document)
+      end
     end
     
   end # describe ".all"

data/spec/favorite_spec.rb

@@ -6,22 +6,18 @@ describe Favorite do
   before do
     setup_api
     clean_slate
-    
     @user = User.create(
       :name  => "Ted Smith",
       :email => "ted@email.com"
     )
-    
     @source = Source.create(
       :title       => "Some FCC Data",
       :url         => "http://fcc.gov/somedata.csv",
       :source_type => "dataset"
     )
-    
     DataCatalog.with_key(@user.primary_api_key) do
       @favorite = Favorite.create(:source_id => @source.id)
     end
-    
   end
   
   describe ".create" do
@@ -50,12 +46,12 @@ describe Favorite do
   end # describe ".get"
   
   describe ".all" do
-  
-    it "should return a collection of all the user's favorites" do
+
+    it "should return an enumeration of favorites" do
       favorites = Favorite.all(:user_id => @user.id)
-  
-      favorites.should be_an_instance_of(Array)
-      favorites[0].should be_an_instance_of(Favorite)
+      favorites.each do |o|
+        o.should be_an_instance_of(Favorite)
+      end
     end
     
   end # describe ".all"

data/spec/note_spec.rb

@@ -6,18 +6,21 @@ describe Note do
   before do
     setup_api
     clean_slate
-    
-    @user = User.create(:name  => "Ted Smith",
-                        :email => "ted@email.com")
-    
-    @source = Source.create(:title       => "Some FCC Data",
-                            :url         => "http://fcc.gov/somedata.csv",
-                            :source_type => "dataset")
-    
+    @user = User.create(
+      :name  => "Ted Smith",
+      :email => "ted@email.com"
+    )
+    @source = Source.create(
+      :title       => "Some FCC Data",
+      :url         => "http://fcc.gov/somedata.csv",
+      :source_type => "dataset"
+    )
     DataCatalog.with_key(@user.primary_api_key) do
-      @note = Note.create(:source_id => @source.id, :text => "This is my note.")
+      @note = Note.create(
+        :source_id => @source.id,
+        :text => "This is my note."
+      )
     end
-    
   end
   
   describe ".create" do
@@ -48,12 +51,12 @@ describe Note do
   end # describe ".get"
   
   describe ".all" do
-  
-    it "should return a collection of all the user's notes" do
+
+    it "should return an enumeration of notes" do
       notes = Note.all(:user_id => @user.id)
-  
-      notes.should be_an_instance_of(Array)
-      notes[0].should be_an_instance_of(Note)
+      notes.each do |o|
+        o.should be_an_instance_of(Note)
+      end
     end
     
   end # describe ".all"

data/spec/organization_spec.rb

@@ -4,7 +4,11 @@ include DataCatalog
 describe Organization do
 
   def create_3_organizations
-    @organization_names = ["Federal Communications Commission", "Federal Election Commission", "Department of State"]
+    @organization_names = [
+      "Federal Communications Commission",
+      "Federal Election Commission",
+      "Department of State"
+    ]
     @organization_names.each do |name|
       Organization.create(:name => name, :org_type => "governmental")
     end
@@ -22,24 +26,34 @@ describe Organization do
     end
     
     it "should return an enumeration of Organizations" do
-      @organizations = Organization.all
-      @organizations.each do |o|
+      Organization.all.each do |o|
         o.should be_an_instance_of(Organization)
       end
     end
     
     it "should return an enumeration with organization name set" do
-      @organizations = Organization.all
-      @organizations.map(&:name).sort.should == @organization_names.sort
+      Organization.all.map(&:name).sort.should == @organization_names.sort
     end
     
     it "should return the matching organizations when options are passed in" do
-      @organizations = Organization.all(:name => "Federal Communications Commission")
-      @organizations.map(&:name).should == ["Federal Communications Commission"]
+      orgs = Organization.all(:name => "Federal Communications Commission")
+      orgs.map(&:name).should == ["Federal Communications Commission"]
     end
     
   end # describe ".all"
 
+  describe ".search" do
+    before do
+      create_3_organizations
+    end
+    
+    it "should return correct search result" do
+      @organizations = Organization.search("Federal")
+      @organizations.size.should == 2
+      @organizations.first.name.should == "Federal Communications Commission"
+    end
+  end # describe organizations
+
   describe ".first" do
     
     before do
@@ -47,8 +61,8 @@ describe Organization do
     end
     
     it "should return the matching organization when options are passed in" do
-      @organization = Organization.first(:name => "Federal Communications Commission")
-      @organization.name.should == "Federal Communications Commission"
+      org = Organization.first(:name => "Federal Communications Commission")
+      org.name.should == "Federal Communications Commission"
     end
     
   end # describe ".first"
@@ -76,9 +90,9 @@ describe Organization do
   describe ".create" do
     
     it "should create a new organization when valid params are passed in" do
-      @organization = Organization.create(:name => "Federal Communications Commission", :org_type => "governmental")
-      @organization.should be_an_instance_of(Organization)
-      @organization.name.should == "Federal Communications Commission"
+      org = Organization.create(:name => "Federal Communications Commission", :org_type => "governmental")
+      org.should be_an_instance_of(Organization)
+      org.name.should == "Federal Communications Commission"
     end
     
     it "should raise BadRequest when a bad URL is passed in" do

data/spec/rating_spec.rb

@@ -6,18 +6,24 @@ describe Rating do
   before do
     setup_api
     clean_slate
-    
-    @user = User.create(:name  => "Ted Smith",
-                        :email => "ted@email.com")
-    @user2 = User.create(:name  => "Chad Johnson",
-                        :email => "chad@email.com")
-    
-    @source = Source.create(:title       => "Some FCC Data",
-                            :url         => "http://fcc.gov/somedata.csv",
-                            :source_type => "dataset")
-    
+    @user = User.create(
+      :name  => "Ted Smith",
+      :email => "ted@email.com"
+    )
+    @user2 = User.create(
+      :name  => "Chad Johnson",
+      :email => "chad@email.com"
+    )
+    @source = Source.create(
+      :title       => "Some FCC Data",
+      :url         => "http://fcc.gov/somedata.csv",
+      :source_type => "dataset"
+    )
     DataCatalog.with_key(@user.primary_api_key) do
-      @comment = Comment.create(:source_id => @source.id, :text => "This is a useful comment.")
+      @comment = Comment.create(
+        :source_id => @source.id,
+        :text      => "This is a useful comment."
+      )
     end
   end
   

data/spec/source_spec.rb

@@ -20,6 +20,16 @@ module SourceHelpers
       })
     end
   end
+  
+  def create_30_sources
+    (1..30).each do |n|
+      Source.create({
+        :title        => "Dataset #{n.to_s}",
+        :url          => "http://awesome.gov/dataset#{n.to_s}.xml",
+        :source_type  => "dataset"
+      })
+    end
+  end
 end
 
 describe Source do
@@ -35,17 +45,40 @@ describe Source do
       create_3_sources
       @sources = Source.all
     end
+    
+    it "should return an Enumerable object" do
+      @sources.is_a?(Enumerable).should == true
+    end
   
     it "should return an enumeration of sources" do
       @sources.each do |source|
         source.should be_an_instance_of(Source)
       end
     end
-  
+    
     it "should return correct titles" do
       expected = ["FCC Data", "NASA Data", "DOE Data"]
       @sources.map(&:title).sort.should == expected.sort
     end
+    
+    
+    describe "with cursor" do
+    
+      it "should raise an error on bad index" do
+        executing do
+          @sources.page(0)
+        end.should raise_error(RuntimeError)
+      end
+    
+      it "should return 20 objects when there are more than 20" do
+        create_30_sources
+        sources = Source.all
+        sources.size.should == 20
+        sources.page(1).size.should == 20
+        sources.page(2).size.should == 13
+      end
+      
+    end
   end
 
   describe ".all with conditions" do
@@ -77,7 +110,7 @@ describe Source do
       @sources.first.title.should == "FCC Data"
     end
   end
-
+  
   describe ".create" do
     it "should create a new source from basic params" do
       source = create_source

data/spec/spec_helper.rb

@@ -7,19 +7,20 @@ end
 
 alias :executing :lambda
 
-def clean_slate
+KLASSES = [
+  DataCatalog::Source,
+  DataCatalog::Organization
+]
 
+def clean_slate
   DataCatalog::User.all.each do |u|
     DataCatalog::User.destroy(u.id) unless u.name == "Primary Admin"
   end
-
-  classes = [DataCatalog::Source, DataCatalog::Organization]
-  classes.each do |class_constant|
-    class_constant.all.each do |instance|
-      class_constant.destroy(instance.id)
-    end
+  KLASSES.each do |klass|
+    to_delete = []
+    klass.all.each { |instance| to_delete << instance.id }
+    to_delete.each { |id| klass.destroy(id) }
   end
-
 end
 
 def mangle(string)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: datacatalog
 version: !ruby/object:Gem::Version 
-  version: 0.3.8
+  version: 0.4.0
 platform: ruby
 authors: 
 - Luigi Montanez
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-02 00:00:00 -05:00
+date: 2009-12-03 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -89,6 +89,7 @@ files:
 - Rakefile
 - datacatalog.gemspec
 - lib/base.rb
+- lib/cursor.rb
 - lib/datacatalog.rb
 - lib/main.rb
 - lib/resources/about.rb