analysand 1.0.1 → 1.1.0

data/CHANGELOG

@@ -0,0 +1,16 @@
+Issue numbers refer to issues on Analysand's Github tracker:
+https://github.com/yipdw/analysand/issues
+
+1.1.0 (2012-11-03)
+------------------
+
+* View streaming (#3)
+* ChangeWatchers now pass credentials when checking CouchDB status (#4)
+* Some code organization cleanups
+* require "analysand" now loads the Database and Instance classes
+
+
+1.0.1 (2012-10-01)
+------------------
+
+* Initial release

data/README

@@ -12,9 +12,10 @@ Features:
 * GET, PUT, DELETE on databases
 * GET, PUT, DELETE, HEAD, COPY on documents
 * GET, PUT on document attachments
-* GET on views
+* GET, POST on views
 * POST /_session
 * POST /_bulk_docs
+* View streaming
 * Celluloid::IO-based change feed watchers
 * Cookie and HTTP Basic authentication for all of the above
 * Database objects can be safely shared across threads
@@ -31,7 +32,7 @@ information.
 
 Naturally, we hang with all the cool kids:
 
-* Travis CI: http://travis-ci.org/#!/yipdw/analysand
+* Travis CI: https://travis-ci.org/#!/yipdw/analysand
 * Code Climate: https://codeclimate.com/github/yipdw/analysand
 * Gemnasium: https://gemnasium.com/yipdw/analysand
 

data/analysand.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |gem|
   gem.email         = ["yipdw@member.fsf.org"]
   gem.description   = %q{A terrible burden for a couch}
   gem.summary       = %q{A CouchDB client of dubious worth}
-  gem.homepage      = ""
+  gem.homepage      = "https://github.com/yipdw/analysand"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -15,10 +15,13 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Analysand::VERSION
 
+  gem.required_ruby_version = '>= 1.9'
+
   gem.add_dependency 'celluloid', '>= 0.12'
   gem.add_dependency 'celluloid-io'
   gem.add_dependency 'http_parser.rb'
   gem.add_dependency 'json'
+  gem.add_dependency 'json-stream'
   gem.add_dependency 'net-http-persistent'
   gem.add_dependency 'rack'
   gem.add_dependency 'yajl-ruby'

data/bin/analysand

@@ -2,13 +2,12 @@
 
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 
-require 'analysand/database'
+require 'analysand'
 require 'irb'
 require 'uri'
 
 $URI = URI('http://localhost:5984/analysand_test')
 
-
 def make_db(uri = $URI)
   Analysand::Database.new(uri)
 end
@@ -21,7 +20,7 @@ Type make_db to make an Analysand::Database object.  The default URI is
 
 To point at different databases, supply a URI object to make_db, e.g.
 
-    make_db(URI('https://couchdb.example.org:6984/supersekrit'))"
+    make_db(URI('https://couchdb.example.org:6984/supersekrit'))
 ------------------------------------------------------------------------------
 END
 

data/lib/analysand.rb

@@ -1,5 +1,3 @@
-require "analysand/version"
-
-module Analysand
-  # Your code goes here...
-end
+require 'analysand/database'
+require 'analysand/instance'
+require 'analysand/version'

data/lib/analysand/change_watcher.rb

@@ -129,7 +129,9 @@ module Analysand
     #       ok && my_other_test
     #     end
     def connection_ok
-      test_http_connection(changes_feed_uri)
+      test_http_connection(changes_feed_uri) do |req|
+        customize_request(req)
+      end
     end
 
     def start

data/lib/analysand/connection_testing.rb

@@ -7,9 +7,15 @@ module Analysand
     ##
     # Issues a HEAD request to the given URI.  If it responds with a success or
     # redirection code, returns true; otherwise, returns false.
+    #
+    # If a block is given, yields the request object for customization.
     def test_http_connection(uri)
       begin
-        resp = Net::HTTP.start(uri.host, uri.port) { |h| h.head(uri.request_uri) }
+        resp = Net::HTTP.start(uri.host, uri.port) do |h|
+          req = Net::HTTP::Head.new(uri.request_uri)
+          yield req if block_given?
+          h.request(req)
+        end
 
         case resp
         when Net::HTTPSuccess then true

data/lib/analysand/database.rb

@@ -1,9 +1,11 @@
-require 'analysand/bulk_response'
 require 'analysand/errors'
+require 'analysand/reading'
 require 'analysand/response'
-require 'analysand/view_response'
+require 'analysand/viewing'
+require 'analysand/writing'
 require 'net/http/persistent'
 require 'rack/utils'
+require 'uri'
 
 module Analysand
   ##
@@ -141,15 +143,26 @@ module Analysand
   #     vdb.view('video/recent', :key => ['member1'])
   #     vdb.view('video/by_artist', :startkey => 'a', :endkey => 'b')
   #
-  # Keys are automatically JSON-encoded.  The view method returns a
-  # ViewResponse, which may be accessed like this:
+  # Keys are automatically JSON-encoded, as required by CouchDB.
+  #
+  # If you're running into problems with large key sets generating very long
+  # query strings, you can use POST mode (CouchDB 0.9+):
+  #
+  #     vdb.view('video/by_artist', :keys => many_keys, :post => true)
+  #
+  # If you're reading many records from a view, you may want to stream them
+  # in:
+  #
+  #     vdb.view('video/all', :stream => true)
+  #
+  # View data and metadata may be accessed as follows:
   #
   #     resp = vdb.view('video/recent', :limit => 10)
   #     resp.total_rows   # => 16
   #     resp.offset       # => 0
-  #     resp.rows         # => [ { 'id' => ... }, ... } ]
+  #     resp.rows         # => an Enumerable
   #
-  # See ViewResponse for more details.
+  # See ViewResponse and StreamingViewResponse for more details.
   #
   # You can also use view!, which will raise Analysand::CannotAccessView on a
   # non-success response.
@@ -251,8 +264,9 @@ module Analysand
   # is also done.
   class Database
     include Rack::Utils
-
-    JSON_VALUE_PARAMETERS = %w(key keys startkey endkey).map(&:to_sym)
+    include Reading
+    include Viewing
+    include Writing
 
     attr_reader :http
     attr_reader :uri
@@ -268,21 +282,18 @@ module Analysand
     def initialize(uri)
       raise InvalidURIError, 'You must supply an absolute URI' unless uri.absolute?
 
-      @http = Net::HTTP::Persistent.new('catalog_database')
+      @http = Net::HTTP::Persistent.new('analysand_database')
       @uri = uri
 
-      # URI.join (used to calculate a document URI) will replace the database
-      # name unless we make it clear that the database is part of the path
+      # Document IDs and other database bits are appended to the URI path,
+      # so we need to make sure that it ends in a /.
       unless uri.path.end_with?('/')
         uri.path += '/'
       end
     end
 
     def ping(credentials = nil)
-      req = Net::HTTP::Get.new(uri.to_s)
-      set_credentials(req, credentials)
-
-      Response.new(http.request(uri, req))
+      Response.new _get('', credentials)
     end
 
     def status(credentials = nil)
@@ -293,130 +304,8 @@ module Analysand
       http.shutdown
     end
 
-    def put(doc_id, doc, credentials = nil, options = {})
-      query = options
-      headers = { 'Content-Type' => 'application/json' }
-
-      Response.new _put(doc_id, credentials, options, headers, doc.to_json)
-    end
-
-    def put!(doc_id, doc, credentials = nil, options = {})
-      put(doc_id, doc, credentials, options).tap do |resp|
-        raise ex(DocumentNotSaved, resp) unless resp.success?
-      end
-    end
-
-    def ensure_full_commit(credentials = nil, options = {})
-      headers = { 'Content-Type' => 'application/json' }
-
-      Response.new _post('_ensure_full_commit', credentials, options, headers, {}.to_json)
-    end
-
-    def bulk_docs(docs, credentials = nil, options = {})
-      headers = { 'Content-Type' => 'application/json' }
-      body = { 'docs' => docs }
-      body['all_or_nothing'] = true if options[:all_or_nothing]
-
-      BulkResponse.new _post('_bulk_docs', credentials, {}, headers, body.to_json)
-    end
-
-    def bulk_docs!(docs, credentials = nil, options = {})
-      bulk_docs(docs, credentials, options).tap do |resp|
-        raise ex(BulkOperationFailed, resp) unless resp.success?
-      end
-    end
-
-    def copy(source, destination, credentials = nil)
-      headers = { 'Destination' => destination }
-
-      Response.new _copy(source, credentials, {}, headers, nil)
-    end
-
-    def put_attachment(loc, io, credentials = nil, options = {})
-      query = {}
-      headers = {}
-
-      if options[:rev]
-        query['rev'] = options[:rev]
-      end
-
-      if options[:content_type]
-        headers['Content-Type'] = options[:content_type]
-      end
-
-      Response.new _put(loc, credentials, query, headers, io.read)
-    end
-
-    def delete(doc_id, rev, credentials = nil)
-      headers = { 'If-Match' => rev }
-
-      Response.new _delete(doc_id, credentials, {}, headers, nil)
-    end
-
-    def delete!(doc_id, rev, credentials = nil)
-      delete(doc_id, rev, credentials).tap do |resp|
-        raise ex(DocumentNotDeleted, resp) unless resp.success?
-      end
-    end
-
-    def get(doc_id, credentials = nil)
-      Response.new(_get(doc_id, credentials))
-    end
-
-    def get!(doc_id, credentials = nil)
-      get(doc_id, credentials).tap do |resp|
-        raise ex(CannotAccessDocument, resp) unless resp.success?
-      end
-    end
-
-    def head(doc_id, credentials = nil)
-      Response.new(_head(doc_id, credentials))
-    end
-
-    def get_attachment(loc, credentials = nil)
-      _get(loc, credentials)
-    end
-
-    def all_docs(parameters = {}, credentials = nil)
-      view('_all_docs', parameters, credentials)
-    end
-
-    def all_docs!(parameters = {}, credentials = nil)
-      view!('_all_docs', parameters, credentials)
-    end
-
-    def view(view_name, parameters = {}, credentials = nil)
-      view_path = expand_view_path(view_name)
-
-      JSON_VALUE_PARAMETERS.each do |p|
-        if parameters.has_key?(p)
-          parameters[p] = parameters[p].to_json
-        end
-      end
-
-      ViewResponse.new _get(view_path, credentials, parameters, {})
-    end
-
-    def expand_view_path(view_name)
-      if view_name.include?('/')
-        design_doc, view_name = view_name.split('/', 2)
-        "_design/#{design_doc}/_view/#{view_name}"
-      else
-        view_name
-      end
-    end
-
-    def view!(view_name, parameters = {}, credentials = nil)
-      view(view_name, parameters, credentials).tap do |resp|
-        raise ex(CannotAccessView, resp) unless resp.success?
-      end
-    end
-
     def create(credentials = nil)
-      req = Net::HTTP::Put.new(uri.to_s)
-      set_credentials(req, credentials)
-
-      Response.new(http.request(uri, req))
+      Response.new _put('', credentials)
     end
 
     def create!(credentials = nil)
@@ -426,10 +315,7 @@ module Analysand
     end
 
     def drop(credentials = nil)
-      req = Net::HTTP::Delete.new(uri.to_s)
-      set_credentials(req, credentials)
-
-      Response.new(http.request(uri, req))
+      Response.new _delete('', credentials)
     end
 
     def drop!(credentials = nil)
@@ -440,8 +326,8 @@ module Analysand
 
     %w(Head Get Put Post Delete Copy).each do |m|
       str = <<-END
-        def _#{m.downcase}(doc_id, credentials, query = {}, headers = {}, body = nil)
-          _req(Net::HTTP::#{m}, doc_id, credentials, query, headers, body)
+        def _#{m.downcase}(doc_id, credentials, query = {}, headers = {}, body = nil, block = nil)
+          _req(Net::HTTP::#{m}, doc_id, credentials, query, headers, body, block)
         end
       END
 
@@ -450,8 +336,9 @@ module Analysand
 
     ##
     # @private
-    def _req(klass, doc_id, credentials, query, headers, body)
-      uri = URI(self.uri.to_s + URI.escape(doc_id))
+    def _req(klass, doc_id, credentials, query, headers, body, block)
+      uri = self.uri.dup
+      uri.path += URI.escape(doc_id)
       uri.query = build_query(query) unless query.empty?
 
       req = klass.new(uri.request_uri)
@@ -460,7 +347,7 @@ module Analysand
       req.body = body if body && req.request_body_permitted?
       set_credentials(req, credentials)
 
-      http.request(uri, req)
+      http.request(uri, req, &block)
     end
 
     ##
@@ -479,6 +366,10 @@ module Analysand
       end
     end
 
+    def json_headers
+      { 'Content-Type' => 'application/json' }
+    end
+
     ##
     # @private
     def ex(klass, response)

data/lib/analysand/errors.rb

@@ -23,4 +23,7 @@ module Analysand
 
   class BulkOperationFailed < DatabaseError
   end
+
+  class UnexpectedViewKey < StandardError
+  end
 end

data/lib/analysand/reading.rb

@@ -0,0 +1,26 @@
+require 'analysand/errors'
+require 'analysand/response'
+
+module Analysand
+  module Reading
+    def get(doc_id, credentials = nil)
+      Response.new(_get(doc_id, credentials))
+    end
+
+    def get!(doc_id, credentials = nil)
+      get(doc_id, credentials).tap do |resp|
+        raise ex(CannotAccessDocument, resp) unless resp.success?
+      end
+    end
+
+    def head(doc_id, credentials = nil)
+      Response.new(_head(doc_id, credentials))
+    end
+
+    def get_attachment(loc, credentials = nil)
+      _get(loc, credentials)
+    end
+  end
+end
+
+# vim:ts=2:sw=2:et:tw=78

data/lib/analysand/streaming_view_response.rb

@@ -0,0 +1,100 @@
+require 'analysand/view_streaming/builder'
+require 'fiber'
+
+module Analysand
+  # Public: Controls streaming of view data.
+  #
+  # This class is meant to be used by Analysand::Database#view.  It exports the
+  # same interface as ViewResponse.
+  #
+  # Examples:
+  #
+  #     resp = db.view('view/something', :stream => true)
+  #
+  #     resp.total_rows       # => 1000000
+  #     resp.offset           # => 0
+  #     resp.rows.take(100)   # => first 100 rows
+  class StreamingViewResponse
+    include Enumerable
+
+    # Private: The HTTP response.
+    #
+    # This is set by Analysand::Database#stream_view.  The #etag and #code
+    # methods use this for header information.
+    attr_accessor :http_response
+
+    def initialize
+      @reader = Fiber.new { yield self; "" }
+      @generator = ViewStreaming::Builder.new
+
+      # Analysand::Database#stream_view issues the request.  When the response
+      # arrives, it yields control back here.  Subsequent resumes read the
+      # body.
+      #
+      # We do this to provide the response headers as soon as possible.
+      @reader.resume
+    end
+
+    def etag
+      http_response.get_fields('ETag').first.gsub('"', '')
+    end
+
+    # Public: Yields documents in the view stream.
+    #
+    # Note that #docs and #rows advance the same stream, so expect to miss half
+    # your rows if you do something like
+    #
+    #     resp.docs.zip(resp.rows)
+    #
+    # If this is a problem for you, let me know and we can work out a solution.
+    def docs
+      to_enum(:get_docs)
+    end
+
+    def get_docs
+      each { |r| yield r['doc'] if r.has_key?('doc') }
+    end
+
+    def code
+      http_response.code
+    end
+
+    def success?
+      c = code.to_i
+
+      c >= 200 && c <= 299
+    end
+
+    def total_rows
+      read until @generator.total_rows
+
+      @generator.total_rows
+    end
+
+    def offset
+      read until @generator.offset
+
+      @generator.offset
+    end
+
+    def read
+      @generator << @reader.resume
+    end
+
+    def each
+      return to_enum unless block_given?
+
+      while @reader.alive?
+        read while @reader.alive? && @generator.staged_rows.empty?
+
+        until @generator.staged_rows.empty?
+          yield @generator.staged_rows.shift
+        end
+      end
+    end
+
+    def rows
+      self
+    end
+  end
+end