samlown-couchrest 0.35

data/lib/couchrest/core/design.rb

@@ -0,0 +1,79 @@
+module CouchRest  
+  class Design < Document
+    def view_by *keys
+      opts = keys.pop if keys.last.is_a?(Hash)
+      opts ||= {}
+      self['views'] ||= {}
+      method_name = "by_#{keys.join('_and_')}"
+      
+      if opts[:map]
+        view = {}
+        view['map'] = opts.delete(:map)
+        if opts[:reduce]
+          view['reduce'] = opts.delete(:reduce)
+          opts[:reduce] = false
+        end
+        self['views'][method_name] = view
+      else
+        doc_keys = keys.collect{|k|"doc['#{k}']"} # this is where :require => 'doc.x == true' would show up
+        key_emit = doc_keys.length == 1 ? "#{doc_keys.first}" : "[#{doc_keys.join(', ')}]"
+        guards = opts.delete(:guards) || []
+        guards.concat doc_keys
+        map_function = <<-JAVASCRIPT
+function(doc) {
+  if (#{guards.join(' && ')}) {
+    emit(#{key_emit}, null);
+  }
+}
+JAVASCRIPT
+        self['views'][method_name] = {
+          'map' => map_function
+        }
+      end
+      self['views'][method_name]['couchrest-defaults'] = opts unless opts.empty?
+      method_name
+    end
+    
+    # Dispatches to any named view.
+    # (using the database where this design doc was saved)
+    def view view_name, query={}, &block
+      view_on database, view_name, query, &block
+    end
+
+    # Dispatches to any named view in a specific database
+    def view_on db, view_name, query={}, &block
+      view_name = view_name.to_s
+      view_slug = "#{name}/#{view_name}"
+      defaults = (self['views'][view_name] && self['views'][view_name]["couchrest-defaults"]) || {}
+      db.view(view_slug, defaults.merge(query), &block)
+    end
+
+    def name
+      id.sub('_design/','') if id
+    end
+
+    def name= newname
+      self['_id'] = "_design/#{newname}"
+    end
+
+    def save
+      raise ArgumentError, "_design docs require a name" unless name && name.length > 0
+      super
+    end
+
+    private
+
+    # returns stored defaults if the there is a view named this in the design doc
+    def has_view?(view)
+      view = view.to_s
+      self['views'][view] &&
+        (self['views'][view]["couchrest-defaults"]||{})
+    end
+
+    def fetch_view view_name, opts, &block
+      database.view(view_name, opts, &block)
+    end
+
+  end
+  
+end

data/lib/couchrest/core/document.rb

@@ -0,0 +1,84 @@
+require 'delegate'
+
+module CouchRest  
+  class Document < Response
+    include CouchRest::Mixins::Attachments
+    
+    extlib_inheritable_accessor :database
+    attr_accessor :database
+    
+    # override the CouchRest::Model-wide default_database
+    # This is not a thread safe operation, do not change the model
+    # database at runtime.
+    def self.use_database(db)
+      self.database = db
+    end
+    
+    def id
+      self['_id']
+    end
+    
+    def rev
+      self['_rev']
+    end
+    
+    # returns true if the document has never been saved
+    def new?
+      !rev
+    end
+    alias :new_document? :new?
+    
+    # Saves the document to the db using create or update. Also runs the :save
+    # callbacks. Sets the <tt>_id</tt> and <tt>_rev</tt> fields based on
+    # CouchDB's response.
+    # If <tt>bulk</tt> is <tt>true</tt> (defaults to false) the document is cached for bulk save.
+    def save(bulk = false)
+      raise ArgumentError, "doc.database required for saving" unless database
+      result = database.save_doc self, bulk
+      result['ok']
+    end
+
+    # Deletes the document from the database. Runs the :delete callbacks.
+    # Removes the <tt>_id</tt> and <tt>_rev</tt> fields, preparing the
+    # document to be saved to a new <tt>_id</tt>.
+    # If <tt>bulk</tt> is <tt>true</tt> (defaults to false) the document won't 
+    # actually be deleted from the db until bulk save.
+    def destroy(bulk = false)
+      raise ArgumentError, "doc.database required to destroy" unless database
+      result = database.delete_doc(self, bulk)
+      if result['ok']
+        self['_rev'] = nil
+        self['_id'] = nil
+      end
+      result['ok']
+    end
+    
+    # copies the document to a new id. If the destination id currently exists, a rev must be provided.
+    # <tt>dest</tt> can take one of two forms if overwriting: "id_to_overwrite?rev=revision" or the actual doc
+    # hash with a '_rev' key
+    def copy(dest)
+      raise ArgumentError, "doc.database required to copy" unless database
+      result = database.copy_doc(self, dest)
+      result['ok']
+    end
+    
+    # Returns the CouchDB uri for the document
+    def uri(append_rev = false)
+      return nil if new?
+      couch_uri = "#{database.root}/#{CGI.escape(id)}"
+      if append_rev == true
+        couch_uri << "?rev=#{rev}"
+      elsif append_rev.kind_of?(Integer)
+        couch_uri << "?rev=#{append_rev}"
+      end
+      couch_uri
+    end
+    
+    # Returns the document's database
+    def database
+      @database || self.class.database
+    end
+    
+  end
+  
+end

data/lib/couchrest/core/http_abstraction.rb

@@ -0,0 +1,48 @@
+require 'couchrest/core/adapters/restclient'
+
+# Abstraction layet for HTTP communications.
+#                                           
+# By defining a basic API that CouchRest is relying on,
+# it allows for easy experimentations and implementations of various libraries.
+# 
+# Most of the API is based on the RestClient API that was used in the early version of CouchRest.
+#
+module HttpAbstraction
+  
+  # here is the list of exception expected by CouchRest
+  # please convert the underlying errors in this set of known
+  # exceptions.
+  class ResourceNotFound < StandardError; end
+  class RequestFailed < StandardError; end
+  class RequestTimeout < StandardError; end
+  class ServerBrokeConnection < StandardError; end
+  class Conflict < StandardError; end
+  
+  
+  # # Here is the API you need to implement if you want to write a new adapter
+  # # See adapters/restclient.rb for more information.
+  #
+  # def self.proxy=(url)
+  # end
+  #      
+  # def self.proxy
+  # end
+  #
+  # def self.get(uri, headers=nil)
+  # end
+  # 
+  # def self.post(uri, payload, headers=nil)
+  # end
+  # 
+  # def self.put(uri, payload, headers=nil)
+  # end
+  # 
+  # def self.delete(uri, headers=nil)
+  # end
+  # 
+  # def self.copy(uri, headers)
+  # end    
+ 
+end
+
+HttpAbstraction.extend(RestClientAdapter::API)

data/lib/couchrest/core/response.rb

@@ -0,0 +1,16 @@
+module CouchRest
+  class Response < Hash
+    def initialize(pkeys = {})
+      pkeys ||= {}
+      pkeys.each do |k,v|
+        self[k.to_s] = v
+      end
+    end
+    def []=(key, value)
+      super(key.to_s, value)
+    end
+    def [](key)
+      super(key.to_s)
+    end
+  end
+end

data/lib/couchrest/core/rest_api.rb

@@ -0,0 +1,49 @@
+module RestAPI
+
+  def put(uri, doc = nil)
+    payload = doc.to_json if doc
+    begin
+      JSON.parse(HttpAbstraction.put(uri, payload))
+    rescue Exception => e
+      if $DEBUG
+        raise "Error while sending a PUT request #{uri}\npayload: #{payload.inspect}\n#{e}"
+      else
+        raise e
+      end
+    end
+  end
+
+  def get(uri)
+    begin
+      JSON.parse(HttpAbstraction.get(uri), :max_nesting => false)
+    rescue => e
+      if $DEBUG
+        raise "Error while sending a GET request #{uri}\n: #{e}"
+      else
+        raise e
+      end
+    end
+  end
+
+  def post(uri, doc = nil)
+    payload = doc.to_json if doc
+    begin
+      JSON.parse(HttpAbstraction.post(uri, payload))
+    rescue Exception => e
+      if $DEBUG
+        raise "Error while sending a POST request #{uri}\npayload: #{payload.inspect}\n#{e}"
+      else
+        raise e
+      end
+    end
+  end
+
+  def delete(uri)
+    JSON.parse(HttpAbstraction.delete(uri))
+  end
+
+  def copy(uri, destination) 
+    JSON.parse(HttpAbstraction.copy(uri, {'Destination' => destination}))
+  end 
+
+end

data/lib/couchrest/core/server.rb

@@ -0,0 +1,88 @@
+module CouchRest
+  class Server
+    attr_accessor :uri, :uuid_batch_count, :available_databases
+    def initialize(server = 'http://127.0.0.1:5984', uuid_batch_count = 1000)
+      @uri = server
+      @uuid_batch_count = uuid_batch_count
+    end
+  
+    # Lists all "available" databases.
+    # An available database, is a database that was specified
+    # as avaiable by your code.
+    # It allows to define common databases to use and reuse in your code
+    def available_databases
+      @available_databases ||= {}
+    end
+    
+    # Adds a new available database and create it unless it already exists
+    #
+    # Example:
+    #
+    # @couch = CouchRest::Server.new
+    # @couch.define_available_database(:default, "tech-blog")
+    #
+    def define_available_database(reference, db_name, create_unless_exists = true)
+      available_databases[reference.to_sym] = create_unless_exists ? database!(db_name) : database(db_name)
+    end
+  
+    # Checks that a database is set as available
+    #
+    # Example:
+    #
+    # @couch.available_database?(:default)
+    #
+    def available_database?(ref_or_name)
+      ref_or_name.is_a?(Symbol) ? available_databases.keys.include?(ref_or_name) : available_databases.values.map{|db| db.name}.include?(ref_or_name)
+    end
+  
+    def default_database=(name, create_unless_exists = true)
+      define_available_database(:default, name, create_unless_exists = true)
+    end
+    
+    def default_database
+      available_databases[:default]
+    end
+  
+    # Lists all databases on the server
+    def databases
+      CouchRest.get "#{@uri}/_all_dbs"
+    end
+  
+    # Returns a CouchRest::Database for the given name
+    def database(name)
+      CouchRest::Database.new(self, name)
+    end
+  
+    # Creates the database if it doesn't exist
+    def database!(name)
+      create_db(name) rescue nil
+      database(name)
+    end
+  
+    # GET the welcome message
+    def info
+      CouchRest.get "#{@uri}/"
+    end
+
+    # Create a database
+    def create_db(name)
+      CouchRest.put "#{@uri}/#{name}"
+      database(name)
+    end
+
+    # Restart the CouchDB instance
+    def restart!
+      CouchRest.post "#{@uri}/_restart"
+    end
+
+    # Retrive an unused UUID from CouchDB. Server instances manage caching a list of unused UUIDs.
+    def next_uuid(count = @uuid_batch_count)
+      @uuids ||= []
+      if @uuids.empty?
+        @uuids = CouchRest.get("#{@uri}/_uuids?count=#{count}")["uuids"]
+      end
+      @uuids.pop
+    end
+
+  end
+end

data/lib/couchrest/core/view.rb

@@ -0,0 +1,4 @@
+module CouchRest
+  class View
+  end
+end

data/lib/couchrest/helper/pager.rb

@@ -0,0 +1,103 @@
+module CouchRest
+  class Pager
+    attr_accessor :db
+    def initialize db
+      @db = db
+    end
+    
+    def all_docs(limit=100, &block)
+      startkey = nil
+      oldend = nil
+      
+      while docrows = request_all_docs(limit+1, startkey)        
+        startkey = docrows.last['key']
+        docrows.pop if docrows.length > limit
+        if oldend == startkey
+          break
+        end
+        yield(docrows)
+        oldend = startkey
+      end
+    end
+    
+    def key_reduce(view, limit=2000, firstkey = nil, lastkey = nil, &block)
+      # start with no keys
+      startkey = firstkey
+      # lastprocessedkey = nil
+      keepgoing = true
+      
+      while keepgoing && viewrows = request_view(view, limit, startkey)
+        startkey = viewrows.first['key']
+        endkey = viewrows.last['key']
+
+        if (startkey == endkey)
+          # we need to rerequest to get a bigger page
+          # so we know we have all the rows for that key
+          viewrows = @db.view(view, :key => startkey)['rows']
+          # we need to do an offset thing to find the next startkey
+          # otherwise we just get stuck
+          lastdocid = viewrows.last['id']
+          fornextloop = @db.view(view, :startkey => startkey, :startkey_docid => lastdocid, :limit => 2)['rows']
+
+          newendkey = fornextloop.last['key']
+          if (newendkey == endkey)
+            keepgoing = false
+          else
+            startkey = newendkey
+          end
+          rows = viewrows
+        else
+          rows = []
+          for r in viewrows
+            if (lastkey && r['key'] == lastkey)
+              keepgoing = false
+              break
+            end
+            break if (r['key'] == endkey)
+            rows << r
+          end   
+          startkey = endkey
+        end
+        
+        key = :begin
+        values = []
+
+        rows.each do |r|
+          if key != r['key']
+            # we're on a new key, yield the old first and then reset
+            yield(key, values) if key != :begin
+            key = r['key']
+            values = []
+          end
+          # keep accumulating
+          values << r['value']
+        end
+        yield(key, values)
+
+      end
+    end
+
+    private
+    
+    def request_all_docs limit, startkey = nil
+      opts = {}
+      opts[:limit] = limit if limit
+      opts[:startkey] = startkey if startkey      
+      results = @db.documents(opts)
+      rows = results['rows']
+      rows unless rows.length == 0
+    end
+
+    def request_view view, limit = nil, startkey = nil, endkey = nil
+      opts = {}
+      opts[:limit] = limit if limit
+      opts[:startkey] = startkey if startkey
+      opts[:endkey] = endkey if endkey
+      
+      results = @db.view(view, opts)
+      rows = results['rows']
+      rows unless rows.length == 0
+    end
+
+  end
+end

data/lib/couchrest/helper/streamer.rb

@@ -0,0 +1,51 @@
+module CouchRest
+  class Streamer
+    attr_accessor :db
+    def initialize db
+      @db = db
+    end
+    
+    # Stream a view, yielding one row at a time. Shells out to <tt>curl</tt> to keep RAM usage low when you have millions of rows.
+    def view name, params = nil, &block
+      urlst = if /^_/.match(name) then
+        "#{@db.root}/#{name}"
+      else
+        name = name.split('/')
+        dname = name.shift
+        vname = name.join('/')
+        "#{@db.root}/_design/#{dname}/_view/#{vname}"
+      end
+      url = CouchRest.paramify_url urlst, params
+      # puts "stream #{url}"
+      first = nil
+      IO.popen("curl --silent \"#{url}\"") do |view|
+        first = view.gets # discard header
+        while line = view.gets 
+          row = parse_line(line)
+          block.call row unless row.nil? # last line "}]" discarded
+        end
+      end
+      parse_first(first)
+    end
+    
+    private
+    
+    def parse_line line
+      return nil unless line
+      if /(\{.*\}),?/.match(line.chomp)
+        JSON.parse($1)
+      end
+    end
+
+    def parse_first first
+      return nil unless first
+      parts = first.split(',')
+      parts.pop
+      line = parts.join(',')
+      JSON.parse("#{line}}")
+    rescue
+      nil
+    end
+    
+  end
+end

data/lib/couchrest/helper/upgrade.rb

@@ -0,0 +1,51 @@
+module CouchRest
+  class Upgrade
+    attr_accessor :olddb, :newdb, :dbname
+    def initialize dbname, old_couch, new_couch
+      @dbname = dbname
+      @olddb = old_couch.database dbname
+      @newdb = new_couch.database!(dbname)
+      @bulk_docs = []
+    end
+    def clone!
+      puts "#{dbname} - #{olddb.info['doc_count']} docs"
+      streamer  = CouchRest::Streamer.new(olddb)
+      streamer.view("_all_docs_by_seq") do |row|
+        load_row_docs(row) if row
+        maybe_flush_bulks
+      end
+      flush_bulks!
+    end
+    
+    private
+    
+    def maybe_flush_bulks
+      flush_bulks! if (@bulk_docs.length > 99)
+    end
+    
+    def flush_bulks!
+      url = CouchRest.paramify_url "#{@newdb.uri}/_bulk_docs", {:all_or_nothing => true}
+      puts "posting #{@bulk_docs.length} bulk docs to #{url}"
+      begin
+        CouchRest.post url, {:docs => @bulk_docs}      
+        @bulk_docs = []
+      rescue Exception => e
+        puts e.response
+        raise e
+      end
+    end
+    
+    def load_row_docs(row)
+      results = @olddb.get(row["id"], {:open_revs => "all", :attachments => true})
+      results.select{|r|r["ok"]}.each do |r|
+        doc = r["ok"]
+        if /^_/.match(doc["_id"]) && !/^_design/.match(doc["_id"])
+          puts "invalid docid #{doc["_id"]} -- trimming"
+          doc["_id"] = doc["_id"].sub('_','')
+        end
+        doc.delete('_rev')
+        @bulk_docs << doc
+      end
+    end
+  end
+end