dpla-couchrest 1.2.1.pre.dpla

data/lib/couchrest/design.rb

@@ -0,0 +1,91 @@
+module CouchRest
+  class Design < Document
+
+    def view_by *keys
+      opts = keys.pop if keys.last.is_a?(Hash)
+      opts ||= {}
+      self['views'] ||= {}
+      method_name = "by_#{keys.join('_and_')}"
+
+      if opts[:map]
+        view = {}
+        view['map'] = opts.delete(:map)
+        view['reduce'] = opts.delete(:reduce) if opts[:reduce]
+        self['views'][method_name] = view
+      else
+        doc_keys = keys.collect{|k| "doc['#{k}']"}
+        key_emit = doc_keys.length == 1 ? "#{doc_keys.first}" : "[#{doc_keys.join(', ')}]"
+        guards = opts.delete(:guards) || []
+        guards += doc_keys.map{|k| "(#{k} != null)"} unless opts.delete(:allow_nil)
+        guards << 'true' if guards.empty?
+        map_function = <<-JAVASCRIPT
+function(doc) {
+  if (#{guards.join(' && ')}) {
+    emit(#{key_emit}, null);
+  }
+}
+JAVASCRIPT
+        self['views'][method_name] = {
+          'map' => map_function
+        }
+      end
+      self['views'][method_name]['couchrest-defaults'] = opts unless opts.empty?
+      method_name
+    end
+
+    # Dispatches to any named view.
+    # (using the database where this design doc was saved)
+    def view view_name, query={}, &block
+      view_on database, view_name, query, &block
+    end
+
+    # Dispatches to any named view in a specific database
+    def view_on db, view_name, query = {}, &block
+      raise ArgumentError, "View query options must be set as symbols!" if query.keys.find{|k| k.is_a?(String)}
+      view_name = view_name.to_s
+      view_slug = "#{name}/#{view_name}"
+      # Set the default query options
+      query = view_defaults(view_name).merge(query)
+      # Ensure reduce is set if dealing with a reduceable view
+      # This is a requirement of CouchDB.
+      query[:reduce] ||= false if can_reduce_view?(view_name)
+      db.view(view_slug, query, &block)
+    end
+
+    def name
+      id.sub('_design/','') if id
+    end
+
+    def name= newname
+      self['_id'] = "_design/#{newname}"
+    end
+
+    def save
+      raise ArgumentError, "_design docs require a name" unless name && name.length > 0
+      super
+    end
+
+    # Return the hash of default values to include in all queries sent
+    # to a view from couchrest.
+    def view_defaults(name)
+      (self['views'][name.to_s] && self['views'][name.to_s]["couchrest-defaults"]) || {}
+    end
+
+    # Returns true or false if the view is available.
+    def has_view?(name)
+      !self['views'][name.to_s].nil?
+    end
+
+    # Check if the view has a reduce method defined.
+    def can_reduce_view?(name)
+      has_view?(name) && !self['views'][name.to_s]['reduce'].to_s.empty?
+    end
+
+    private
+
+    def fetch_view view_name, opts, &block
+      database.view(view_name, opts, &block)
+    end
+
+  end
+end

data/lib/couchrest/document.rb

@@ -0,0 +1,105 @@
+
+#
+# CouchRest::Document
+#
+# Provides basic functions for controlling documents returned from
+# the CouchDB database and provides methods to act as a wrapper around
+# a Hash of @_attributes.
+#
+# The idea is to provide the basic functionality of a Hash, just
+# enought to support the needs of CouchRest, but not inherit all
+# of the functionality found in a basic Hash.
+#
+# A Response is similar to Rails' HashWithIndifferentAccess as all
+# requests will convert the keys into Symbols and be stored in the
+# master hash as such.
+#
+
+module CouchRest
+  class Document
+    include CouchRest::Attributes
+    include CouchRest::Attachments
+    extend CouchRest::InheritableAttributes
+
+    couchrest_inheritable_accessor :database
+    attr_accessor :database
+
+    def id
+      self['_id']
+    end
+    def id=(id)
+      self['_id'] = id
+    end
+    def rev
+      self['_rev']
+    end
+
+    # returns true if the document has never been saved
+    def new?
+      !rev
+    end
+    alias :new_document? :new?
+
+    # Saves the document to the db using create or update. Also runs the :save
+    # callbacks. Sets the <tt>_id</tt> and <tt>_rev</tt> fields based on
+    # CouchDB's response.
+    # If <tt>bulk</tt> is <tt>true</tt> (defaults to false) the document is cached for bulk save.
+    def save(bulk = false)
+      raise ArgumentError, "doc.database required for saving" unless database
+      result = database.save_doc self, bulk
+      result['ok']
+    end
+
+    # Deletes the document from the database. Runs the :delete callbacks.
+    # Removes the <tt>_id</tt> and <tt>_rev</tt> fields, preparing the
+    # document to be saved to a new <tt>_id</tt>.
+    # If <tt>bulk</tt> is <tt>true</tt> (defaults to false) the document won't 
+    # actually be deleted from the db until bulk save.
+    def destroy(bulk = false)
+      raise ArgumentError, "doc.database required to destroy" unless database
+      result = database.delete_doc(self, bulk)
+      if result['ok']
+        self['_rev'] = nil
+        self['_id'] = nil
+      end
+      result['ok']
+    end
+
+    # copies the document to a new id. If the destination id currently exists, a rev must be provided.
+    # <tt>dest</tt> can take one of two forms if overwriting: "id_to_overwrite?rev=revision" or the actual doc
+    # hash with a '_rev' key
+    def copy(dest)
+      raise ArgumentError, "doc.database required to copy" unless database
+      result = database.copy_doc(self, dest)
+      result['ok']
+    end
+
+    # Returns the CouchDB uri for the document
+    def uri(append_rev = false)
+      return nil if new?
+      couch_uri = "#{database.root}/#{CGI.escape(id)}"
+      if append_rev == true
+        couch_uri << "?rev=#{rev}"
+      elsif append_rev.kind_of?(Integer)
+        couch_uri << "?rev=#{append_rev}"
+      end
+      couch_uri
+    end
+
+    # Returns the document's database
+    def database
+      @database || self.class.database
+    end
+
+    class << self
+      # override the CouchRest::Model-wide default_database
+      # This is not a thread safe operation, do not change the model
+      # database at runtime.
+      def use_database(db)
+        self.database = db
+      end
+    end
+
+  end
+
+end

data/lib/couchrest/helper/attachments.rb

@@ -0,0 +1,29 @@
+module CouchRest
+  module Attachments
+  
+    # saves an attachment directly to couchdb
+    def put_attachment(name, file, options={})
+      raise ArgumentError, "doc must be saved" unless self.rev
+      raise ArgumentError, "doc.database required to put_attachment" unless database
+      result = database.put_attachment(self, name, file, options)
+      self['_rev'] = result['rev']
+      result['ok']
+    end
+
+    # returns an attachment's data
+    def fetch_attachment(name)
+      raise ArgumentError, "doc must be saved" unless self.rev
+      raise ArgumentError, "doc.database required to put_attachment" unless database
+      database.fetch_attachment(self, name)
+    end
+
+    # deletes an attachment directly from couchdb
+    def delete_attachment(name, force=false)
+      raise ArgumentError, "doc.database required to delete_attachment" unless database
+      result = database.delete_attachment(self, name, force)
+      self['_rev'] = result['rev']
+      result['ok']
+    end
+  
+  end
+end

data/lib/couchrest/helper/pager.rb

@@ -0,0 +1,103 @@
+module CouchRest
+  class Pager
+    attr_accessor :db
+    def initialize db
+      @db = db
+    end
+    
+    def all_docs(limit=100, &block)
+      startkey = nil
+      oldend = nil
+      
+      while docrows = request_all_docs(limit+1, startkey)        
+        startkey = docrows.last['key']
+        docrows.pop if docrows.length > limit
+        if oldend == startkey
+          break
+        end
+        yield(docrows)
+        oldend = startkey
+      end
+    end
+    
+    def key_reduce(view, limit=2000, firstkey = nil, lastkey = nil, &block)
+      # start with no keys
+      startkey = firstkey
+      # lastprocessedkey = nil
+      keepgoing = true
+      
+      while keepgoing && viewrows = request_view(view, limit, startkey)
+        startkey = viewrows.first['key']
+        endkey = viewrows.last['key']
+
+        if (startkey == endkey)
+          # we need to rerequest to get a bigger page
+          # so we know we have all the rows for that key
+          viewrows = @db.view(view, :key => startkey)['rows']
+          # we need to do an offset thing to find the next startkey
+          # otherwise we just get stuck
+          lastdocid = viewrows.last['id']
+          fornextloop = @db.view(view, :startkey => startkey, :startkey_docid => lastdocid, :limit => 2)['rows']
+
+          newendkey = fornextloop.last['key']
+          if (newendkey == endkey)
+            keepgoing = false
+          else
+            startkey = newendkey
+          end
+          rows = viewrows
+        else
+          rows = []
+          for r in viewrows
+            if (lastkey && r['key'] == lastkey)
+              keepgoing = false
+              break
+            end
+            break if (r['key'] == endkey)
+            rows << r
+          end   
+          startkey = endkey
+        end
+        
+        key = :begin
+        values = []
+
+        rows.each do |r|
+          if key != r['key']
+            # we're on a new key, yield the old first and then reset
+            yield(key, values) if key != :begin
+            key = r['key']
+            values = []
+          end
+          # keep accumulating
+          values << r['value']
+        end
+        yield(key, values)
+
+      end
+    end
+
+    private
+    
+    def request_all_docs limit, startkey = nil
+      opts = {}
+      opts[:limit] = limit if limit
+      opts[:startkey] = startkey if startkey      
+      results = @db.documents(opts)
+      rows = results['rows']
+      rows unless rows.length == 0
+    end
+
+    def request_view view, limit = nil, startkey = nil, endkey = nil
+      opts = {}
+      opts[:limit] = limit if limit
+      opts[:startkey] = startkey if startkey
+      opts[:endkey] = endkey if endkey
+      
+      results = @db.view(view, opts)
+      rows = results['rows']
+      rows unless rows.length == 0
+    end
+
+  end
+end

data/lib/couchrest/helper/streamer.rb

@@ -0,0 +1,60 @@
+module CouchRest
+  class Streamer
+
+    attr_accessor :default_curl_opts
+
+    def initialize
+      self.default_curl_opts = "--silent --no-buffer --tcp-nodelay -H \"Content-Type: application/json\""
+    end
+
+    def view(*args)
+      raise "CouchRest::Streamer#view is deprecated. Please use Database#view with block."
+    end
+
+    def get(url, &block)
+      open_pipe("curl #{default_curl_opts} \"#{url}\"", &block)
+    end
+
+    def post(url, params = {}, &block)
+      open_pipe("curl #{default_curl_opts} -d \"#{escape_quotes(MultiJson.encode(params))}\" \"#{url}\"", &block)
+    end
+
+    protected
+
+    def escape_quotes(data)
+      data.gsub(/"/, '\"')
+    end
+
+    def open_pipe(cmd, &block)
+      first = nil
+      prev = nil
+      IO.popen(cmd) do |f|
+        first = f.gets # discard header
+        while line = f.gets 
+          row = parse_line(line)
+          block.call row unless row.nil? # last line "}]" discarded
+          prev = line
+        end
+      end
+
+      raise RestClient::ServerBrokeConnection if $? && $?.exitstatus != 0
+
+      parse_first(first, prev)
+    end
+
+    def parse_line line
+      return nil unless line
+      if /(\{.*\}),?/.match(line.chomp)
+        MultiJson.decode($1)
+      end
+    end
+
+    def parse_first first, last
+      return nil unless first
+      header = MultiJson.decode(last ? first + last : first)
+      header.delete("rows")
+      header
+    end
+
+  end
+end

data/lib/couchrest/helper/upgrade.rb

@@ -0,0 +1,51 @@
+module CouchRest
+  class Upgrade
+    attr_accessor :olddb, :newdb, :dbname
+    def initialize dbname, old_couch, new_couch
+      @dbname = dbname
+      @olddb = old_couch.database dbname
+      @newdb = new_couch.database!(dbname)
+      @bulk_docs = []
+    end
+    def clone!
+      puts "#{dbname} - #{olddb.info['doc_count']} docs"
+      streamer  = CouchRest::Streamer.new(olddb)
+      streamer.view("_all_docs_by_seq") do |row|
+        load_row_docs(row) if row
+        maybe_flush_bulks
+      end
+      flush_bulks!
+    end
+    
+    private
+    
+    def maybe_flush_bulks
+      flush_bulks! if (@bulk_docs.length > 99)
+    end
+    
+    def flush_bulks!
+      url = CouchRest.paramify_url "#{@newdb.uri}/_bulk_docs", {:all_or_nothing => true}
+      puts "posting #{@bulk_docs.length} bulk docs to #{url}"
+      begin
+        CouchRest.post url, {:docs => @bulk_docs}      
+        @bulk_docs = []
+      rescue Exception => e
+        puts e.response
+        raise e
+      end
+    end
+    
+    def load_row_docs(row)
+      results = @olddb.get(row["id"], {:open_revs => "all", :attachments => true})
+      results.select{|r|r["ok"]}.each do |r|
+        doc = r["ok"]
+        if /^_/.match(doc["_id"]) && !/^_design/.match(doc["_id"])
+          puts "invalid docid #{doc["_id"]} -- trimming"
+          doc["_id"] = doc["_id"].sub('_','')
+        end
+        doc.delete('_rev')
+        @bulk_docs << doc
+      end
+    end
+  end
+end

data/lib/couchrest/middlewares/logger.rb

@@ -0,0 +1,263 @@
+####################################
+# USAGE
+#
+# in your rack.rb file
+# require this file and then:
+#
+# couch = CouchRest.new
+# LOG_DB = couch.database!('couchrest-logger')
+# use CouchRest::Logger, LOG_DB
+# 
+# Note:
+# to require just this middleware, if you have the gem installed do:
+# require 'couchrest/middlewares/logger'
+#
+# For log processing examples, see examples at the bottom of this file
+
+module CouchRest
+  class Logger 
+    
+    def self.log
+      Thread.current["couchrest.logger"] ||= {:queries => []}
+    end
+        
+    def initialize(app, db=nil)
+      @app = app
+      @db = db
+    end
+    
+    def self.record(log_info)
+      log[:queries] << log_info
+    end
+    
+    def log
+      Thread.current["couchrest.logger"] ||= {:queries => []}
+    end
+    
+    def reset_log
+      Thread.current["couchrest.logger"] = nil
+    end
+ 
+    def call(env)
+      reset_log
+      log['started_at'] = Time.now
+      log['env'] = env
+      log['url'] = 'http://' + env['HTTP_HOST'] + env['REQUEST_URI'] 
+      response = @app.call(env)
+      log['ended_at'] = Time.now
+      log['duration'] = log['ended_at'] - log['started_at']
+      # let's report the log in a different thread so we don't slow down the app
+      @db ? Thread.new(@db, log){|db, rlog| db.save_doc(rlog);} : p(log.inspect)
+      response
+    end
+  end
+end
+
+# inject our logger into key RestClient methods
+module RestClient 
+ 
+  def self.get(uri, headers=nil)
+    start_query = Time.now
+    log = {:method => :get, :uri => uri, :headers => headers}
+    response = super(uri, headers=nil)
+    end_query = Time.now
+    log[:duration] = (end_query - start_query)
+    CouchRest::Logger.record(log)
+    response
+  end 
+  
+  def self.post(uri, payload, headers=nil)
+    start_query = Time.now
+    log = {:method => :post, :uri => uri, :payload =>  (payload ? (MultiJson.decode(payload) rescue 'parsing error') : nil), :headers => headers}
+    response = super(uri, payload, headers=nil)
+    end_query = Time.now
+    log[:duration] = (end_query - start_query)
+    CouchRest::Logger.record(log)
+    response
+  end
+   
+  def self.put(uri, payload, headers=nil)
+    start_query = Time.now
+    log = {:method => :put, :uri => uri, :payload => (payload ? (MultiJson.decode(payload) rescue 'parsing error') : nil), :headers => headers}
+    response = super(uri, payload, headers=nil)
+    end_query = Time.now
+    log[:duration] = (end_query - start_query)
+    CouchRest::Logger.record(log)
+    response
+  end
+  
+  def self.delete(uri, headers=nil)
+    start_query = Time.now
+    log = {:method => :delete, :uri => uri, :headers => headers}
+    response = super(uri, headers=nil)
+    end_query = Time.now
+    log[:duration] = (end_query - start_query)
+    CouchRest::Logger.record(log)
+    response
+  end   
+  
+end 
+
+
+# Advanced usage example 
+#
+#
+# # DB VIEWS 
+# by_url = {
+#   :map => 
+#     "function(doc) {
+#       if(doc['url']){ emit(doc['url'], 1) };
+#     }",
+#  :reduce => 
+#     'function (key, values, rereduce) {
+#        return(sum(values));
+#     };'
+# }
+# req_duration = {
+#   :map => 
+#     "function(doc) {
+#       if(doc['duration']){ emit(doc['url'], doc['duration']) };
+#     }",
+#  :reduce => 
+#     'function (key, values, rereduce) {
+#        return(sum(values)/values.length);
+#     };'
+# }
+# 
+# query_duration = {
+#   :map => 
+#     "function(doc) {
+#           if(doc['queries']){ 
+#             doc.queries.forEach(function(query){
+#     if(query['duration'] && query['method']){ 
+#               emit(query['method'], query['duration'])
+#     }
+#             });
+#           };
+#          }" ,
+#    :reduce => 
+#       'function (key, values, rereduce) {
+#         return(sum(values)/values.length);
+#       };'
+# }
+# 
+# action_queries = {
+#   :map => 
+#     "function(doc) {
+#           if(doc['queries']){
+#              emit(doc['url'], doc['queries'].length)
+#           };
+#          }",
+#  :reduce => 
+#     'function (key, values, rereduce) {
+#       return(sum(values)/values.length);
+#     };'
+# }  
+# 
+# action_time_spent_in_db = {
+#   :map => 
+#     "function(doc) {
+#           if(doc['queries']){
+#             var totalDuration = 0;
+#             doc.queries.forEach(function(query){
+#               totalDuration += query['duration']
+#             })
+#              emit(doc['url'], totalDuration)
+#           };
+#          }",
+#  :reduce => 
+#     'function (key, values, rereduce) {
+#       return(sum(values)/values.length);
+#     };'
+# }
+# 
+# show_queries =  %Q~function(doc, req) {
+#                    var body = ""
+#                    body += "<h1>" + doc['url'] + "</h1>"
+#                    body += "<h2>Request duration in seconds: " + doc['duration'] + "</h2>"
+#                    body += "<h3>" + doc['queries'].length + " queries</h3><ul>"
+#                    if (doc.queries){
+#                      doc.queries.forEach(function(query){
+#                        body += "<li>"+ query['uri'] +"</li>"
+#                      });
+#                    };
+#                    body += "</ul>"
+#                    if(doc){ return { body: body} }
+#                  }~
+# 
+# 
+# couch = CouchRest.new
+# LOG_DB = couch.database!('couchrest-logger')   
+# design_doc = LOG_DB.get("_design/stats") rescue nil
+# LOG_DB.delete_doc design_doc rescue nil
+# LOG_DB.save_doc({
+#   "_id" => "_design/stats",
+#   :views => {
+#     :by_url           => by_url,
+#     :request_duration => req_duration,
+#     :query_duration   => query_duration,
+#     :action_queries   => action_queries,
+#     :action_time_spent_in_db => action_time_spent_in_db
+#   },
+#  :shows => {
+#    :queries => show_queries
+#  }
+# })
+# 
+# module CouchRest
+#   class Logger
+#     
+#     def self.roundup(value)
+#       begin
+#         value = Float(value)
+#         (value * 100).round.to_f / 100
+#       rescue
+#         value
+#       end
+#     end
+#     
+#     # Usage example:
+#     # CouchRest::Logger.average_request_duration(LOG_DB)['rows'].first['value']
+#     def self.average_request_duration(db)
+#       raw = db.view('stats/request_duration', :reduce => true)
+#       (raw.has_key?('rows') && !raw['rows'].empty?) ? roundup(raw['rows'].first['value']) : 'not available yet'
+#     end
+#     
+#     def self.average_query_duration(db)
+#       raw =  db.view('stats/query_duration', :reduce => true)
+#       (raw.has_key?('rows') && !raw['rows'].empty?) ? roundup(raw['rows'].first['value']) : 'not available yet'
+#     end   
+#     
+#     def self.average_get_query_duration(db)
+#       raw = db.view('stats/query_duration', :key => 'get', :reduce => true)
+#       (raw.has_key?('rows') && !raw['rows'].empty?) ? roundup(raw['rows'].first['value']) : 'not available yet'
+#     end  
+#     
+#     def self.average_post_query_duration(db)
+#       raw = db.view('stats/query_duration', :key => 'post', :reduce => true)
+#       (raw.has_key?('rows') && !raw['rows'].empty?) ? roundup(raw['rows'].first['value']) : 'not available yet'
+#     end 
+#     
+#     def self.average_queries_per_action(db)
+#       raw = db.view('stats/action_queries', :reduce => true)
+#       (raw.has_key?('rows') && !raw['rows'].empty?) ? roundup(raw['rows'].first['value']) : 'not available yet'
+#     end
+#     
+#     def self.average_db_time_per_action(db)
+#      raw = db.view('stats/action_time_spent_in_db', :reduce => true)
+#      (raw.has_key?('rows') && !raw['rows'].empty?) ? roundup(raw['rows'].first['value']) : 'not available yet'
+#     end
+#     
+#     def self.stats(db)
+#       Thread.new(db){|db|
+#         puts "===  STATS  ===\n"
+#         puts "average request duration: #{average_request_duration(db)}\n"
+#         puts "average query duration: #{average_query_duration(db)}\n"
+#         puts "average queries per action : #{average_queries_per_action(db)}\n"
+#         puts "average time spent in DB (per action): #{average_db_time_per_action(db)}\n"
+#         puts "===============\n"   
+#      }
+#     end
+#     
+#   end
+# end