couchdb-extras 0.0.1

data/bin/couchdb2couchdb

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'couch_http'
+
+ARGV.each {|couch|
+    puts "Server #{couch}"
+    get("#{couch}/_all_dbs")
+        .select { |db| !db.start_with?("_") && !db.end_with?("_history")}
+        .each { |db| 
+           puts "Database #{couch}/#{db}"
+           ARGV.select {|server| server != couch}
+               .each {|other_couch|
+                   replication = {:source=>"#{couch}/#{db}",:target=>"#{other_couch}/#{db}",:continuous=>true,:create_target=>true}
+                   puts "Replication #{replication}"
+                   r = post("#{couch}/_replicator",replication)
+                   puts "#{replication} = #{r}"
+               }
+        }
+}
+

data/bin/couchdb2elasticsearch

@@ -0,0 +1,119 @@
+#!/usr/bin/env ruby
+
+# a bot is uniq in it's arguments
+require 'base64'
+id = Base64.encode64("#{ARGV[0]}+#{ARGV[1]}").gsub("=","").gsub("\n","")
+
+# ensure no overlap
+exit 1 unless File.new("/tmp/bot_es_#{id}.lock", "a").flock(File::LOCK_EX | File::LOCK_NB) 
+
+# read args
+@couch = ARGV[0] || "http://localhost:5984";
+@es = ARGV[1] || "http://localhost:9200";
+
+require 'couch_http'
+
+def onchange(db,change)
+    if !change["doc"].has_key?("deleted") && !change["doc"].has_key?("_deleted")
+        doc = change["doc"].clone
+        doc["id"] = doc["_id"]
+        doc["rev"] = doc["_rev"]
+        doc.delete("_id")
+        doc.delete("_rev")
+        doc.delete("_attachments")
+        type = "default"
+        if doc.has_key?("metadata") && doc["metadata"].has_key?("type") 
+            type = doc["metadata"]["type"]
+        elsif doc.has_key?("type")
+            type = doc["type"]
+        end
+        response = post("#{@es}/#{db}/#{URI.encode(type)}/#{URI.encode(doc["id"])}",doc)
+        puts "post #{@es}/#{db}/#{type}/#{doc["id"]} #{response}"
+    elsif change["doc"].has_key?("_deleted") || change["doc"].has_key?("deleted")
+        doc = change["doc"].clone
+        q = URI.encode("id:\"#{doc["_id"]}\"");
+        response = delete("#{@es}/#{db}/_query?q=#{q}")
+        puts "delete #{@es}/#{db}/_query?q=#{q} #{ response }"
+    end
+end
+
+def listen_changes(db)
+    puts "listen #{db}"
+
+    system("echo 0 > /tmp/es_changes_#{db}.last_seq") #unless File.exists?("/tmp/es_changes_#{db}.last_seq") 
+
+    while true do
+        # reads last sequence
+        last_file = File.open("/tmp/es_changes_#{db}.last_seq",'r');
+        last_seq = last_file.gets.to_i;
+        last_file.close
+
+        puts "info #{db} #{last_seq}"
+
+        # get latest changes
+        changes_uri = URI("#{@couch}/#{db}/_changes?since=#{last_seq}&limit=500&feed=longpoll&include_docs=true");
+        changes = JSON.parse(Net::HTTP.get(changes_uri))['results']
+        changes.each { |change| 
+            onchange(db,change)
+            last_seq = change["seq"]
+        }
+
+        # writes last_seq
+        system("echo #{last_seq} > /tmp/es_changes_#{db}.last_seq")
+    end
+end
+
+puts "Starting..."
+
+# hold the threads
+dbs_changes = {}
+
+# listen dbs 
+Thread.new do
+    while true do
+        update = get("#{@couch}/_db_updates?feed=longpoll");
+        Thread.new do
+            ok = update["ok"]
+            db = update["db_name"]
+            type = update["type"]
+            if ok && !db.start_with?("_") && !db.end_with?("_history") then
+                if type == "created" then
+                    puts "DB created: #{db}"
+                    r = put("#{@es}/#{db}",{});
+                    puts "Creating #{@es}/#{db} = #{r}"
+                    dbs_changes[db] = Thread.new do
+                        listen_changes(db)
+                    end
+                elsif type == 'deleted'
+                    puts "DB deleted: #{db}"
+                    dbs_changes[db].stop();
+                    r = delete("#{@es}/#{db}");
+                    puts "Delete #{@es}/#{db} = #{r}"
+                    system("echo 0 > /tmp/es_changes_#{db}.last_seq")
+                end
+            end
+        end
+    end
+end
+
+# initial run
+get("#{@couch}/_all_dbs")
+     .select { |db| !db.start_with?("_") && !db.end_with?("_history")}
+     .each { |db| 
+        puts "DB existed: #{db}"
+        r=put("#{@es}/#{db}",{});
+        puts "Creating #{@es}/#{db} = #{r}"
+        dbs_changes[db] = Thread.new do
+            listen_changes(db)
+        end
+     }
+        
+
+puts "Started "
+
+while true
+    sleep 10
+end
+
+puts "Ended"
+

data/bin/couchdb2history

@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+
+# a bot is uniq in it's arguments
+require 'base64'
+id = Base64.encode64("#{ARGV[0]}+#{ARGV[1]}").gsub("=","").gsub("\n","")
+
+# ensure no overlap
+exit 1 unless File.new("/tmp/h_bot_#{id}.lock", "a").flock(File::LOCK_EX | File::LOCK_NB) 
+
+require 'couch_http'
+
+def past(couch,db,id)
+     get("#{couch}/#{db}/#{id}?revs_info=true")["_revs_info"]
+        .reverse
+        .select {|r| r["status"] == 'available'}
+        .slice([1,2])
+        .each {|r|
+            doc = get("#{couch}/#{db}/#{id}?rev=#{r["rev"]}")
+            doc["_id"] = "#{doc["_id"]}:#{doc["_rev"]}"
+            doc.delete("_rev")
+            doc.delete("_attachments")
+            response = post("#{couch}/#{db}_history",doc)
+            puts "post #{couch}/#{db}/#{doc["_id"]} = #{response}"
+        }
+end
+
+def listen_changes(couch,db)
+    puts "listen #{couch}/#{db}"
+    id = Base64.encode64("#{couch}/#{db}").gsub("=","").gsub("\n","")
+    system("echo 0 > /tmp/h_changes_#{id}.last_seq") # unless File.exists?("/tmp/h_changes_#{id}.last_seq") 
+    while true do
+        # reads last sequence
+        last_file = File.open("/tmp/h_changes_#{id}.last_seq",'r');
+        last_seq = last_file.gets.to_i;
+        last_file.close
+
+        puts "info #{couch}/#{db} (#{id}) #{last_seq}"
+
+        # get latest changes
+        changes_uri = URI("#{couch}/#{db}/_changes?since=#{last_seq}&limit=500&feed=longpoll&include_docs=true");
+        changes = JSON.parse(Net::HTTP.get(changes_uri))['results']
+        changes.each { |change| 
+            if !change["doc"].has_key?("deleted") && !change["doc"].has_key?("_deleted")
+                doc = change["doc"].clone
+                doc["_id"] = "#{doc["_id"]}:#{doc["_rev"]}"
+                doc.delete("_rev")
+                doc.delete("_attachments")
+                response = post("#{couch}/#{db}_history",doc)
+                puts "post #{couch}/#{db}/#{change["id"]} = #{response}"
+                Thread.new do
+                    past(couch,db,change["id"])
+                end
+            end
+            last_seq = change["seq"]
+        }
+
+        # writes last_seq
+        system("echo #{last_seq} > /tmp/h_changes_#{id}.last_seq")
+    end
+end
+
+puts "Starting..."
+
+ARGV.each {|arg|
+    couch = arg.clone
+    puts "Server #{couch}"
+
+    # hold the threads
+    dbs_changes = {}
+
+    # listen dbs 
+    Thread.new do
+        couch = couch.clone
+        while true do
+            update = get("#{couch}/_db_updates?feed=longpoll");
+            Thread.new do
+                ok = update["ok"]
+                db = update["db_name"]
+                type = update["type"]
+                if ok && !db.start_with?("_") && !db.end_with?("_history") then
+                    if type == "created" then
+                        puts "DB created: #{db}"
+                        r = put("#{couch}/#{db}_history",{});
+                        puts "Create #{couch}/#{db}_history = #{r}"
+                        dbs_changes["#{couch}/#{db}"] = Thread.new do
+                            listen_changes(couch,db)
+                        end
+                    elsif type == 'deleted'
+                        puts "DB deleted: #{db}"
+                        dbs_changes["#{couch}/#{db}"].stop();
+                    end
+                end
+            end
+        end
+    end
+
+    # initial run
+    get("#{couch}/_all_dbs")
+         .select { |db| !db.start_with?("_") && !db.end_with?("_history")}
+         .each { |db| 
+            puts "DB existed: #{couch}/#{db}"
+            r = put("#{couch}/#{db}_history",{});
+            puts "Create #{couch}/#{db}_history = #{r}"
+            dbs_changes["#{couch}/#{db}"] = Thread.new do
+                listen_changes(couch,db)
+            end
+         }
+        
+}
+
+puts "Started"
+
+while true
+    sleep 10
+end
+
+puts "Ended"
+

data/lib/couch_http.rb

@@ -0,0 +1,36 @@
+
+require 'net/http'
+require 'uri'
+require 'json'
+
+def get(uri)
+    JSON.parse(Net::HTTP.get(URI(uri)))
+end
+
+def put(uri,doc) 
+    uri = URI.parse(uri)
+    header = {'Content-Type'=> 'application/json'}
+    http = Net::HTTP.new(uri.host, uri.port)
+    request = Net::HTTP::Put.new(uri.request_uri, header)
+    request.body = doc.to_json
+    response = http.request(request)
+    JSON.parse(response.body)
+end
+
+def post(uri,doc) 
+    uri = URI.parse(uri)
+    header = {'Content-Type'=> 'application/json'}
+    http = Net::HTTP.new(uri.host, uri.port)
+    request = Net::HTTP::Post.new(uri.request_uri, header)
+    request.body = doc.to_json
+    response = http.request(request)
+    JSON.parse(response.body)
+end
+
+def delete(uri)
+    uri = URI.parse(uri)
+    http = Net::HTTP.new(uri.host, uri.port)
+    request = Net::HTTP::Delete.new(uri.request_uri)
+    response = http.request(request)
+    JSON.parse(response.body)
+end

metadata

@@ -0,0 +1,52 @@
+--- !ruby/object:Gem::Specification
+name: couchdb-extras
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Diogo Silva
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-26 00:00:00.000000000 Z
+dependencies: []
+description: CouchDB to ElasticSearch, CouchDB to CouchDB History and CouchDB to CouchDB.
+email: diogo@diogok.net
+executables:
+- couchdb2couchdb
+- couchdb2history
+- couchdb2elasticsearch
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/couch_http.rb
+- bin/couchdb2couchdb
+- bin/couchdb2history
+- bin/couchdb2elasticsearch
+homepage: https://github.com/cncflora/couchdb-extras
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Small CouchDB utils
+test_files: []