membase-dump 0.0.1

data/lib/tap.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+"""
+TAP protocol client library.
+
+Copyright (c) 2010  Dustin Sallings <dustin@spy.net>
+"""
+
+import socket
+import string
+import random
+import struct
+import asyncore
+
+import mc_bin_server
+import mc_bin_client
+
+from memcacheConstants import REQ_MAGIC_BYTE, RES_MAGIC_BYTE
+from memcacheConstants import REQ_PKT_FMT, RES_PKT_FMT, MIN_RECV_PACKET
+from memcacheConstants import SET_PKT_FMT, DEL_PKT_FMT, INCRDECR_RES_FMT
+
+import memcacheConstants
+
+class TapConnection(mc_bin_server.MemcachedBinaryChannel):
+
+    def __init__(self, server, port, callback, clientId=None, opts={}, user=None, pswd=None):
+        mc_bin_server.MemcachedBinaryChannel.__init__(self, None, None,
+                                                      self._createTapCall(clientId,
+                                                                          opts))
+        self.server = server
+        self.port = port
+        self.callback = callback
+        self.identifier = (server, port)
+        self.user = user
+        self.pswd = pswd
+        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.connect((server, port))
+
+    def create_socket(self, family, type):
+        if not self.user:
+            mc_bin_server.MemcachedBinaryChannel.create_socket(self, family, type)
+            return
+
+        self.family_and_type = family, type
+
+        self.mc = mc_bin_client.MemcachedClient(self.server, self.port)
+        self.mc.sasl_auth_plain(self.user, self.pswd or "")
+
+        sock = self.mc.s
+        sock.setblocking(0)
+        self.set_socket(sock)
+
+    def _createTapCall(self, key=None, opts={}):
+        # Client identifier
+        if not key:
+            key = "".join(random.sample(string.letters, 16))
+        dtype=0
+        opaque=0
+        cas=0
+
+        extraHeader, val = self._encodeOpts(opts)
+
+        msg=struct.pack(REQ_PKT_FMT, REQ_MAGIC_BYTE,
+                        memcacheConstants.CMD_TAP_CONNECT,
+                        len(key), len(extraHeader), dtype, 0,
+                        len(key) + len(extraHeader) + len(val),
+                        opaque, cas)
+        return msg + extraHeader + key + val
+
+    def _encodeOpts(self, opts):
+        header = 0
+        val = []
+        for op in sorted(opts.keys()):
+            header |= op
+            if op in memcacheConstants.TAP_FLAG_TYPES:
+                val.append(struct.pack(memcacheConstants.TAP_FLAG_TYPES[op],
+                                       opts[op]))
+            elif op == memcacheConstants.TAP_FLAG_LIST_VBUCKETS:
+                val.append(self._encodeVBucketList(opts[op]))
+            else:
+                val.append(opts[op])
+        return struct.pack(">I", header), ''.join(val)
+
+    def _encodeVBucketList(self, vbl):
+        l = list(vbl) # in case it's a generator
+        vals = [struct.pack("!H", len(l))]
+        for v in vbl:
+            vals.append(struct.pack("!H", v))
+        return ''.join(vals)
+
+    def processCommand(self, cmd, klen, vb, extralen, cas, data):
+        extra = data[0:extralen]
+        key = data[extralen:(extralen+klen)]
+        val = data[(extralen+klen):]
+        return self.callback(self.identifier, cmd, extra, key, vb, val, cas)
+
+    def handle_connect(self):
+        pass
+
+    def handle_close(self):
+        self.close()
+
+class TapClient(object):
+
+    def __init__(self, servers, callback, opts={}, user=None, pswd=None):
+        for t in servers:
+            tc = TapConnection(t.host, t.port, callback, t.id, opts, user, pswd)
+
+class TapDescriptor(object):
+    port = 11211
+    id = None
+
+    def __init__(self, s):
+        self.host = s
+        if ':' in s:
+            self.host, self.port = s.split(':', 1)
+            self.port = int(self.port)
+
+        if '@' in self.host:
+            self.id, self.host = self.host.split('@', 1)
+
+    def __repr__(self):
+        return "<TapDescriptor %s@%s:%d>" % (self.id or "(anon)", self.host, self.port)

data/lib/tap_dump.py

@@ -0,0 +1,91 @@
+#! /usr/bin/env python
+
+import os
+import sys
+import string
+import asyncore
+import signal
+import getopt
+import struct
+import base64
+
+import mc_bin_server
+import mc_bin_client
+
+from memcacheConstants import REQ_MAGIC_BYTE, RES_MAGIC_BYTE
+from memcacheConstants import REQ_PKT_FMT, RES_PKT_FMT, MIN_RECV_PACKET
+from memcacheConstants import SET_PKT_FMT, DEL_PKT_FMT, INCRDECR_RES_FMT
+
+import memcacheConstants
+
+import tap
+
+def usage(err=0):
+    print >> sys.stderr, """
+Usage: %s [-u bucket_user [-p bucket_password]] host:port [... hostN:portN]
+
+Example:
+  %s -u user_profiles -p secret9876 membase-01:11210 membase-02:11210
+""" % (os.path.basename(sys.argv[0]),
+       os.path.basename(sys.argv[0]))
+    sys.exit(err)
+
+def parse_args(args):
+    user = None
+    pswd = None
+
+    try:
+        opts, args = getopt.getopt(args, 'hu:p:', ['help'])
+    except getopt.GetoptError, e:
+        usage("ERROR: " + e.msg)
+
+    for (o, a) in opts:
+        if o == '--help' or o == '-h':
+            usage()
+        elif o == '-u':
+            user = a
+        elif o == '-p':
+            pswd = a
+        else:
+            usage("ERROR: unknown option - " + o)
+
+    if not args or len(args) < 1:
+        usage("ERROR: missing at least one host:port to TAP")
+
+    return user, pswd, args
+
+def signal_handler(signal, frame):
+    print 'Tap stream terminated by user'
+    sys.exit(0)
+
+def mainLoop(serverList, cb, opts={}, user=None, pswd=None):
+    """Run the given callback for each tap message from any of the
+    upstream servers.
+
+    loops until all connections drop
+    """
+    signal.signal(signal.SIGINT, signal_handler)
+
+    connections = (tap.TapDescriptor(a) for a in serverList)
+    tap.TapClient(connections, cb, opts=opts, user=user, pswd=pswd)
+    asyncore.loop()
+
+def parse_mutation_extra(extra):
+    engine_priv, flags, ttl, _, _, _, item_flags, item_expiry = struct.unpack(">2h4c2I", extra)
+    return {'engine_priv': engine_priv,
+            'flags': flags,
+            'ttl': ttl,
+            'item_flags': item_flags,
+            'item_expiry': item_expiry}
+
+if __name__ == '__main__':
+    user, pswd, args = parse_args(sys.argv[1:])
+
+    def cb(identifier, cmd, extra, key, vb, val, cas):
+        if memcacheConstants.COMMAND_NAMES[cmd] == 'CMD_TAP_MUTATION':
+            extra_data = parse_mutation_extra(extra)
+            print "%s\t%s\t%s" % (key, base64.b64encode(val), extra_data['item_expiry'])
+
+    opts = {memcacheConstants.TAP_FLAG_DUMP: ''}
+
+    mainLoop(args, cb, opts, user, pswd)

data/membase-dump.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "membase-dump/version"
+
+Gem::Specification.new do |s|
+  s.name        = "membase-dump"
+  s.version     = Membase::Dump::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Jean-Louis Giordano"]
+  s.email       = ["jean-louis@releware.com"]
+  s.homepage    = ""
+  s.summary     = %q{Compilation of scripts to make and load membase dumps through the Tap interface.}
+  s.description = %q{Compilation of scripts to make and load membase dumps through the Tap interface.}
+
+  s.rubyforge_project = "membase-dump"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency("dalli")
+  s.add_dependency("active_support")
+  s.add_development_dependency("rake", "0.8.7")
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: membase-dump
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jean-Louis Giordano
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-01 00:00:00.000000000 +01:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dalli
+  requirement: &70236485411420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70236485411420
+- !ruby/object:Gem::Dependency
+  name: active_support
+  requirement: &70236485411000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70236485411000
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70236485410500 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :development
+  prerelease: false
+  version_requirements: *70236485410500
+description: Compilation of scripts to make and load membase dumps through the Tap
+  interface.
+email:
+- jean-louis@releware.com
+executables:
+- tap_dump
+- tap_load
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/tap_dump
+- bin/tap_load
+- lib/mc_bin_client.py
+- lib/mc_bin_server.py
+- lib/membase-dump.rb
+- lib/membase-dump/version.rb
+- lib/memcacheConstants.py
+- lib/tap.py
+- lib/tap_dump.py
+- membase-dump.gemspec
+has_rdoc: true
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: membase-dump
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Compilation of scripts to make and load membase dumps through the Tap interface.
+test_files: []