membase-dump 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/tap.py ADDED
@@ -0,0 +1,122 @@
1
+ #!/usr/bin/env python
2
+ """
3
+ TAP protocol client library.
4
+
5
+ Copyright (c) 2010 Dustin Sallings <dustin@spy.net>
6
+ """
7
+
8
+ import socket
9
+ import string
10
+ import random
11
+ import struct
12
+ import asyncore
13
+
14
+ import mc_bin_server
15
+ import mc_bin_client
16
+
17
+ from memcacheConstants import REQ_MAGIC_BYTE, RES_MAGIC_BYTE
18
+ from memcacheConstants import REQ_PKT_FMT, RES_PKT_FMT, MIN_RECV_PACKET
19
+ from memcacheConstants import SET_PKT_FMT, DEL_PKT_FMT, INCRDECR_RES_FMT
20
+
21
+ import memcacheConstants
22
+
23
+ class TapConnection(mc_bin_server.MemcachedBinaryChannel):
24
+
25
+ def __init__(self, server, port, callback, clientId=None, opts={}, user=None, pswd=None):
26
+ mc_bin_server.MemcachedBinaryChannel.__init__(self, None, None,
27
+ self._createTapCall(clientId,
28
+ opts))
29
+ self.server = server
30
+ self.port = port
31
+ self.callback = callback
32
+ self.identifier = (server, port)
33
+ self.user = user
34
+ self.pswd = pswd
35
+ self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
36
+ self.connect((server, port))
37
+
38
+ def create_socket(self, family, type):
39
+ if not self.user:
40
+ mc_bin_server.MemcachedBinaryChannel.create_socket(self, family, type)
41
+ return
42
+
43
+ self.family_and_type = family, type
44
+
45
+ self.mc = mc_bin_client.MemcachedClient(self.server, self.port)
46
+ self.mc.sasl_auth_plain(self.user, self.pswd or "")
47
+
48
+ sock = self.mc.s
49
+ sock.setblocking(0)
50
+ self.set_socket(sock)
51
+
52
+ def _createTapCall(self, key=None, opts={}):
53
+ # Client identifier
54
+ if not key:
55
+ key = "".join(random.sample(string.letters, 16))
56
+ dtype=0
57
+ opaque=0
58
+ cas=0
59
+
60
+ extraHeader, val = self._encodeOpts(opts)
61
+
62
+ msg=struct.pack(REQ_PKT_FMT, REQ_MAGIC_BYTE,
63
+ memcacheConstants.CMD_TAP_CONNECT,
64
+ len(key), len(extraHeader), dtype, 0,
65
+ len(key) + len(extraHeader) + len(val),
66
+ opaque, cas)
67
+ return msg + extraHeader + key + val
68
+
69
+ def _encodeOpts(self, opts):
70
+ header = 0
71
+ val = []
72
+ for op in sorted(opts.keys()):
73
+ header |= op
74
+ if op in memcacheConstants.TAP_FLAG_TYPES:
75
+ val.append(struct.pack(memcacheConstants.TAP_FLAG_TYPES[op],
76
+ opts[op]))
77
+ elif op == memcacheConstants.TAP_FLAG_LIST_VBUCKETS:
78
+ val.append(self._encodeVBucketList(opts[op]))
79
+ else:
80
+ val.append(opts[op])
81
+ return struct.pack(">I", header), ''.join(val)
82
+
83
+ def _encodeVBucketList(self, vbl):
84
+ l = list(vbl) # in case it's a generator
85
+ vals = [struct.pack("!H", len(l))]
86
+ for v in vbl:
87
+ vals.append(struct.pack("!H", v))
88
+ return ''.join(vals)
89
+
90
+ def processCommand(self, cmd, klen, vb, extralen, cas, data):
91
+ extra = data[0:extralen]
92
+ key = data[extralen:(extralen+klen)]
93
+ val = data[(extralen+klen):]
94
+ return self.callback(self.identifier, cmd, extra, key, vb, val, cas)
95
+
96
+ def handle_connect(self):
97
+ pass
98
+
99
+ def handle_close(self):
100
+ self.close()
101
+
102
+ class TapClient(object):
103
+
104
+ def __init__(self, servers, callback, opts={}, user=None, pswd=None):
105
+ for t in servers:
106
+ tc = TapConnection(t.host, t.port, callback, t.id, opts, user, pswd)
107
+
108
+ class TapDescriptor(object):
109
+ port = 11211
110
+ id = None
111
+
112
+ def __init__(self, s):
113
+ self.host = s
114
+ if ':' in s:
115
+ self.host, self.port = s.split(':', 1)
116
+ self.port = int(self.port)
117
+
118
+ if '@' in self.host:
119
+ self.id, self.host = self.host.split('@', 1)
120
+
121
+ def __repr__(self):
122
+ return "<TapDescriptor %s@%s:%d>" % (self.id or "(anon)", self.host, self.port)
data/lib/tap_dump.py ADDED
@@ -0,0 +1,91 @@
1
+ #! /usr/bin/env python
2
+
3
+ import os
4
+ import sys
5
+ import string
6
+ import asyncore
7
+ import signal
8
+ import getopt
9
+ import struct
10
+ import base64
11
+
12
+ import mc_bin_server
13
+ import mc_bin_client
14
+
15
+ from memcacheConstants import REQ_MAGIC_BYTE, RES_MAGIC_BYTE
16
+ from memcacheConstants import REQ_PKT_FMT, RES_PKT_FMT, MIN_RECV_PACKET
17
+ from memcacheConstants import SET_PKT_FMT, DEL_PKT_FMT, INCRDECR_RES_FMT
18
+
19
+ import memcacheConstants
20
+
21
+ import tap
22
+
23
+ def usage(err=0):
24
+ print >> sys.stderr, """
25
+ Usage: %s [-u bucket_user [-p bucket_password]] host:port [... hostN:portN]
26
+
27
+ Example:
28
+ %s -u user_profiles -p secret9876 membase-01:11210 membase-02:11210
29
+ """ % (os.path.basename(sys.argv[0]),
30
+ os.path.basename(sys.argv[0]))
31
+ sys.exit(err)
32
+
33
+ def parse_args(args):
34
+ user = None
35
+ pswd = None
36
+
37
+ try:
38
+ opts, args = getopt.getopt(args, 'hu:p:', ['help'])
39
+ except getopt.GetoptError, e:
40
+ usage("ERROR: " + e.msg)
41
+
42
+ for (o, a) in opts:
43
+ if o == '--help' or o == '-h':
44
+ usage()
45
+ elif o == '-u':
46
+ user = a
47
+ elif o == '-p':
48
+ pswd = a
49
+ else:
50
+ usage("ERROR: unknown option - " + o)
51
+
52
+ if not args or len(args) < 1:
53
+ usage("ERROR: missing at least one host:port to TAP")
54
+
55
+ return user, pswd, args
56
+
57
+ def signal_handler(signal, frame):
58
+ print 'Tap stream terminated by user'
59
+ sys.exit(0)
60
+
61
+ def mainLoop(serverList, cb, opts={}, user=None, pswd=None):
62
+ """Run the given callback for each tap message from any of the
63
+ upstream servers.
64
+
65
+ loops until all connections drop
66
+ """
67
+ signal.signal(signal.SIGINT, signal_handler)
68
+
69
+ connections = (tap.TapDescriptor(a) for a in serverList)
70
+ tap.TapClient(connections, cb, opts=opts, user=user, pswd=pswd)
71
+ asyncore.loop()
72
+
73
+ def parse_mutation_extra(extra):
74
+ engine_priv, flags, ttl, _, _, _, item_flags, item_expiry = struct.unpack(">2h4c2I", extra)
75
+ return {'engine_priv': engine_priv,
76
+ 'flags': flags,
77
+ 'ttl': ttl,
78
+ 'item_flags': item_flags,
79
+ 'item_expiry': item_expiry}
80
+
81
+ if __name__ == '__main__':
82
+ user, pswd, args = parse_args(sys.argv[1:])
83
+
84
+ def cb(identifier, cmd, extra, key, vb, val, cas):
85
+ if memcacheConstants.COMMAND_NAMES[cmd] == 'CMD_TAP_MUTATION':
86
+ extra_data = parse_mutation_extra(extra)
87
+ print "%s\t%s\t%s" % (key, base64.b64encode(val), extra_data['item_expiry'])
88
+
89
+ opts = {memcacheConstants.TAP_FLAG_DUMP: ''}
90
+
91
+ mainLoop(args, cb, opts, user, pswd)
@@ -0,0 +1,24 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "membase-dump/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "membase-dump"
7
+ s.version = Membase::Dump::VERSION
8
+ s.platform = Gem::Platform::RUBY
9
+ s.authors = ["Jean-Louis Giordano"]
10
+ s.email = ["jean-louis@releware.com"]
11
+ s.homepage = ""
12
+ s.summary = %q{Compilation of scripts to make and load membase dumps through the Tap interface.}
13
+ s.description = %q{Compilation of scripts to make and load membase dumps through the Tap interface.}
14
+
15
+ s.rubyforge_project = "membase-dump"
16
+
17
+ s.files = `git ls-files`.split("\n")
18
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
19
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
20
+ s.require_paths = ["lib"]
21
+ s.add_dependency("dalli")
22
+ s.add_dependency("active_support")
23
+ s.add_development_dependency("rake", "0.8.7")
24
+ end
metadata ADDED
@@ -0,0 +1,99 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: membase-dump
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Jean-Louis Giordano
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2011-12-01 00:00:00.000000000 +01:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: dalli
17
+ requirement: &70236485411420 !ruby/object:Gem::Requirement
18
+ none: false
19
+ requirements:
20
+ - - ! '>='
21
+ - !ruby/object:Gem::Version
22
+ version: '0'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: *70236485411420
26
+ - !ruby/object:Gem::Dependency
27
+ name: active_support
28
+ requirement: &70236485411000 !ruby/object:Gem::Requirement
29
+ none: false
30
+ requirements:
31
+ - - ! '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: *70236485411000
37
+ - !ruby/object:Gem::Dependency
38
+ name: rake
39
+ requirement: &70236485410500 !ruby/object:Gem::Requirement
40
+ none: false
41
+ requirements:
42
+ - - =
43
+ - !ruby/object:Gem::Version
44
+ version: 0.8.7
45
+ type: :development
46
+ prerelease: false
47
+ version_requirements: *70236485410500
48
+ description: Compilation of scripts to make and load membase dumps through the Tap
49
+ interface.
50
+ email:
51
+ - jean-louis@releware.com
52
+ executables:
53
+ - tap_dump
54
+ - tap_load
55
+ extensions: []
56
+ extra_rdoc_files: []
57
+ files:
58
+ - .gitignore
59
+ - .rvmrc
60
+ - Gemfile
61
+ - Gemfile.lock
62
+ - README.md
63
+ - Rakefile
64
+ - bin/tap_dump
65
+ - bin/tap_load
66
+ - lib/mc_bin_client.py
67
+ - lib/mc_bin_server.py
68
+ - lib/membase-dump.rb
69
+ - lib/membase-dump/version.rb
70
+ - lib/memcacheConstants.py
71
+ - lib/tap.py
72
+ - lib/tap_dump.py
73
+ - membase-dump.gemspec
74
+ has_rdoc: true
75
+ homepage: ''
76
+ licenses: []
77
+ post_install_message:
78
+ rdoc_options: []
79
+ require_paths:
80
+ - lib
81
+ required_ruby_version: !ruby/object:Gem::Requirement
82
+ none: false
83
+ requirements:
84
+ - - ! '>='
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ required_rubygems_version: !ruby/object:Gem::Requirement
88
+ none: false
89
+ requirements:
90
+ - - ! '>='
91
+ - !ruby/object:Gem::Version
92
+ version: '0'
93
+ requirements: []
94
+ rubyforge_project: membase-dump
95
+ rubygems_version: 1.6.2
96
+ signing_key:
97
+ specification_version: 3
98
+ summary: Compilation of scripts to make and load membase dumps through the Tap interface.
99
+ test_files: []