membase-dump 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +12 -0
- data/.rvmrc +1 -0
- data/Gemfile +4 -0
- data/README.md +25 -0
- data/Rakefile +2 -0
- data/bin/tap_dump +3 -0
- data/bin/tap_load +13 -0
- data/lib/mc_bin_client.py +442 -0
- data/lib/mc_bin_server.py +386 -0
- data/lib/membase-dump.rb +21 -0
- data/lib/membase-dump/version.rb +5 -0
- data/lib/memcacheConstants.py +194 -0
- data/lib/tap.py +122 -0
- data/lib/tap_dump.py +91 -0
- data/membase-dump.gemspec +24 -0
- metadata +99 -0
data/lib/tap.py
ADDED
@@ -0,0 +1,122 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
"""
|
3
|
+
TAP protocol client library.
|
4
|
+
|
5
|
+
Copyright (c) 2010 Dustin Sallings <dustin@spy.net>
|
6
|
+
"""
|
7
|
+
|
8
|
+
import socket
|
9
|
+
import string
|
10
|
+
import random
|
11
|
+
import struct
|
12
|
+
import asyncore
|
13
|
+
|
14
|
+
import mc_bin_server
|
15
|
+
import mc_bin_client
|
16
|
+
|
17
|
+
from memcacheConstants import REQ_MAGIC_BYTE, RES_MAGIC_BYTE
|
18
|
+
from memcacheConstants import REQ_PKT_FMT, RES_PKT_FMT, MIN_RECV_PACKET
|
19
|
+
from memcacheConstants import SET_PKT_FMT, DEL_PKT_FMT, INCRDECR_RES_FMT
|
20
|
+
|
21
|
+
import memcacheConstants
|
22
|
+
|
23
|
+
class TapConnection(mc_bin_server.MemcachedBinaryChannel):
|
24
|
+
|
25
|
+
def __init__(self, server, port, callback, clientId=None, opts={}, user=None, pswd=None):
|
26
|
+
mc_bin_server.MemcachedBinaryChannel.__init__(self, None, None,
|
27
|
+
self._createTapCall(clientId,
|
28
|
+
opts))
|
29
|
+
self.server = server
|
30
|
+
self.port = port
|
31
|
+
self.callback = callback
|
32
|
+
self.identifier = (server, port)
|
33
|
+
self.user = user
|
34
|
+
self.pswd = pswd
|
35
|
+
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
|
36
|
+
self.connect((server, port))
|
37
|
+
|
38
|
+
def create_socket(self, family, type):
|
39
|
+
if not self.user:
|
40
|
+
mc_bin_server.MemcachedBinaryChannel.create_socket(self, family, type)
|
41
|
+
return
|
42
|
+
|
43
|
+
self.family_and_type = family, type
|
44
|
+
|
45
|
+
self.mc = mc_bin_client.MemcachedClient(self.server, self.port)
|
46
|
+
self.mc.sasl_auth_plain(self.user, self.pswd or "")
|
47
|
+
|
48
|
+
sock = self.mc.s
|
49
|
+
sock.setblocking(0)
|
50
|
+
self.set_socket(sock)
|
51
|
+
|
52
|
+
def _createTapCall(self, key=None, opts={}):
|
53
|
+
# Client identifier
|
54
|
+
if not key:
|
55
|
+
key = "".join(random.sample(string.letters, 16))
|
56
|
+
dtype=0
|
57
|
+
opaque=0
|
58
|
+
cas=0
|
59
|
+
|
60
|
+
extraHeader, val = self._encodeOpts(opts)
|
61
|
+
|
62
|
+
msg=struct.pack(REQ_PKT_FMT, REQ_MAGIC_BYTE,
|
63
|
+
memcacheConstants.CMD_TAP_CONNECT,
|
64
|
+
len(key), len(extraHeader), dtype, 0,
|
65
|
+
len(key) + len(extraHeader) + len(val),
|
66
|
+
opaque, cas)
|
67
|
+
return msg + extraHeader + key + val
|
68
|
+
|
69
|
+
def _encodeOpts(self, opts):
|
70
|
+
header = 0
|
71
|
+
val = []
|
72
|
+
for op in sorted(opts.keys()):
|
73
|
+
header |= op
|
74
|
+
if op in memcacheConstants.TAP_FLAG_TYPES:
|
75
|
+
val.append(struct.pack(memcacheConstants.TAP_FLAG_TYPES[op],
|
76
|
+
opts[op]))
|
77
|
+
elif op == memcacheConstants.TAP_FLAG_LIST_VBUCKETS:
|
78
|
+
val.append(self._encodeVBucketList(opts[op]))
|
79
|
+
else:
|
80
|
+
val.append(opts[op])
|
81
|
+
return struct.pack(">I", header), ''.join(val)
|
82
|
+
|
83
|
+
def _encodeVBucketList(self, vbl):
|
84
|
+
l = list(vbl) # in case it's a generator
|
85
|
+
vals = [struct.pack("!H", len(l))]
|
86
|
+
for v in vbl:
|
87
|
+
vals.append(struct.pack("!H", v))
|
88
|
+
return ''.join(vals)
|
89
|
+
|
90
|
+
def processCommand(self, cmd, klen, vb, extralen, cas, data):
|
91
|
+
extra = data[0:extralen]
|
92
|
+
key = data[extralen:(extralen+klen)]
|
93
|
+
val = data[(extralen+klen):]
|
94
|
+
return self.callback(self.identifier, cmd, extra, key, vb, val, cas)
|
95
|
+
|
96
|
+
def handle_connect(self):
|
97
|
+
pass
|
98
|
+
|
99
|
+
def handle_close(self):
|
100
|
+
self.close()
|
101
|
+
|
102
|
+
class TapClient(object):
|
103
|
+
|
104
|
+
def __init__(self, servers, callback, opts={}, user=None, pswd=None):
|
105
|
+
for t in servers:
|
106
|
+
tc = TapConnection(t.host, t.port, callback, t.id, opts, user, pswd)
|
107
|
+
|
108
|
+
class TapDescriptor(object):
|
109
|
+
port = 11211
|
110
|
+
id = None
|
111
|
+
|
112
|
+
def __init__(self, s):
|
113
|
+
self.host = s
|
114
|
+
if ':' in s:
|
115
|
+
self.host, self.port = s.split(':', 1)
|
116
|
+
self.port = int(self.port)
|
117
|
+
|
118
|
+
if '@' in self.host:
|
119
|
+
self.id, self.host = self.host.split('@', 1)
|
120
|
+
|
121
|
+
def __repr__(self):
|
122
|
+
return "<TapDescriptor %s@%s:%d>" % (self.id or "(anon)", self.host, self.port)
|
data/lib/tap_dump.py
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
#! /usr/bin/env python
|
2
|
+
|
3
|
+
import os
|
4
|
+
import sys
|
5
|
+
import string
|
6
|
+
import asyncore
|
7
|
+
import signal
|
8
|
+
import getopt
|
9
|
+
import struct
|
10
|
+
import base64
|
11
|
+
|
12
|
+
import mc_bin_server
|
13
|
+
import mc_bin_client
|
14
|
+
|
15
|
+
from memcacheConstants import REQ_MAGIC_BYTE, RES_MAGIC_BYTE
|
16
|
+
from memcacheConstants import REQ_PKT_FMT, RES_PKT_FMT, MIN_RECV_PACKET
|
17
|
+
from memcacheConstants import SET_PKT_FMT, DEL_PKT_FMT, INCRDECR_RES_FMT
|
18
|
+
|
19
|
+
import memcacheConstants
|
20
|
+
|
21
|
+
import tap
|
22
|
+
|
23
|
+
def usage(err=0):
|
24
|
+
print >> sys.stderr, """
|
25
|
+
Usage: %s [-u bucket_user [-p bucket_password]] host:port [... hostN:portN]
|
26
|
+
|
27
|
+
Example:
|
28
|
+
%s -u user_profiles -p secret9876 membase-01:11210 membase-02:11210
|
29
|
+
""" % (os.path.basename(sys.argv[0]),
|
30
|
+
os.path.basename(sys.argv[0]))
|
31
|
+
sys.exit(err)
|
32
|
+
|
33
|
+
def parse_args(args):
|
34
|
+
user = None
|
35
|
+
pswd = None
|
36
|
+
|
37
|
+
try:
|
38
|
+
opts, args = getopt.getopt(args, 'hu:p:', ['help'])
|
39
|
+
except getopt.GetoptError, e:
|
40
|
+
usage("ERROR: " + e.msg)
|
41
|
+
|
42
|
+
for (o, a) in opts:
|
43
|
+
if o == '--help' or o == '-h':
|
44
|
+
usage()
|
45
|
+
elif o == '-u':
|
46
|
+
user = a
|
47
|
+
elif o == '-p':
|
48
|
+
pswd = a
|
49
|
+
else:
|
50
|
+
usage("ERROR: unknown option - " + o)
|
51
|
+
|
52
|
+
if not args or len(args) < 1:
|
53
|
+
usage("ERROR: missing at least one host:port to TAP")
|
54
|
+
|
55
|
+
return user, pswd, args
|
56
|
+
|
57
|
+
def signal_handler(signal, frame):
|
58
|
+
print 'Tap stream terminated by user'
|
59
|
+
sys.exit(0)
|
60
|
+
|
61
|
+
def mainLoop(serverList, cb, opts={}, user=None, pswd=None):
|
62
|
+
"""Run the given callback for each tap message from any of the
|
63
|
+
upstream servers.
|
64
|
+
|
65
|
+
loops until all connections drop
|
66
|
+
"""
|
67
|
+
signal.signal(signal.SIGINT, signal_handler)
|
68
|
+
|
69
|
+
connections = (tap.TapDescriptor(a) for a in serverList)
|
70
|
+
tap.TapClient(connections, cb, opts=opts, user=user, pswd=pswd)
|
71
|
+
asyncore.loop()
|
72
|
+
|
73
|
+
def parse_mutation_extra(extra):
|
74
|
+
engine_priv, flags, ttl, _, _, _, item_flags, item_expiry = struct.unpack(">2h4c2I", extra)
|
75
|
+
return {'engine_priv': engine_priv,
|
76
|
+
'flags': flags,
|
77
|
+
'ttl': ttl,
|
78
|
+
'item_flags': item_flags,
|
79
|
+
'item_expiry': item_expiry}
|
80
|
+
|
81
|
+
if __name__ == '__main__':
|
82
|
+
user, pswd, args = parse_args(sys.argv[1:])
|
83
|
+
|
84
|
+
def cb(identifier, cmd, extra, key, vb, val, cas):
|
85
|
+
if memcacheConstants.COMMAND_NAMES[cmd] == 'CMD_TAP_MUTATION':
|
86
|
+
extra_data = parse_mutation_extra(extra)
|
87
|
+
print "%s\t%s\t%s" % (key, base64.b64encode(val), extra_data['item_expiry'])
|
88
|
+
|
89
|
+
opts = {memcacheConstants.TAP_FLAG_DUMP: ''}
|
90
|
+
|
91
|
+
mainLoop(args, cb, opts, user, pswd)
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "membase-dump/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "membase-dump"
|
7
|
+
s.version = Membase::Dump::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Jean-Louis Giordano"]
|
10
|
+
s.email = ["jean-louis@releware.com"]
|
11
|
+
s.homepage = ""
|
12
|
+
s.summary = %q{Compilation of scripts to make and load membase dumps through the Tap interface.}
|
13
|
+
s.description = %q{Compilation of scripts to make and load membase dumps through the Tap interface.}
|
14
|
+
|
15
|
+
s.rubyforge_project = "membase-dump"
|
16
|
+
|
17
|
+
s.files = `git ls-files`.split("\n")
|
18
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
|
+
s.require_paths = ["lib"]
|
21
|
+
s.add_dependency("dalli")
|
22
|
+
s.add_dependency("active_support")
|
23
|
+
s.add_development_dependency("rake", "0.8.7")
|
24
|
+
end
|
metadata
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: membase-dump
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Jean-Louis Giordano
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2011-12-01 00:00:00.000000000 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: dalli
|
17
|
+
requirement: &70236485411420 !ruby/object:Gem::Requirement
|
18
|
+
none: false
|
19
|
+
requirements:
|
20
|
+
- - ! '>='
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '0'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: *70236485411420
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: active_support
|
28
|
+
requirement: &70236485411000 !ruby/object:Gem::Requirement
|
29
|
+
none: false
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: *70236485411000
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rake
|
39
|
+
requirement: &70236485410500 !ruby/object:Gem::Requirement
|
40
|
+
none: false
|
41
|
+
requirements:
|
42
|
+
- - =
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: 0.8.7
|
45
|
+
type: :development
|
46
|
+
prerelease: false
|
47
|
+
version_requirements: *70236485410500
|
48
|
+
description: Compilation of scripts to make and load membase dumps through the Tap
|
49
|
+
interface.
|
50
|
+
email:
|
51
|
+
- jean-louis@releware.com
|
52
|
+
executables:
|
53
|
+
- tap_dump
|
54
|
+
- tap_load
|
55
|
+
extensions: []
|
56
|
+
extra_rdoc_files: []
|
57
|
+
files:
|
58
|
+
- .gitignore
|
59
|
+
- .rvmrc
|
60
|
+
- Gemfile
|
61
|
+
- Gemfile.lock
|
62
|
+
- README.md
|
63
|
+
- Rakefile
|
64
|
+
- bin/tap_dump
|
65
|
+
- bin/tap_load
|
66
|
+
- lib/mc_bin_client.py
|
67
|
+
- lib/mc_bin_server.py
|
68
|
+
- lib/membase-dump.rb
|
69
|
+
- lib/membase-dump/version.rb
|
70
|
+
- lib/memcacheConstants.py
|
71
|
+
- lib/tap.py
|
72
|
+
- lib/tap_dump.py
|
73
|
+
- membase-dump.gemspec
|
74
|
+
has_rdoc: true
|
75
|
+
homepage: ''
|
76
|
+
licenses: []
|
77
|
+
post_install_message:
|
78
|
+
rdoc_options: []
|
79
|
+
require_paths:
|
80
|
+
- lib
|
81
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
82
|
+
none: false
|
83
|
+
requirements:
|
84
|
+
- - ! '>='
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
87
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
88
|
+
none: false
|
89
|
+
requirements:
|
90
|
+
- - ! '>='
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: '0'
|
93
|
+
requirements: []
|
94
|
+
rubyforge_project: membase-dump
|
95
|
+
rubygems_version: 1.6.2
|
96
|
+
signing_key:
|
97
|
+
specification_version: 3
|
98
|
+
summary: Compilation of scripts to make and load membase dumps through the Tap interface.
|
99
|
+
test_files: []
|