murder 0.0.0.pre

data/dist/BitTornado/inifile.py

@@ -0,0 +1,169 @@
+# Written by John Hoffman
+# see LICENSE.txt for license information
+
+'''
+reads/writes a Windows-style INI file
+format:
+
+  aa = "bb"
+  cc = 11
+
+  [eee]
+  ff = "gg"
+
+decodes to:
+d = { '': {'aa':'bb','cc':'11'}, 'eee': {'ff':'gg'} }
+
+the encoder can also take this as input:
+
+d = { 'aa': 'bb, 'cc': 11, 'eee': {'ff':'gg'} }
+
+though it will only decode in the above format.  Keywords must be strings.
+Values that are strings are written surrounded by quotes, and the decoding
+routine automatically strips any.
+Booleans are written as integers.  Anything else aside from string/int/float
+may have unpredictable results.
+'''
+
+from cStringIO import StringIO
+from traceback import print_exc
+from types import DictType, StringType
+try:
+    from types import BooleanType
+except ImportError:
+    BooleanType = None
+
+try:
+    True
+except:
+    True = 1
+    False = 0
+
+DEBUG = False
+
+def ini_write(f, d, comment=''):
+    try:
+        a = {'':{}}
+        for k,v in d.items():
+            assert type(k) == StringType
+            k = k.lower()
+            if type(v) == DictType:
+                if DEBUG:
+                    print 'new section:' +k
+                if k:
+                    assert not a.has_key(k)
+                    a[k] = {}
+                aa = a[k]
+                for kk,vv in v:
+                    assert type(kk) == StringType
+                    kk = kk.lower()
+                    assert not aa.has_key(kk)
+                    if type(vv) == BooleanType:
+                        vv = int(vv)
+                    if type(vv) == StringType:
+                        vv = '"'+vv+'"'
+                    aa[kk] = str(vv)
+                    if DEBUG:
+                        print 'a['+k+']['+kk+'] = '+str(vv)
+            else:
+                aa = a['']
+                assert not aa.has_key(k)
+                if type(v) == BooleanType:
+                    v = int(v)
+                if type(v) == StringType:
+                    v = '"'+v+'"'
+                aa[k] = str(v)
+                if DEBUG:
+                    print 'a[\'\']['+k+'] = '+str(v)
+        r = open(f,'w')
+        if comment:
+            for c in comment.split('\n'):
+                r.write('# '+c+'\n')
+            r.write('\n')
+        l = a.keys()
+        l.sort()
+        for k in l:
+            if k:
+                r.write('\n['+k+']\n')
+            aa = a[k]
+            ll = aa.keys()
+            ll.sort()
+            for kk in ll:
+                r.write(kk+' = '+aa[kk]+'\n')
+        success = True
+    except:
+        if DEBUG:
+            print_exc()
+        success = False
+    try:
+        r.close()
+    except:
+        pass
+    return success
+
+
+if DEBUG:
+    def errfunc(lineno, line, err):
+        print '('+str(lineno)+') '+err+': '+line
+else:
+    errfunc = lambda lineno, line, err: None
+
+def ini_read(f, errfunc = errfunc):
+    try:
+        r = open(f,'r')
+        ll = r.readlines()
+        d = {}
+        dd = {'':d}
+        for i in xrange(len(ll)):
+            l = ll[i]
+            l = l.strip()
+            if not l:
+                continue
+            if l[0] == '#':
+                continue
+            if l[0] == '[':
+                if l[-1] != ']':
+                    errfunc(i,l,'syntax error')
+                    continue
+                l1 = l[1:-1].strip().lower()
+                if not l1:
+                    errfunc(i,l,'syntax error')
+                    continue
+                if dd.has_key(l1):
+                    errfunc(i,l,'duplicate section')
+                    d = dd[l1]
+                    continue
+                d = {}
+                dd[l1] = d
+                continue
+            try:
+                k,v = l.split('=',1)
+            except:
+                try:
+                    k,v = l.split(':',1)
+                except:
+                    errfunc(i,l,'syntax error')
+                    continue
+            k = k.strip().lower()
+            v = v.strip()
+            if len(v) > 1 and ( (v[0] == '"' and v[-1] == '"') or
+                                (v[0] == "'" and v[-1] == "'") ):
+                v = v[1:-1]
+            if not k:
+                errfunc(i,l,'syntax error')
+                continue
+            if d.has_key(k):
+                errfunc(i,l,'duplicate entry')
+                continue
+            d[k] = v
+        if DEBUG:
+            print dd
+    except:
+        if DEBUG:
+            print_exc()
+        dd = None
+    try:
+        r.close()
+    except:
+        pass
+    return dd

data/dist/BitTornado/iprangeparse.py

@@ -0,0 +1,194 @@
+# Written by John Hoffman
+# see LICENSE.txt for license information
+
+from bisect import bisect, insort
+
+try:
+    True
+except:
+    True = 1
+    False = 0
+    bool = lambda x: not not x
+
+
+def to_long_ipv4(ip):
+    ip = ip.split('.')
+    if len(ip) != 4:
+        raise ValueError, "bad address"
+    b = 0L
+    for n in ip:
+        b *= 256
+        b += int(n)
+    return b
+
+
+def to_long_ipv6(ip):
+    if ip == '':
+        raise ValueError, "bad address"
+    if ip == '::':      # boundary handling
+        ip = ''
+    elif ip[:2] == '::':
+        ip = ip[1:]
+    elif ip[0] == ':':
+        raise ValueError, "bad address"
+    elif ip[-2:] == '::':
+        ip = ip[:-1]
+    elif ip[-1] == ':':
+        raise ValueError, "bad address"
+
+    b = []
+    doublecolon = False
+    for n in ip.split(':'):
+        if n == '':     # double-colon
+            if doublecolon:
+                raise ValueError, "bad address"
+            doublecolon = True
+            b.append(None)
+            continue
+        if n.find('.') >= 0: # IPv4
+            n = n.split('.')
+            if len(n) != 4:
+                raise ValueError, "bad address"
+            for i in n:
+                b.append(int(i))
+            continue
+        n = ('0'*(4-len(n))) + n
+        b.append(int(n[:2],16))
+        b.append(int(n[2:],16))
+    bb = 0L
+    for n in b:
+        if n is None:
+            for i in xrange(17-len(b)):
+                bb *= 256
+            continue
+        bb *= 256
+        bb += n
+    return bb
+
+ipv4addrmask = 65535L*256*256*256*256
+
+class IP_List:
+    def __init__(self):
+        self.ipv4list = []  # starts of ranges
+        self.ipv4dict = {}  # start: end of ranges
+        self.ipv6list = []  # "
+        self.ipv6dict = {}  # "
+
+    def __nonzero__(self):
+        return bool(self.ipv4list or self.ipv6list)
+
+
+    def append(self, ip_beg, ip_end = None):
+        if ip_end is None:
+            ip_end = ip_beg
+        else:
+            assert ip_beg <= ip_end
+        if ip_beg.find(':') < 0:        # IPv4
+            ip_beg = to_long_ipv4(ip_beg)
+            ip_end = to_long_ipv4(ip_end)
+            l = self.ipv4list
+            d = self.ipv4dict
+        else:
+            ip_beg = to_long_ipv6(ip_beg)
+            ip_end = to_long_ipv6(ip_end)
+            bb = ip_beg % (256*256*256*256)
+            if bb == ipv4addrmask:
+                ip_beg -= bb
+                ip_end -= bb
+                l = self.ipv4list
+                d = self.ipv4dict
+            else:
+                l = self.ipv6list
+                d = self.ipv6dict
+
+        pos = bisect(l,ip_beg)-1
+        done = pos < 0
+        while not done:
+            p = pos
+            while p < len(l):
+                range_beg = l[p]
+                if range_beg > ip_end+1:
+                    done = True
+                    break
+                range_end = d[range_beg]
+                if range_end < ip_beg-1:
+                    p += 1
+                    if p == len(l):
+                        done = True
+                        break
+                    continue
+                # if neither of the above conditions is true, the ranges overlap
+                ip_beg = min(ip_beg, range_beg)
+                ip_end = max(ip_end, range_end)
+                del l[p]
+                del d[range_beg]
+                break
+
+        insort(l,ip_beg)
+        d[ip_beg] = ip_end
+
+
+    def includes(self, ip):
+        if not (self.ipv4list or self.ipv6list):
+            return False
+        if ip.find(':') < 0:        # IPv4
+            ip = to_long_ipv4(ip)
+            l = self.ipv4list
+            d = self.ipv4dict
+        else:
+            ip = to_long_ipv6(ip)
+            bb = ip % (256*256*256*256)
+            if bb == ipv4addrmask:
+                ip -= bb
+                l = self.ipv4list
+                d = self.ipv4dict
+            else:
+                l = self.ipv6list
+                d = self.ipv6dict
+        for ip_beg in l[bisect(l,ip)-1:]:
+            if ip == ip_beg:
+                return True
+            ip_end = d[ip_beg]
+            if ip > ip_beg and ip <= ip_end:
+                return True
+        return False
+
+
+    # reads a list from a file in the format 'whatever:whatever:ip-ip'
+    # (not IPv6 compatible at all)
+    def read_rangelist(self, file):
+        f = open(file, 'r')
+        while True:
+            line = f.readline()
+            if not line:
+                break
+            line = line.strip()
+            if not line or line[0] == '#':
+                continue
+            line = line.split(':')[-1]
+            try:
+                ip1,ip2 = line.split('-')
+            except:
+                ip1 = line
+                ip2 = line
+            try:
+                self.append(ip1.strip(),ip2.strip())
+            except:
+                print '*** WARNING *** could not parse IP range: '+line
+        f.close()
+
+def is_ipv4(ip):
+    return ip.find(':') < 0
+
+def is_valid_ip(ip):
+    try:
+        if is_ipv4(ip):
+            a = ip.split('.')
+            assert len(a) == 4
+            for i in a:
+                chr(int(i))
+            return True
+        to_long_ipv6(ip)
+        return True
+    except:
+        return False

data/dist/BitTornado/launchmanycore.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python
+
+# Written by John Hoffman
+# see LICENSE.txt for license information
+
+from BitTornado import PSYCO
+if PSYCO.psyco:
+    try:
+        import psyco
+        assert psyco.__version__ >= 0x010100f0
+        psyco.full()
+    except:
+        pass
+
+from download_bt1 import BT1Download
+from RawServer import RawServer, UPnP_ERROR
+from RateLimiter import RateLimiter
+from ServerPortHandler import MultiHandler
+from parsedir import parsedir
+from natpunch import UPnP_test
+from random import seed
+from socket import error as socketerror
+from threading import Event
+from sys import argv, exit
+import sys, os
+from clock import clock
+from __init__ import createPeerID, mapbase64, version
+from cStringIO import StringIO
+from traceback import print_exc
+
+try:
+    True
+except:
+    True = 1
+    False = 0
+
+
+def fmttime(n):
+    try:
+        n = int(n)  # n may be None or too large
+        assert n < 5184000  # 60 days
+    except:
+        return 'downloading'
+    m, s = divmod(n, 60)
+    h, m = divmod(m, 60)
+    return '%d:%02d:%02d' % (h, m, s)
+
+class SingleDownload:
+    def __init__(self, controller, hash, response, config, myid):
+        self.controller = controller
+        self.hash = hash
+        self.response = response
+        self.config = config
+        
+        self.doneflag = Event()
+        self.waiting = True
+        self.checking = False
+        self.working = False
+        self.seed = False
+        self.closed = False
+
+        self.status_msg = ''
+        self.status_err = ['']
+        self.status_errtime = 0
+        self.status_done = 0.0
+
+        self.rawserver = controller.handler.newRawServer(hash, self.doneflag)
+
+        d = BT1Download(self.display, self.finished, self.error,
+                        controller.exchandler, self.doneflag, config, response,
+                        hash, myid, self.rawserver, controller.listen_port)
+        self.d = d
+
+    def start(self):
+        if not self.d.saveAs(self.saveAs):
+            self._shutdown()
+            return
+        self._hashcheckfunc = self.d.initFiles()
+        if not self._hashcheckfunc:
+            self._shutdown()
+            return
+        self.controller.hashchecksched(self.hash)
+
+
+    def saveAs(self, name, length, saveas, isdir):
+        return self.controller.saveAs(self.hash, name, saveas, isdir)
+
+    def hashcheck_start(self, donefunc):
+        if self.is_dead():
+            self._shutdown()
+            return
+        self.waiting = False
+        self.checking = True
+        self._hashcheckfunc(donefunc)
+
+    def hashcheck_callback(self):
+        self.checking = False
+        if self.is_dead():
+            self._shutdown()
+            return
+        if not self.d.startEngine(ratelimiter = self.controller.ratelimiter):
+            self._shutdown()
+            return
+        self.d.startRerequester()
+        self.statsfunc = self.d.startStats()
+        self.rawserver.start_listening(self.d.getPortHandler())
+        self.working = True
+
+    def is_dead(self):
+        return self.doneflag.isSet()
+
+    def _shutdown(self):
+        self.shutdown(False)
+
+    def shutdown(self, quiet=True):
+        if self.closed:
+            return
+        self.doneflag.set()
+        self.rawserver.shutdown()
+        if self.checking or self.working:
+            self.d.shutdown()
+        self.waiting = False
+        self.checking = False
+        self.working = False
+        self.closed = True
+        self.controller.was_stopped(self.hash)
+        if not quiet:
+            self.controller.died(self.hash)
+            
+
+    def display(self, activity = None, fractionDone = None):
+        # really only used by StorageWrapper now
+        if activity:
+            self.status_msg = activity
+        if fractionDone is not None:
+            self.status_done = float(fractionDone)
+
+    def finished(self):
+        self.seed = True
+
+    def error(self, msg):
+        if self.doneflag.isSet():
+            self._shutdown()
+        self.status_err.append(msg)
+        self.status_errtime = clock()
+
+
+class LaunchMany:
+    def __init__(self, config, Output):
+        try:
+            self.config = config
+            self.Output = Output
+
+            self.torrent_dir = config['torrent_dir']
+            self.torrent_cache = {}
+            self.file_cache = {}
+            self.blocked_files = {}
+            self.scan_period = config['parse_dir_interval']
+            self.stats_period = config['display_interval']
+
+            self.torrent_list = []
+            self.downloads = {}
+            self.counter = 0
+            self.doneflag = Event()
+
+            self.hashcheck_queue = []
+            self.hashcheck_current = None
+            
+            self.rawserver = RawServer(self.doneflag, config['timeout_check_interval'],
+                              config['timeout'], ipv6_enable = config['ipv6_enabled'],
+                              failfunc = self.failed, errorfunc = self.exchandler)
+            upnp_type = UPnP_test(config['upnp_nat_access'])
+            while True:
+                try:
+                    self.listen_port = self.rawserver.find_and_bind(
+                                    config['minport'], config['maxport'], config['bind'],
+                                    ipv6_socket_style = config['ipv6_binds_v4'],
+                                    upnp = upnp_type, randomizer = config['random_port'])
+                    break
+                except socketerror, e:
+                    if upnp_type and e == UPnP_ERROR:
+                        self.Output.message('WARNING: COULD NOT FORWARD VIA UPnP')
+                        upnp_type = 0
+                        continue
+                    self.failed("Couldn't listen - " + str(e))
+                    return
+
+            self.ratelimiter = RateLimiter(self.rawserver.add_task,
+                                           config['upload_unit_size'])
+            self.ratelimiter.set_upload_rate(config['max_upload_rate'])
+
+            self.handler = MultiHandler(self.rawserver, self.doneflag)
+            seed(createPeerID())
+            self.rawserver.add_task(self.scan, 0)
+            self.rawserver.add_task(self.stats, 0)
+
+            self.handler.listen_forever()
+
+            self.Output.message('shutting down')
+            self.hashcheck_queue = []
+            for hash in self.torrent_list:
+                self.Output.message('dropped "'+self.torrent_cache[hash]['path']+'"')
+                self.downloads[hash].shutdown()
+            self.rawserver.shutdown()
+
+        except:
+            data = StringIO()
+            print_exc(file = data)
+            Output.exception(data.getvalue())
+
+
+    def scan(self):
+        self.rawserver.add_task(self.scan, self.scan_period)
+                                
+        r = parsedir(self.torrent_dir, self.torrent_cache,
+                     self.file_cache, self.blocked_files,
+                     return_metainfo = True, errfunc = self.Output.message)
+
+        ( self.torrent_cache, self.file_cache, self.blocked_files,
+            added, removed ) = r
+
+        for hash, data in removed.items():
+            self.Output.message('dropped "'+data['path']+'"')
+            self.remove(hash)
+        for hash, data in added.items():
+            self.Output.message('added "'+data['path']+'"')
+            self.add(hash, data)
+
+    def stats(self):            
+        self.rawserver.add_task(self.stats, self.stats_period)
+        data = []
+        for hash in self.torrent_list:
+            cache = self.torrent_cache[hash]
+            if self.config['display_path']:
+                name = cache['path']
+            else:
+                name = cache['name']
+            size = cache['length']
+            d = self.downloads[hash]
+            progress = '0.0%'
+            peers = 0
+            seeds = 0
+            seedsmsg = "S"
+            dist = 0.0
+            uprate = 0.0
+            dnrate = 0.0
+            upamt = 0
+            dnamt = 0
+            t = 0
+            if d.is_dead():
+                status = 'stopped'
+            elif d.waiting:
+                status = 'waiting for hash check'
+            elif d.checking:
+                status = d.status_msg
+                progress = '%.1f%%' % (d.status_done*100)
+            else:
+                stats = d.statsfunc()
+                s = stats['stats']
+                if d.seed:
+                    status = 'seeding'
+                    progress = '100.0%'
+                    seeds = s.numOldSeeds
+                    seedsmsg = "s"
+                    dist = s.numCopies
+                else:
+                    if s.numSeeds + s.numPeers:
+                        t = stats['time']
+                        if t == 0:  # unlikely
+                            t = 0.01
+                        status = fmttime(t)
+                    else:
+                        t = -1
+                        status = 'connecting to peers'
+                    progress = '%.1f%%' % (int(stats['frac']*1000)/10.0)
+                    seeds = s.numSeeds
+                    dist = s.numCopies2
+                    dnrate = stats['down']
+                peers = s.numPeers
+                uprate = stats['up']
+                upamt = s.upTotal
+                dnamt = s.downTotal
+                   
+            if d.is_dead() or d.status_errtime+300 > clock():
+                msg = d.status_err[-1]
+            else:
+                msg = ''
+
+            data.append(( name, status, progress, peers, seeds, seedsmsg, dist,
+                          uprate, dnrate, upamt, dnamt, size, t, msg ))
+        stop = self.Output.display(data)
+        if stop:
+            self.doneflag.set()
+
+    def remove(self, hash):
+        self.torrent_list.remove(hash)
+        self.downloads[hash].shutdown()
+        del self.downloads[hash]
+        
+    def add(self, hash, data):
+        c = self.counter
+        self.counter += 1
+        x = ''
+        for i in xrange(3):
+            x = mapbase64[c & 0x3F]+x
+            c >>= 6
+        peer_id = createPeerID(x)
+        d = SingleDownload(self, hash, data['metainfo'], self.config, peer_id)
+        self.torrent_list.append(hash)
+        self.downloads[hash] = d
+        d.start()
+
+
+    def saveAs(self, hash, name, saveas, isdir):
+        x = self.torrent_cache[hash]
+        style = self.config['saveas_style']
+        if style == 1 or style == 3:
+            if saveas:
+                saveas = os.path.join(saveas,x['file'][:-1-len(x['type'])])
+            else:
+                saveas = x['path'][:-1-len(x['type'])]
+            if style == 3:
+                if not os.path.isdir(saveas):
+                    try:
+                        os.mkdir(saveas)
+                    except:
+                        raise OSError("couldn't create directory for "+x['path']
+                                      +" ("+saveas+")")
+                if not isdir:
+                    saveas = os.path.join(saveas, name)
+        else:
+            if saveas:
+                saveas = os.path.join(saveas, name)
+            else:
+                saveas = os.path.join(os.path.split(x['path'])[0], name)
+                
+        if isdir and not os.path.isdir(saveas):
+            try:
+                os.mkdir(saveas)
+            except:
+                raise OSError("couldn't create directory for "+x['path']
+                                      +" ("+saveas+")")
+        return saveas
+
+
+    def hashchecksched(self, hash = None):
+        if hash:
+            self.hashcheck_queue.append(hash)
+        if not self.hashcheck_current:
+            self._hashcheck_start()
+
+    def _hashcheck_start(self):
+        self.hashcheck_current = self.hashcheck_queue.pop(0)
+        self.downloads[self.hashcheck_current].hashcheck_start(self.hashcheck_callback)
+
+    def hashcheck_callback(self):
+        self.downloads[self.hashcheck_current].hashcheck_callback()
+        if self.hashcheck_queue:
+            self._hashcheck_start()
+        else:
+            self.hashcheck_current = None
+
+    def died(self, hash):
+        if self.torrent_cache.has_key(hash):
+            self.Output.message('DIED: "'+self.torrent_cache[hash]['path']+'"')
+        
+    def was_stopped(self, hash):
+        try:
+            self.hashcheck_queue.remove(hash)
+        except:
+            pass
+        if self.hashcheck_current == hash:
+            self.hashcheck_current = None
+            if self.hashcheck_queue:
+                self._hashcheck_start()
+
+    def failed(self, s):
+        self.Output.message('FAILURE: '+s)
+
+    def exchandler(self, s):
+        self.Output.exception(s)