murder 0.0.0.pre

data/dist/BitTornado/BT1/FileSelector.py

@@ -0,0 +1,245 @@
+# Written by John Hoffman
+# see LICENSE.txt for license information
+
+from random import shuffle
+from traceback import print_exc
+try:
+    True
+except:
+    True = 1
+    False = 0
+
+
+class FileSelector:
+    def __init__(self, files, piece_length, bufferdir,
+                 storage, storagewrapper, sched, failfunc):
+        self.files = files
+        self.storage = storage
+        self.storagewrapper = storagewrapper
+        self.sched = sched
+        self.failfunc = failfunc
+        self.downloader = None
+        self.picker = None
+
+        storage.set_bufferdir(bufferdir)
+        
+        self.numfiles = len(files)
+        self.priority = [1] * self.numfiles
+        self.new_priority = None
+        self.new_partials = None
+        self.filepieces = []
+        total = 0L
+        for file, length in files:
+            if not length:
+                self.filepieces.append(())
+            else:
+                pieces = range( int(total/piece_length),
+                                int((total+length-1)/piece_length)+1 )
+                self.filepieces.append(tuple(pieces))
+                total += length
+        self.numpieces = int((total+piece_length-1)/piece_length)
+        self.piece_priority = [1] * self.numpieces
+        
+
+
+    def init_priority(self, new_priority):
+        try:
+            assert len(new_priority) == self.numfiles
+            for v in new_priority:
+                assert type(v) in (type(0),type(0L))
+                assert v >= -1
+                assert v <= 2
+        except:
+#           print_exc()            
+            return False
+        try:
+            files_updated = False
+            for f in xrange(self.numfiles):
+                if new_priority[f] < 0:
+                    self.storage.disable_file(f)
+                    files_updated = True
+            if files_updated:
+                self.storage.reset_file_status()
+            self.new_priority = new_priority
+        except (IOError, OSError), e:
+            self.failfunc("can't open partial file for "
+                          + self.files[f][0] + ': ' + str(e))
+            return False
+        return True
+
+    '''
+    d['priority'] = [file #1 priority [,file #2 priority...] ]
+                    a list of download priorities for each file.
+                    Priority may be -1, 0, 1, 2.  -1 = download disabled,
+                    0 = highest, 1 = normal, 2 = lowest.
+    Also see Storage.pickle and StorageWrapper.pickle for additional keys.
+    '''
+    def unpickle(self, d):
+        if d.has_key('priority'):
+            if not self.init_priority(d['priority']):
+                return
+        pieces = self.storage.unpickle(d)
+        if not pieces:  # don't bother, nothing restoreable
+            return
+        new_piece_priority = self._get_piece_priority_list(self.new_priority)
+        self.storagewrapper.reblock([i == -1 for i in new_piece_priority])
+        self.new_partials = self.storagewrapper.unpickle(d, pieces)
+
+
+    def tie_in(self, picker, cancelfunc, requestmorefunc, rerequestfunc):
+        self.picker = picker
+        self.cancelfunc = cancelfunc
+        self.requestmorefunc = requestmorefunc
+        self.rerequestfunc = rerequestfunc
+
+        if self.new_priority:
+            self.priority = self.new_priority
+            self.new_priority = None
+            self.new_piece_priority = self._set_piece_priority(self.priority)
+
+        if self.new_partials:
+            shuffle(self.new_partials)
+            for p in self.new_partials:
+                self.picker.requested(p)
+        self.new_partials = None
+        
+
+    def _set_files_disabled(self, old_priority, new_priority):
+        old_disabled = [p == -1 for p in old_priority]
+        new_disabled = [p == -1 for p in new_priority]
+        data_to_update = []
+        for f in xrange(self.numfiles):
+            if new_disabled[f] != old_disabled[f]:
+                data_to_update.extend(self.storage.get_piece_update_list(f))
+        buffer = []
+        for piece, start, length in data_to_update:
+            if self.storagewrapper.has_data(piece):
+                data = self.storagewrapper.read_raw(piece, start, length)
+                if data is None:
+                    return False
+                buffer.append((piece, start, data))
+
+        files_updated = False        
+        try:
+            for f in xrange(self.numfiles):
+                if new_disabled[f] and not old_disabled[f]:
+                    self.storage.disable_file(f)
+                    files_updated = True
+                if old_disabled[f] and not new_disabled[f]:
+                    self.storage.enable_file(f)
+                    files_updated = True
+        except (IOError, OSError), e:
+            if new_disabled[f]:
+                msg = "can't open partial file for "
+            else:
+                msg = 'unable to open '
+            self.failfunc(msg + self.files[f][0] + ': ' + str(e))
+            return False
+        if files_updated:
+            self.storage.reset_file_status()
+
+        changed_pieces = {}
+        for piece, start, data in buffer:
+            if not self.storagewrapper.write_raw(piece, start, data):
+                return False
+            data.release()
+            changed_pieces[piece] = 1
+        if not self.storagewrapper.doublecheck_data(changed_pieces):
+            return False
+
+        return True        
+
+
+    def _get_piece_priority_list(self, file_priority_list):
+        l = [-1] * self.numpieces
+        for f in xrange(self.numfiles):
+            if file_priority_list[f] == -1:
+                continue
+            for i in self.filepieces[f]:
+                if l[i] == -1:
+                    l[i] = file_priority_list[f]
+                    continue
+                l[i] = min(l[i],file_priority_list[f])
+        return l
+        
+
+    def _set_piece_priority(self, new_priority):
+        was_complete = self.storagewrapper.am_I_complete()
+        new_piece_priority = self._get_piece_priority_list(new_priority)
+        pieces = range(self.numpieces)
+        shuffle(pieces)
+        new_blocked = []
+        new_unblocked = []
+        for piece in pieces:
+            self.picker.set_priority(piece,new_piece_priority[piece])
+            o = self.piece_priority[piece] == -1
+            n = new_piece_priority[piece] == -1
+            if n and not o:
+                new_blocked.append(piece)
+            if o and not n:
+                new_unblocked.append(piece)
+        if new_blocked:
+            self.cancelfunc(new_blocked)
+        self.storagewrapper.reblock([i == -1 for i in new_piece_priority])
+        if new_unblocked:
+            self.requestmorefunc(new_unblocked)
+        if was_complete and not self.storagewrapper.am_I_complete():
+            self.rerequestfunc()
+
+        return new_piece_priority        
+
+
+    def set_priorities_now(self, new_priority = None):
+        if not new_priority:
+            new_priority = self.new_priority
+            self.new_priority = None    # potential race condition
+            if not new_priority:
+                return
+        old_priority = self.priority
+        self.priority = new_priority
+        if not self._set_files_disabled(old_priority, new_priority):
+            return
+        self.piece_priority = self._set_piece_priority(new_priority)
+
+    def set_priorities(self, new_priority):
+        self.new_priority = new_priority
+        self.sched(self.set_priorities_now)
+        
+    def set_priority(self, f, p):
+        new_priority = self.get_priorities()
+        new_priority[f] = p
+        self.set_priorities(new_priority)
+
+    def get_priorities(self):
+        priority = self.new_priority
+        if not priority:
+            priority = self.priority    # potential race condition
+        return [i for i in priority]
+
+    def __setitem__(self, index, val):
+        self.set_priority(index, val)
+
+    def __getitem__(self, index):
+        try:
+            return self.new_priority[index]
+        except:
+            return self.priority[index]
+
+
+    def finish(self):
+        for f in xrange(self.numfiles):
+            if self.priority[f] == -1:
+                self.storage.delete_file(f)
+
+    def pickle(self):
+        d = {'priority': self.priority}
+        try:
+            s = self.storage.pickle()
+            sw = self.storagewrapper.pickle()
+            for k in s.keys():
+                d[k] = s[k]
+            for k in sw.keys():
+                d[k] = sw[k]
+        except (IOError, OSError):
+            pass
+        return d

data/dist/BitTornado/BT1/Filter.py

@@ -0,0 +1,12 @@
+class Filter:
+    def __init__(self, callback):
+        self.callback = callback
+
+    def check(self, ip, paramslist, headers):
+
+        def params(key, default = None, l = paramslist):
+            if l.has_key(key):
+                return l[key][0]
+            return default
+
+        return None

data/dist/BitTornado/BT1/HTTPDownloader.py

@@ -0,0 +1,251 @@
+# Written by John Hoffman
+# see LICENSE.txt for license information
+
+from BitTornado.CurrentRateMeasure import Measure
+from random import randint
+from urlparse import urlparse
+from httplib import HTTPConnection
+from urllib import quote
+from threading import Thread
+from BitTornado.__init__ import product_name,version_short
+try:
+    True
+except:
+    True = 1
+    False = 0
+
+EXPIRE_TIME = 60 * 60
+
+VERSION = product_name+'/'+version_short
+
+class haveComplete:
+    def complete(self):
+        return True
+    def __getitem__(self, x):
+        return True
+haveall = haveComplete()
+
+class SingleDownload:
+    def __init__(self, downloader, url):
+        self.downloader = downloader
+        self.baseurl = url
+        try:
+            (scheme, self.netloc, path, pars, query, fragment) = urlparse(url)
+        except:
+            self.downloader.errorfunc('cannot parse http seed address: '+url)
+            return
+        if scheme != 'http':
+            self.downloader.errorfunc('http seed url not http: '+url)
+            return
+        try:
+            self.connection = HTTPConnection(self.netloc)
+        except:
+            self.downloader.errorfunc('cannot connect to http seed: '+url)
+            return
+        self.seedurl = path
+        if pars:
+            self.seedurl += ';'+pars
+        self.seedurl += '?'
+        if query:
+            self.seedurl += query+'&'
+        self.seedurl += 'info_hash='+quote(self.downloader.infohash)
+
+        self.measure = Measure(downloader.max_rate_period)
+        self.index = None
+        self.url = ''
+        self.requests = []
+        self.request_size = 0
+        self.endflag = False
+        self.error = None
+        self.retry_period = 30
+        self._retry_period = None
+        self.errorcount = 0
+        self.goodseed = False
+        self.active = False
+        self.cancelled = False
+        self.resched(randint(2,10))
+
+    def resched(self, len = None):
+        if len is None:
+            len = self.retry_period
+        if self.errorcount > 3:
+            len = len * (self.errorcount - 2)
+        self.downloader.rawserver.add_task(self.download, len)
+
+    def _want(self, index):
+        if self.endflag:
+            return self.downloader.storage.do_I_have_requests(index)
+        else:
+            return self.downloader.storage.is_unstarted(index)
+
+    def download(self):
+        self.cancelled = False
+        if self.downloader.picker.am_I_complete():
+            self.downloader.downloads.remove(self)
+            return
+        self.index = self.downloader.picker.next(haveall, self._want)
+        if ( self.index is None and not self.endflag
+                     and not self.downloader.peerdownloader.has_downloaders() ):
+            self.endflag = True
+            self.index = self.downloader.picker.next(haveall, self._want)
+        if self.index is None:
+            self.endflag = True
+            self.resched()
+        else:
+            self.url = ( self.seedurl+'&piece='+str(self.index) )
+            self._get_requests()
+            if self.request_size < self.downloader.storage._piecelen(self.index):
+                self.url += '&ranges='+self._request_ranges()
+            rq = Thread(target = self._request)
+            rq.setDaemon(False)
+            rq.start()
+            self.active = True
+
+    def _request(self):
+        import encodings.ascii
+        import encodings.punycode
+        import encodings.idna
+        
+        self.error = None
+        self.received_data = None
+        try:
+            self.connection.request('GET',self.url, None,
+                                {'User-Agent': VERSION})
+            r = self.connection.getresponse()
+            self.connection_status = r.status
+            self.received_data = r.read()
+        except Exception, e:
+            self.error = 'error accessing http seed: '+str(e)
+            try:
+                self.connection.close()
+            except:
+                pass
+            try:
+                self.connection = HTTPConnection(self.netloc)
+            except:
+                self.connection = None  # will cause an exception and retry next cycle
+        self.downloader.rawserver.add_task(self.request_finished)
+
+    def request_finished(self):
+        self.active = False
+        if self.error is not None:
+            if self.goodseed:
+                self.downloader.errorfunc(self.error)
+            self.errorcount += 1
+        if self.received_data:
+            self.errorcount = 0
+            if not self._got_data():
+                self.received_data = None
+        if not self.received_data:
+            self._release_requests()
+            self.downloader.peerdownloader.piece_flunked(self.index)
+        if self._retry_period:
+            self.resched(self._retry_period)
+            self._retry_period = None
+            return
+        self.resched()
+
+    def _got_data(self):
+        if self.connection_status == 503:   # seed is busy
+            try:
+                self.retry_period = max(int(self.received_data),5)
+            except:
+                pass
+            return False
+        if self.connection_status != 200:
+            self.errorcount += 1
+            return False
+        self._retry_period = 1
+        if len(self.received_data) != self.request_size:
+            if self.goodseed:
+                self.downloader.errorfunc('corrupt data from http seed - redownloading')
+            return False
+        self.measure.update_rate(len(self.received_data))
+        self.downloader.measurefunc(len(self.received_data))
+        if self.cancelled:
+            return False
+        if not self._fulfill_requests():
+            return False
+        if not self.goodseed:
+            self.goodseed = True
+            self.downloader.seedsfound += 1
+        if self.downloader.storage.do_I_have(self.index):
+            self.downloader.picker.complete(self.index)
+            self.downloader.peerdownloader.check_complete(self.index)
+            self.downloader.gotpiecefunc(self.index)
+        return True
+    
+    def _get_requests(self):
+        self.requests = []
+        self.request_size = 0L
+        while self.downloader.storage.do_I_have_requests(self.index):
+            r = self.downloader.storage.new_request(self.index)
+            self.requests.append(r)
+            self.request_size += r[1]
+        self.requests.sort()
+
+    def _fulfill_requests(self):
+        start = 0L
+        success = True
+        while self.requests:
+            begin, length = self.requests.pop(0)
+            if not self.downloader.storage.piece_came_in(self.index, begin,
+                            self.received_data[start:start+length]):
+                success = False
+                break
+            start += length
+        return success
+
+    def _release_requests(self):
+        for begin, length in self.requests:
+            self.downloader.storage.request_lost(self.index, begin, length)
+        self.requests = []
+
+    def _request_ranges(self):
+        s = ''
+        begin, length = self.requests[0]
+        for begin1, length1 in self.requests[1:]:
+            if begin + length == begin1:
+                length += length1
+                continue
+            else:
+                if s:
+                    s += ','
+                s += str(begin)+'-'+str(begin+length-1)
+                begin, length = begin1, length1
+        if s:
+            s += ','
+        s += str(begin)+'-'+str(begin+length-1)
+        return s
+        
+    
+class HTTPDownloader:
+    def __init__(self, storage, picker, rawserver,
+                 finflag, errorfunc, peerdownloader,
+                 max_rate_period, infohash, measurefunc, gotpiecefunc):
+        self.storage = storage
+        self.picker = picker
+        self.rawserver = rawserver
+        self.finflag = finflag
+        self.errorfunc = errorfunc
+        self.peerdownloader = peerdownloader
+        self.infohash = infohash
+        self.max_rate_period = max_rate_period
+        self.gotpiecefunc = gotpiecefunc
+        self.measurefunc = measurefunc
+        self.downloads = []
+        self.seedsfound = 0
+
+    def make_download(self, url):
+        self.downloads.append(SingleDownload(self, url))
+        return self.downloads[-1]
+
+    def get_downloads(self):
+        if self.finflag.isSet():
+            return []
+        return self.downloads
+
+    def cancel_piece_download(self, pieces):
+        for d in self.downloads:
+            if d.active and d.index in pieces:
+                d.cancelled = True