quartz_torrent 0.0.1

data/lib/quartz_torrent/memprofiler.rb

@@ -0,0 +1,27 @@
+module QuartzTorrent
+
+  # Utility class used for debugging memory leaks. It can be used to count the number of reachable
+  # instances of selected classes.
+  class MemProfiler
+    def initialize
+      @classes = []
+    end
+
+    # Add a class to the list of classes we count the reachable instances of.
+    def trackClass(clazz)
+      @classes.push clazz
+    end
+
+    # Return a hashtable keyed by class where the value is the number of that class of object still reachable.
+    def getCounts
+      result = {}
+      @classes.each do |c|
+        count = 0
+        ObjectSpace.each_object(c){ count += 1 }
+        result[c] = count
+      end
+      result
+    end
+  end
+
+end

data/lib/quartz_torrent/metainfo.rb

@@ -0,0 +1,221 @@
+require 'quartz_torrent/log'
+require 'bencode'
+require 'digest/sha1'
+
+module QuartzTorrent
+  
+  # Torrent metainfo structure. This is what's usually found in .torrent files. This class
+  # generally follows the structure of the metadata format.
+  class Metainfo
+
+    # If 'v' is null, throw an exception. Otherwise return 'v'.
+    def self.valueOrException(v, msg)
+      if ! v
+        LogManager.getLogger("metainfo").error msg
+        raise "Invalid torrent metainfo"
+      end
+      v
+    end
+
+    # Information about a file contained in the torrent.
+    class FileInfo
+      def initialize(length = nil, path = nil)
+        @length = length
+        @path = path
+      end
+
+      # Relative path to the file. For a single-file torrent this is simply the name of the file. For a multi-file torrent,
+      # this is the directory names from the torrent and the filename separated by the file separator.
+      attr_accessor :path
+      # Length of the file.
+      attr_accessor :length
+  
+      # Create a FileInfo object from a bdecoded structure.
+      def self.createFromBdecode(bdecode)
+        result = FileInfo.new
+        result.length = Metainfo.valueOrException(bdecode['length'], "Torrent metainfo listed multiple files, and one is missing the length property.")
+        path = Metainfo.valueOrException(bdecode['path'], "Torrent metainfo listed multiple files, and one is missing the path property.")
+
+        result.path = ""
+        path.each do |part|
+          result.path << File::SEPARATOR if result.path.length > 0
+          result.path << part
+        end
+  
+        result
+      end
+    end
+
+    # The 'info' property of the torrent metainfo.
+    class Info
+      def initialize
+        @pieceLen = nil
+        @pieces = nil
+        @private = nil
+        # Suggested file or directory name
+        @name = nil
+        # List of file info for files in the torrent. These include the directory name if this is a
+        # multi-file download. For a single-file download the 
+        @files = []
+        @logger = LogManager.getLogger("metainfo")
+      end
+
+      # Array of FileInfo objects
+      attr_accessor :files
+      # Suggested file or directory name
+      attr_accessor :name
+      # Length of each piece in bytes. The last piece may be shorter than this.
+      attr_accessor :pieceLen
+      # Array of SHA1 digests of all peices. These digests are in binary format. 
+      attr_accessor :pieces
+      # True if no external peer source is allowed.
+      attr_accessor :private
+
+      # Total length of the torrent data in bytes.
+      def dataLength
+        files.reduce(0){ |memo,f| memo + f.length}
+      end
+
+      # Create a FileInfo object from a bdecoded structure.
+      def self.createFromBdecode(infoDict)
+        result = Info.new
+        result.pieceLen = infoDict['piece length']
+        result.private = infoDict['private']
+        result.pieces = parsePieces(Metainfo.valueOrException(infoDict['pieces'], "Torrent metainfo is missing the pieces property."))
+        result.name = Metainfo.valueOrException(infoDict['name'], "Torrent metainfo is missing the name property.")
+
+        if infoDict.has_key? 'files'
+          # This is a multi-file torrent
+          infoDict['files'].each do |file|
+            result.files.push FileInfo.createFromBdecode(file)
+            result.files.last.path = result.name + File::SEPARATOR + result.files.last.path
+          end
+        else
+          # This is a single-file torrent
+          length = Metainfo.valueOrException(infoDict['length'], "Torrent metainfo listed a single file, but it is missing the length property.")
+          result.files.push FileInfo.new(length, result.name)
+        end
+
+        result
+      end
+
+      # BEncode this info and return the result.
+      def bencode
+        hash = {}
+    
+        raise "Cannot encode Info object with nil pieceLen" if ! @pieceLen
+        raise "Cannot encode Info object with nil name" if ! @name
+        raise "Cannot encode Info object with nil pieces" if ! @pieces
+        raise "Cannot encode Info object with nil files or empty files" if ! @files || @files.empty?
+
+        hash['piece length'] = @pieceLen
+        hash['private'] = @private if @private
+        hash['name'] = @name
+        hash['pieces'] = @pieces.join
+
+        if @files.length > 1
+          # This is a multi-file torrent
+          # When we loaded the torrent, we prepended the 'name' element of the info hash to the path. We need to remove this
+          # name element to end up with the same result.
+          hash['files'] = @files.collect{ |file| {'length' => file.length, 'path' => file.path.split(File::SEPARATOR).drop(1) }  }
+        else
+          hash['length'] = @files.first.length
+        end
+        hash.bencode
+      end
+
+      private
+      # Parse the pieces of the torrent out of the metainfo.
+      def self.parsePieces(p)
+        # Break into 20-byte portions.
+        if p.length % 20 != 0
+          @logger.error "Torrent metainfo contained a pieces property that was not a multiple of 20 bytes long."
+          raise "Invalid torrent metainfo"
+        end
+
+        result = []
+        index = 0
+        while index < p.length
+          result.push p[index,20].unpack("a20")[0]
+          index += 20
+        end
+        result
+      end
+    end
+
+    def initialize
+      @info = nil
+      @announce = nil
+      @announceList = nil
+      @creationDate = nil
+      @comment = nil
+      @createdBy = nil
+      @encoding = nil
+    end
+
+    # A Metainfo::Info object
+    attr_accessor :info
+
+    # A 20-byte SHA1 hash of the value of the info key from the metainfo. This is neede when connecting
+    # to the tracker or to a peer.
+    attr_accessor :infoHash
+
+    # Announce URL of the tracker
+    attr_accessor :announce
+    attr_accessor :announceList
+
+    # Creation date as a ruby Time object
+    attr_accessor :creationDate
+    # Comment
+    attr_accessor :comment
+    # Created By
+    attr_accessor :createdBy
+    # The string encoding format used to generate the pieces part of the info dictionary in the .torrent metafile
+    attr_accessor :encoding
+
+    # Create a Metainfo object from the passed bencoded string.
+    def self.createFromString(data)
+      logger = LogManager.getLogger("metainfo")
+
+      decoded = data.bdecode
+      logger.debug "Decoded torrent metainfo: #{decoded.inspect}"
+      result = Metainfo.new
+      result.createdBy = decoded['created by']
+      result.comment = decoded['comment']
+      result.creationDate = decoded['creation date']
+      if result.creationDate 
+        if !result.creationDate.is_a?(Integer)
+          if result.creationDate =~ /^\d+$/
+            result.creationDate = result.creationDate.to_i
+          else
+            logger.warn "Torrent metainfo contained invalid date: '#{result.creationDate.class}'"
+            result.creationDate = nil
+          end
+        end
+
+        result.creationDate = Time.at(result.creationDate) if result.creationDate
+      end
+      result.encoding = decoded['encoding']
+      result.announce = decoded['announce'].strip
+      result.announceList = decoded['announce-list']
+      result.info = Info.createFromBdecode(decoded['info'])
+      result.infoHash = Digest::SHA1.digest( decoded['info'].bencode )
+    
+      result
+    end
+
+    # Create a Metainfo object from the passed IO.
+    def self.createFromIO(io)
+      self.createFromString(io.read)
+    end
+
+    # Create a Metainfo object from the named file.
+    def self.createFromFile(path)
+      result = 
+      File.open(path,"rb") do |io|
+        result = self.createFromIO(io)
+      end
+      result
+    end
+  end
+end

data/lib/quartz_torrent/metainfopiecestate.rb

@@ -0,0 +1,265 @@
+require 'quartz_torrent/util'
+require 'quartz_torrent/filemanager'
+require 'quartz_torrent/metainfo'
+require "quartz_torrent/piecemanagerrequestmetadata.rb"
+require "quartz_torrent/peerholder.rb"
+require 'digest/sha1' 
+require 'fileutils' 
+
+# This class is used when we don't have the info struct for the torrent (no .torrent file) and must
+# download it piece by piece from peers. It keeps track of the pieces we have.
+#
+# When a piece is requested from a peer and that peer responds with a reject saying it doesn't have
+# that metainfo piece, we take a simple approach and mark that peer as bad, and don't request any more 
+# pieces from that peer even though they may have other pieces. This simplifies the code.
+module QuartzTorrent
+  class MetainfoPieceState
+    BlockSize = 16384
+  
+    # Create a new MetainfoPieceState that can be used to manage downloading the metainfo
+    # for a torrent. The metainfo is stored in a file under baseDirectory named <infohash>.info,
+    # where <infohash> is infoHash hex-encoded. The parameter metainfoSize should be the size of
+    # the metainfo, and info can be used to pass in the complete metainfo Info object if it is available. This
+    # is needed for when other peers request the metainfo from us.
+    def initialize(baseDirectory, infoHash, metainfoSize = nil, info = nil)
+ 
+      @logger = LogManager.getLogger("metainfo_piece_state")
+
+      @requestTimeout = 5
+      @baseDirectory = baseDirectory
+      @infoFileName = MetainfoPieceState.generateInfoFileName(infoHash)
+
+      path = infoFilePath
+
+      completed = MetainfoPieceState.downloaded(baseDirectory, infoHash)
+      metainfoSize = File.size(path) if ! metainfoSize && completed
+
+      if !completed && info 
+        File.open(path, "wb") do |file|
+          bencoded = info.bencode
+          metainfoSize = bencoded.length
+          file.write bencoded
+          # Sanity check
+          testInfoHash = Digest::SHA1.digest( bencoded )
+          raise "The computed infoHash #{QuartzTorrent.bytesToHex(testInfoHash)} doesn't match the original infoHash #{QuartzTorrent.bytesToHex(infoHash)}" if testInfoHash != infoHash
+        end
+      end
+
+      raise "Unless the metainfo has already been successfully downloaded or the torrent file is available, the metainfoSize is needed" if ! metainfoSize
+
+      # We use the PieceManager to manage the pieces of the metainfo file. The PieceManager is designed
+      # for the pieces and blocks of actual torrent data, so we need to build a fake metainfo object that
+      # describes our one metainfo file itself so that we can store the pieces if it on disk.
+      # In this case we map metainfo pieces to 'torrent' pieces, and our blocks are the full length of the 
+      # metainfo piece.
+      torrinfo = Metainfo::Info.new
+      torrinfo.pieceLen = BlockSize
+      torrinfo.files = []
+      torrinfo.files.push Metainfo::FileInfo.new(metainfoSize, @infoFileName)
+    
+
+      @pieceManager = PieceManager.new(baseDirectory, torrinfo)
+      @pieceManagerRequests = {}
+
+      @numPieces = metainfoSize/BlockSize
+      @numPieces += 1 if (metainfoSize%BlockSize) != 0
+      @completePieces = Bitfield.new(@numPieces)
+      @completePieces.setAll if info || completed
+
+      @lastPieceLength = metainfoSize - (@numPieces-1)*BlockSize
+  
+      @badPeers = PeerHolder.new
+      @requestedPieces = Bitfield.new(@numPieces)
+      @requestedPieces.clearAll
+
+      @metainfoLength = metainfoSize
+
+      # Time at which the piece in requestedPiece was requested. Used for timeouts.
+      @pieceRequestTime = []
+    end
+
+    # Check if the metainfo has already been downloaded successfully during a previous session.
+    # Returns the completed, Metainfo::Info object if it is complete, and nil otherwise.
+    def self.downloaded(baseDirectory, infoHash)
+      logger = LogManager.getLogger("metainfo_piece_state")
+      infoFileName = generateInfoFileName(infoHash)
+      path = "#{baseDirectory}#{File::SEPARATOR}#{infoFileName}"
+
+      result = nil
+      if File.exists?(path)
+        File.open(path, "rb") do |file|
+          bencoded = file.read
+          # Sanity check
+          testInfoHash = Digest::SHA1.digest( bencoded )
+          if testInfoHash == infoHash
+            result = Metainfo::Info.createFromBdecode(bencoded.bdecode) 
+          else
+            logger.info "the computed SHA1 hash doesn't match the specified infoHash in #{path}"
+          end
+        end
+      else
+        logger.info "the metainfo file #{path} doesn't exist"
+      end
+      result
+    end
+
+    attr_accessor :infoFileName
+    attr_accessor :metainfoLength
+
+    # Return the number of bytes of the metainfo that we have downloaded so far.
+    def metainfoCompletedLength
+      num = @completePieces.countSet
+      # Last block may be smaller
+      extra = 0
+      if @completePieces.set?(@completePieces.length-1)
+        num -= 1
+        extra = @lastPieceLength
+      end
+      num*BlockSize + extra
+    end
+
+    def pieceCompleted?(pieceIndex)
+      @completePieces.set? pieceIndex
+    end
+
+    # Do we have all the pieces of the metadata?
+    def complete?
+      @completePieces.allSet?
+    end
+  
+    # Get the completed metainfo. Raises an exception if it's not yet complete.
+    def completedMetainfo
+      raise "Metadata is not yet complete" if ! complete?
+    end
+
+    def savePiece(pieceIndex, data)
+      id = @pieceManager.writeBlock pieceIndex, 0, data
+      @pieceManagerRequests[id] = PieceManagerRequestMetadata.new(:write, pieceIndex)
+      id
+    end
+
+    def readPiece(pieceIndex)
+      length = BlockSize
+      length = @lastPieceLength if pieceIndex == @numPieces - 1
+      id = @pieceManager.readBlock pieceIndex, 0, length
+      #result = manager.nextResult
+      @pieceManagerRequests[id] = PieceManagerRequestMetadata.new(:read, pieceIndex)
+      id
+    end
+
+    # Check the results of savePiece and readPiece. This method returns a list
+    # of the PieceManager results.
+    def checkResults
+      results = []
+      while true
+        result = @pieceManager.nextResult
+        break if ! result
+
+        results.push result
+          
+        metaData = @pieceManagerRequests.delete(result.requestId)
+        if ! metaData
+          @logger.error "Can't find metadata for PieceManager request #{result.requestId}"
+          next
+        end 
+
+        if metaData.type == :write
+          if result.successful?
+            @completePieces.set(metaData.data)
+          else
+            @requestedPieces.clear(metaData.data)
+            @pieceRequestTime[metaData.data] = nil
+            @logger.error "Writing metainfo piece failed: #{result.error}"
+          end
+        elsif metaData.type == :read
+          if ! result.successful?
+            @logger.error "Reading metainfo piece failed: #{result.error}"
+          end
+        end
+      end
+      results
+    end
+
+    def findRequestablePieces
+      piecesRequired = []
+
+      removeOldRequests
+
+      @numPieces.times do |pieceIndex|
+        piecesRequired.push pieceIndex if ! @completePieces.set?(pieceIndex) && ! @requestedPieces.set?(pieceIndex)
+      end
+
+      piecesRequired
+    end
+
+    def findRequestablePeers(classifiedPeers)
+      result = []
+
+      classifiedPeers.establishedPeers.each do |peer|
+        result.push peer if ! @badPeers.findByAddr(peer.trackerPeer.ip, peer.trackerPeer.port)
+      end
+
+      result
+    end
+
+    # Set whether the piece with the passed pieceIndex is requested or not.
+    def setPieceRequested(pieceIndex, bool)
+      if bool
+        @requestedPieces.set pieceIndex
+        @pieceRequestTime[pieceIndex] = Time.new
+      else
+        @requestedPieces.clear pieceIndex
+        @pieceRequestTime[pieceIndex] = nil
+      end
+    end
+
+    def markPeerBad(peer)
+      @badPeers.add peer
+    end
+
+    # Flush all pieces to disk
+    def flush
+      id = @pieceManager.flush
+      @pieceManagerRequests[id] = PieceManagerRequestMetadata.new(:flush, nil)
+      @pieceManager.wait
+    end
+
+    # Wait for the next a pending request to complete.
+    def wait
+      @pieceManager.wait
+    end
+
+    def self.generateInfoFileName(infoHash)
+      "#{QuartzTorrent.bytesToHex(infoHash)}.info"
+    end
+
+    # Remove the metainfo file
+    def remove
+      path = infoFilePath
+      FileUtils.rm path
+    end
+
+    # Stop the underlying PieceManager.
+    def stop
+      @pieceManager.stop
+    end
+
+    private
+    # Remove any pending requests after a timeout.
+    def removeOldRequests
+      now = Time.new
+      @requestedPieces.length.times do |i|
+        if @requestedPieces.set? i
+          if now - @pieceRequestTime[i] > @requestTimeout
+            @requestedPieces.clear i
+            @pieceRequestTime[i] = nil
+          end
+        end
+      end
+    end
+
+    def infoFilePath
+      "#{@baseDirectory}#{File::SEPARATOR}#{@infoFileName}"
+    end
+  end
+end