quartz_flow 0.0.1

data/lib/quartz_flow/torrent_manager.rb

@@ -0,0 +1,247 @@
+require 'open-uri'
+require 'quartz_torrent.rb'
+require 'fileutils'
+require 'quartz_flow/usagetracker'
+require 'thread'
+
+class TorrentManager
+  def initialize(peerClient, torrentFileDir, monthlyResetDay)
+    @peerClient = peerClient
+    @cachedTorrentData = nil
+    @cachedAt = nil
+    @cacheLifetime = 2
+    @torrentFileDir = torrentFileDir
+    @peerClientStopped = false
+    @usageTracker = UsageTracker.new(monthlyResetDay)
+    # Start a thread to keep track of usage.
+    startUsageTrackerThread
+  end
+
+  attr_reader :peerClient
+
+  def torrentData(infoHash = nil)
+    if (! @cachedTorrentData || Time.new - @cachedAt > @cacheLifetime) && ! @peerClientStopped
+      @cachedTorrentData = @peerClient.torrentData
+      @cachedAt = Time.new
+    end
+    
+    @cachedTorrentData
+  end
+
+  def stopPeerClient
+    @peerClient.stop
+    @peerClientStopped = true
+  end
+
+  # Start torrents that already exist in the torrent file directory
+  def startExistingTorrents
+    Dir.new(@torrentFileDir).each do |e|
+      path = @torrentFileDir + File::SEPARATOR + e
+      if e =~ /\.torrent$/
+        puts "Starting .torrent '#{path}'"
+        startTorrentFile(path)
+      elsif e =~ /\.magnet$/   
+        magnet = loadMagnet(path)
+        puts "Starting magnet '#{magnet.raw}'"
+        startMagnet magnet
+      end
+    end
+  end
+
+  # Convert torrent data such that:
+  # - The TorrentDataDelegate objects are converted to hashes.
+  def simplifiedTorrentData
+    result = {}
+    torrentData.each do |k,d|
+      h = d.to_h
+      asciiInfoHash = QuartzTorrent::bytesToHex(h[:infoHash])
+      h[:infoHash] = asciiInfoHash
+      h[:downloadRate] = QuartzTorrent::Formatter.formatSpeed(h[:downloadRate])
+      h[:uploadRate] = QuartzTorrent::Formatter.formatSpeed(h[:uploadRate])
+      h[:downloadRateDataOnly] = QuartzTorrent::Formatter.formatSpeed(h[:downloadRateDataOnly])
+      h[:uploadRateDataOnly] = QuartzTorrent::Formatter.formatSpeed(h[:uploadRateDataOnly])
+      h[:dataLength] = QuartzTorrent::Formatter.formatSize(h[:dataLength])
+      h[:completedBytes] = QuartzTorrent::Formatter.formatSize(h[:completedBytes])
+      # Sort peers
+      h[:peers].sort! do |a,b|
+        c = (b[:uploadRate].to_i <=> a[:uploadRate].to_i)
+        c = (b[:downloadRate].to_i <=> a[:downloadRate].to_i) if c == 0
+        c
+      end
+      # Format peer rates
+      h[:peers].each do |p| 
+        p[:uploadRate] = QuartzTorrent::Formatter.formatSpeed(p[:uploadRate])
+        p[:downloadRate] = QuartzTorrent::Formatter.formatSpeed(p[:downloadRate])
+      end
+      if h[:info] 
+        h[:info][:files].each do |file|
+          file[:length] = QuartzTorrent::Formatter.formatSize(file[:length])
+        end
+      end
+      h[:uploadRateLimit] = QuartzTorrent::Formatter.formatSpeed(h[:uploadRateLimit])
+      h[:downloadRateLimit] = QuartzTorrent::Formatter.formatSize(h[:downloadRateLimit])
+      h[:bytesUploaded] = QuartzTorrent::Formatter.formatSize(h[:bytesUploaded])
+      h[:bytesDownloaded] = QuartzTorrent::Formatter.formatSize(h[:bytesDownloaded])
+
+      h[:completePieces] = d.completePieceBitfield ? d.completePieceBitfield.countSet : 0
+      h[:totalPieces] = d.completePieceBitfield ? d.completePieceBitfield.length : 0
+
+      result[asciiInfoHash] = h
+    end
+    result
+  end
+
+  # Download a .torrent file from a specified URL. Return the path to the 
+  # downloaded .torrent file.
+  def downloadTorrentFile(url)
+    # open-uri doesn't handle [ and ] properly
+    encodedSourcePath = URI.escape(url, /[\[\]]/)
+
+    path = nil
+    open(encodedSourcePath) do |f|
+      uriPath = f.base_uri.path
+      raise "The file '#{uriPath}' doesn't have the .torrent extension" if uriPath !~ /.torrent$/
+      path = @torrentFileDir + File::SEPARATOR + File.basename(uriPath)
+      File.open(path, "w"){ |outfile| outfile.write(f.read) }
+    end
+    path
+  end
+
+  # Store a magnet link in a file in the torrent file directory.
+  def storeMagnet(magnet)
+    asciiInfoHash = QuartzTorrent::bytesToHex(magnet.btInfoHash)
+    path = @torrentFileDir + File::SEPARATOR + asciiInfoHash + ".magnet"
+    File.open(path, "w"){ |outfile| outfile.write(magnet.raw) }
+  end
+
+  # Load a magnet link in a file 
+  def loadMagnet(path)
+    raw = nil
+    File.open(path, "r"){ |infile| raw = infile.read }
+    QuartzTorrent::MagnetURI.new(raw)
+  end
+
+  # Store an uploaded .torrent file in the torrent directory. Return the path to the 
+  # final .torrent file.
+  def storeUploadedTorrentFile(path, name)
+    name += ".torrent" if name !~ /\.torrent$/
+    dpath = @torrentFileDir + File::SEPARATOR + name
+    FileUtils.mv path, dpath
+    dpath
+  end
+
+  # Start running the torrent specified by the .torrent file given in path.
+  def startTorrentFile(path)
+    startTorrent do
+      begin
+        metainfo = QuartzTorrent::Metainfo.createFromFile(path)
+        @peerClient.addTorrentByMetainfo(metainfo)
+      rescue BEncode::DecodeError
+        # Delete the file
+        begin
+          FileUtils.rm path
+        rescue
+        end
+        raise $!
+      end
+    end
+  end
+
+  # Start running the magnet
+  def startMagnet(magnet)
+    startTorrent do
+      @peerClient.addTorrentByMagnetURI(magnet)
+    end
+  end
+
+  # Remove the specified torrent. Pass the infoHash as an ascii string, not binary.
+  def removeTorrent(infoHash, deleteFiles)
+    infoHashBytes = QuartzTorrent::hexToBytes(infoHash)
+    @peerClient.removeTorrent infoHashBytes, deleteFiles
+
+    # Remove torrent from torrent dir
+    Dir.new(@torrentFileDir).each do |e|
+      if e =~ /\.torrent$/
+        path = @torrentFileDir + File::SEPARATOR + e
+        metainfo = QuartzTorrent::Metainfo.createFromFile(path)
+        if metainfo.infoHash == infoHashBytes
+          FileUtils.rm path
+          break
+        end
+      end
+    end
+
+    # Remove torrent settings
+    helper = SettingsHelper.new
+    helper.deleteForOwner infoHash
+   
+    # Remove magnet file if it exists
+    magnetFile = @torrentFileDir + File::SEPARATOR + infoHash + ".magnet"
+    FileUtils.rm magnetFile if File.exists?(magnetFile)
+  end
+
+  # Update the torrent settings (upload rate limit, etc) from database values
+  def applyTorrentSettings(infoHash)
+    asciiInfoHash = QuartzTorrent::bytesToHex(infoHash)
+    helper = SettingsHelper.new
+
+    # Set limits based on per-torrent settings if they exist, otherwise to default global limits if they exist.
+    uploadRateLimit = to_i(helper.get(:uploadRateLimit, :unfiltered, asciiInfoHash))
+    uploadRateLimit = to_i(helper.get(:defaultUploadRateLimit, :unfiltered)) if ! uploadRateLimit
+
+    downloadRateLimit = to_i(helper.get(:downloadRateLimit, :unfiltered, asciiInfoHash))
+    downloadRateLimit = to_i(helper.get(:defaultDownloadRateLimit, :unfiltered)) if ! downloadRateLimit
+
+    ratio = helper.get(:ratio, :filter, asciiInfoHash)
+    ratio = helper.get(:defaultRatio, :filter) if ! ratio
+
+    @peerClient.setUploadRateLimit infoHash, uploadRateLimit
+    @peerClient.setDownloadRateLimit infoHash, downloadRateLimit
+    @peerClient.setUploadRatio infoHash, ratio
+  end
+
+  # Get the usage for the current period of the specified type.
+  # periodType should be one of :daily or :monthly.
+  def currentPeriodUsage(periodType)
+    @usageTracker.currentUsage(periodType).value
+  end
+
+  private
+  # Helper for starting torrents. Expects a block that when called will add a torrent to the 
+  # @peerClient, and return the infoHash.
+  def startTorrent
+    raise "Torrent client is shutting down" if @peerClientStopped
+    infoHash = yield 
+
+    applyTorrentSettings infoHash
+  end
+
+  def to_i(val)
+    val = val.to_i if val
+    val
+  end
+
+  # Start a thread to keep track of usage.
+  def startUsageTrackerThread
+    @usageTrackerThread = Thread.new do
+      QuartzTorrent.initThread("torrent_usage_tracking")
+      
+      Thread.current[:stopped] = false
+
+      while ! Thread.current[:stopped]
+        begin
+          sleep 4
+          torrentData = @peerClient.torrentData
+          usage = 0
+          torrentData.each do |k,v|
+            usage += v.bytesUploaded + v.bytesDownloaded
+          end
+          @usageTracker.update(usage)
+        rescue
+          puts "Error in usage tracking thread: #{$!}"
+          puts $!.backtrace.join "\n"
+        end
+      end
+    end
+  end
+end

data/lib/quartz_flow/usagetracker.rb

@@ -0,0 +1,335 @@
+require 'quartz_flow/model'
+
+class Bucket
+  def initialize(label = nil, criteriaData = nil, value = nil)
+    @label = label
+    @criteriaData = criteriaData
+    @value = value
+    @absoluteUsageAtStartOfBucket = nil
+  end
+  attr_accessor :label
+  # Data used by BucketChangeCriteria to determine when we need a new bucket. 
+  attr_accessor :criteriaData
+  # At the time this bucket was created, the value of the absolute usage for all time
+  attr_accessor :absoluteUsageAtStartOfBucket
+  # The amount of usage for this bucket alone
+  attr_accessor :value
+
+  def toHash
+    {"label" => @label, "absoluteUsageAtStartOfBucket" => @absoluteUsageAtStartOfBucket, "criteriaData" => @criteriaData, "value" => @value}
+  end
+
+  def fromHash(hash)
+    @label = hash["label"]
+    @absoluteUsageAtStartOfBucket = hash["absoluteUsageAtStartOfBucket"]
+    @criteriaData = hash["criteriaData"]
+    @value = hash["value"]
+  end
+
+  def fromModel(bucket)
+    @label = bucket.label
+    @absoluteUsageAtStartOfBucket = bucket.absoluteUsage
+    @criteriaData = bucket.criteriaData
+    @value = bucket.value
+  end
+
+  def toModel
+    model = UsageBucket.new
+    model.attributes = { 
+      :label => @label,
+      :absoluteUsage => @absoluteUsageAtStartOfBucket,
+      :criteriaData => @criteriaData,
+      :value => @value
+    }
+    model
+  end
+end
+
+class BucketChangeCriteria
+  # Is it now time for a new bucket?
+  def newBucket?(currentBucket)
+    false
+  end
+
+  # Make a new bucket and return it.
+  def newBucket
+    innerNewBucket 
+  end
+
+  def innerNewBucket
+    raise "Implement me!"
+  end
+end
+
+class PeriodicBuckets
+  def initialize(criteria, maxBuckets = nil)
+    setBucketChangeCriteria(criteria)
+    @buckets = []
+    @maxBuckets = maxBuckets
+    @maxBuckets = 1 if @maxBuckets && @maxBuckets < 1
+  end
+
+  # Set the criteria that determines when the current bucket is full
+  # and we should make a new empty bucket the current bucket, and that is used
+  # to set the label for the new bucket.
+  def setBucketChangeCriteria(criteria)
+    @bucketChangeCriteria = criteria
+  end
+
+  def update(absoluteUsage = nil)
+    if @buckets.size == 0
+      prev = nil
+      @buckets.push @bucketChangeCriteria.newBucket
+      setAbsoluteUsage(prev, @buckets.last, absoluteUsage) if absoluteUsage
+    else
+      prev = @buckets.last
+      # Time for a new bucket?
+      if @bucketChangeCriteria.newBucket?(@buckets.last)
+        @buckets.push @bucketChangeCriteria.newBucket
+        setAbsoluteUsage(prev, @buckets.last, absoluteUsage) if absoluteUsage
+      end
+      @buckets.shift if @maxBuckets && @buckets.size > @maxBuckets
+    end
+
+    setValue(@buckets.last, absoluteUsage) if absoluteUsage
+  end
+
+  def current(absoluteUsage = nil)
+    @buckets.last
+  end
+  
+  def all
+    @buckets
+  end
+
+  def toHash
+    array = []
+    @buckets.each do |b|
+      array.push b.toHash 
+    end
+    { "buckets" => array }
+  end
+
+  def fromHash(hash)
+    @buckets = []
+    hash["buckets"].each do |b|
+      bucket = Bucket.new(nil, nil, nil)
+      bucket.fromHash b
+      @buckets.push bucket
+    end
+  end
+
+  def fromModel(type)
+    @buckets = []
+    buckets = UsageBucket.all(:type => type, :order => [:index.asc])
+    buckets.each do |model|
+      bucket = Bucket.new(nil, nil, nil)
+      bucket.fromModel model
+      @buckets.push bucket
+    end
+  end
+  
+  def toModel(type)
+    UsageBucket.all(:type => type).destroy!
+    index = 0
+    @buckets.each do |b|
+      model = b.toModel
+      model.type = type
+      model.index = index
+      index += 1
+      model.save
+    end
+  end
+
+  private
+  def setAbsoluteUsage(previousBucket, newBucket, absoluteUsage)
+    if previousBucket
+      newBucket.absoluteUsageAtStartOfBucket = previousBucket.absoluteUsageAtStartOfBucket + previousBucket.value
+    else
+      newBucket.absoluteUsageAtStartOfBucket = absoluteUsage
+    end
+  end
+  def setValue(newBucket, absoluteUsage)
+    newBucket.value = 
+      absoluteUsage - 
+      newBucket.absoluteUsageAtStartOfBucket
+  end
+end
+
+class DailyBucketChangeCriteria < BucketChangeCriteria
+  def newBucket?(currentBucket)
+    now = Time.new
+    currentBucket.criteriaData.day != now.day
+  end
+
+  def newBucket
+    now = Time.new
+    Bucket.new(now.strftime("%b %e"), now, 0)
+  end
+
+  def criteriaData
+    Time.new
+  end
+end
+
+class MonthlyBucketChangeCriteria < BucketChangeCriteria
+  def initialize(resetDay)
+    @resetDay = resetDay
+  end
+  
+  def newBucket?(currentBucket)
+    Time.new > currentBucket.criteriaData
+  end
+
+  def newBucket
+    now = Time.new
+    # Set the bucket's criteriaData to the date after which we need a new bucket.
+    data = criteriaData
+    Bucket.new(now.strftime("%b %Y"), data, 0)
+  end
+
+  def criteriaData
+    now = Time.new
+    nextMonth = now.mon % 12 + 1
+    Time.local(now.year, nextMonth, @resetDay)
+  end
+end
+
+
+# For testing
+class MinuteBucketChangeCriteria < BucketChangeCriteria
+  def newBucket?(currentBucket)
+    now = Time.new
+    currentBucket.criteriaData.min != now.min
+  end
+
+  def newBucket
+    now = Time.new
+    Bucket.new(now.strftime("%H:%M"), now, 0)
+  end
+end
+
+class UsageTracker
+  def initialize(monthlyResetDay)
+    @buckets = {}
+    @buckets[:daily] = PeriodicBuckets.new(DailyBucketChangeCriteria.new,31)
+    #@buckets[:minute] = PeriodicBuckets.new(MinuteBucketChangeCriteria.new,3)
+    @buckets[:monthly] = PeriodicBuckets.new(MonthlyBucketChangeCriteria.new(monthlyResetDay),2)
+    @usageForAllTimeAdjustment = 0
+    loadBucketsFromDatastore
+  end
+
+  # Update the UsageTracker with more usage. The value passed
+  # should be the usage since the torrentflow session was created.
+  # If a datastore is not used, then this means stopping and starting the session
+  # will cause UsageTracking to only track usage for the session. However
+  # if Mongo is used then the usage can be saved and persisted between sessions
+  # and internally the value passed here is added to the value loaded from Mongo.
+  def update(usageForAllTime)
+    usageForAllTime += @usageForAllTimeAdjustment
+    @buckets.each do |k,buckets|
+      buckets.update usageForAllTime
+    end
+    saveBucketsToDatastore
+  end
+
+  # This method returns the usage in the current bucket for the specified
+  # period type (:daily or :monthly). The usage is accurate as of the last 
+  # time update() was called.
+  # The returned value is a single Bucket object.
+  def currentUsage(periodType)
+    getBuckets(periodType).current
+  end
+
+  # Returns the usage as of the last time update() was called.
+  # This method returns all the tracked usage for the specified
+  # period type (:daily or :monthly). The usage is accurate as of the last 
+  # time update() was called.
+  # The returned value is an array of Bucket objects.
+  def allUsage(periodType)
+    getBuckets(periodType).all
+  end
+
+  private
+  def getBuckets(type)
+    buckets = @buckets[type]
+    raise "Unsupported periodType #{periodType.to_s}" if ! buckets
+    buckets
+  end
+
+  def saveBucketsToDatastore
+    @buckets[:daily].toModel(:daily)
+    @buckets[:monthly].toModel(:monthly)
+=begin
+    if @mongoDb
+      dailyCollection = @mongoDb.collection("daily_usage")
+      monthlyCollection = @mongoDb.collection("monthly_usage")
+      # Remove all previous documents
+      dailyCollection.remove
+      monthlyCollection.remove
+
+      dailyCollection.insert @buckets[:daily].toHash
+      monthlyCollection.insert @buckets[:monthly].toHash
+    end
+=end
+  end
+
+  def loadBucketsFromDatastore
+    @buckets[:daily].fromModel(:daily)
+    @buckets[:monthly].fromModel(:monthly)
+
+    # If we are loading from datastore it means that the absolute usage returned from the current torrent session will not
+    # contain the usage that we previously tracked, so we must add the old tracked value to what the torrent
+    # session reports.
+    if @buckets[:daily].current
+      @usageForAllTimeAdjustment = @buckets[:daily].current.absoluteUsageAtStartOfBucket + @buckets[:daily].current.value
+    end
+  
+=begin
+    if @mongoDb
+      $logger.info "Loading usage from Mongo."
+      dailyCollection = @mongoDb.collection("daily_usage")
+      monthlyCollection = @mongoDb.collection("monthly_usage")
+
+      arr = dailyCollection.find_one
+      @buckets[:daily].fromHash arr if arr
+      arr = monthlyCollection.find_one
+      @buckets[:monthly].fromHash arr if arr
+
+      # If we are loading from Mongo it means that the absolute usage returned from the torrentflow session will not
+      # contain the usage that we previously tracked, so we must add the old tracked value to what the torrentflow
+      # session reports.
+      if @buckets[:daily].current
+        @usageForAllTimeAdjustment = @buckets[:daily].current.absoluteUsageAtStartOfBucket + @buckets[:daily].current.value
+        $logger.info "Absolute usage at start of current daily bucket: " + @buckets[:daily].current.absoluteUsageAtStartOfBucket.to_s
+        $logger.info "Usage in current daily bucket: " + @buckets[:daily].current.value.to_s
+        $logger.info "Usage for all time adjustment: " + @usageForAllTimeAdjustment.to_s
+      else
+        $logger.info "No usage loaded in Mongo (empty collection)."
+      end
+    else
+      $logger.info "Not loading usage from Mongo."
+    end
+=end
+  end
+end
+
+=begin
+  # Testing
+  tracker = UsageTracker.new
+  
+  abs = 200
+  while true
+    tracker.update(abs)
+    
+    puts
+    puts "Usage for all time: #{abs}"
+    puts "Buckets:"
+    tracker.allUsage(:minute).each do |b|
+      puts "  #{b.label}: #{b.value}"   
+    end
+    abs += 10
+    sleep 10
+  end  
+
+=end

data/lib/quartz_flow/wrappers.rb

@@ -0,0 +1,124 @@
+require 'quartz_torrent/formatter.rb'
+
+module QuartzTorrent
+
+  class Metainfo::FileInfo
+    def to_h
+      { path: @path, length: @length }
+    end
+  end
+
+  class Metainfo::Info
+    def to_h
+      result = {}
+
+      result[:name] = @name
+      result[:pieceLen] = @pieceLen
+      result[:files] = @files.collect{ |e| e.to_h }
+
+      result
+    end
+  end
+
+  class TrackerPeer
+    def to_h
+      { ip: @ip, port: @port }
+    end
+  end
+
+  class Peer
+    def to_h
+      result = {}
+      
+      result[:trackerPeer] = @trackerPeer.to_h
+      result[:amChoked] = @amChoked
+      result[:amInterested] = @amInterested
+      result[:peerChoked] = @peerChoked
+      result[:peerInterested] = @peerInterested
+      result[:firstEstablishTime] = @firstEstablishTime
+      result[:maxRequestedBlocks] = @maxRequestedBlocks
+      result[:state] = @state
+      result[:isUs] = @isUs
+      result[:uploadRate] = @uploadRate
+      result[:downloadRate] = @downloadRate
+      if @bitfield
+        result[:pctComplete] = "%.2f" % (100.0 * @bitfield.countSet / @bitfield.length)
+      else
+        result[:pctComplete] = 0
+      end
+
+      result
+    end
+  end
+
+  class TorrentDataDelegate
+    # Convert to a hash. Also flattens some of the data into new fields.
+    def to_h
+      result = {}
+
+      ## Extra fields added by this method:
+      # Length of the torrent
+      result[:dataLength] = @info ? @info.dataLength : 0
+      # Percent complete
+      pct = withCurrentAndTotalBytes{ |cur, total| (cur.to_f / total.to_f * 100.0).round 1 }
+      result[:percentComplete] = pct
+      # Time left
+      secondsLeft = withCurrentAndTotalBytes do |cur, total|
+        if @downloadRateDataOnly && @downloadRateDataOnly > 0
+          (total.to_f - cur.to_f) / @downloadRateDataOnly 
+        else
+          0
+        end
+      end
+      result[:timeLeft] = Formatter.formatTime(secondsLeft)
+
+      ## Regular fields
+      result[:info] = @info ? @info.to_h : nil
+      result[:infoHash] = @infoHash
+      result[:recommendedName] = @recommendedName
+      result[:downloadRate] = @downloadRate
+      result[:uploadRate] = @uploadRate
+      result[:downloadRateDataOnly] = @downloadRateDataOnly
+      result[:uploadRateDataOnly] = @uploadRateDataOnly
+      result[:completedBytes] = @completedBytes
+      result[:peers] = @peers.collect{ |p| p.to_h }
+      result[:state] = @state
+      #result[:completePieceBitfield] = @completePieceBitfield
+      result[:metainfoLength] = @metainfoLength
+      result[:metainfoCompletedLength] = @metainfoCompletedLength
+      result[:paused] = @paused
+      result[:uploadRateLimit] = @uploadRateLimit
+      result[:downloadRateLimit] = @downloadRateLimit
+      result[:ratio] = @ratio
+      result[:bytesUploaded] = @bytesUploaded
+      result[:bytesDownloaded] = @bytesDownloaded
+
+      result
+    end
+
+    private
+    def withCurrentAndTotalBytes
+      if @info
+        yield @completedBytes, @info.dataLength
+      elsif @state == :downloading_metainfo && @metainfoCompletedLength && @metainfoLength
+        yield @metainfoCompletedLength, @metainfoLength
+      else
+        0
+      end
+    end
+    
+
+    def calcEstimatedTime(torrent)
+      # Time left = amount_left / download_rate
+      #           = total_size * (1-progress) / download_rate
+      if torrentHandle.has_metadata && torrentHandle.status.download_rate.to_f > 0
+      secondsLeft = torrentHandle.info.total_size.to_f * (1 - torrentHandle.status.progress.to_f) / torrentHandle.status.download_rate.to_f
+      Formatter.formatTime(secondsLeft)
+    else
+      "unknown"
+    end
+  end 
+
+  end
+end
+