sinatra-torrent 0.0.3 → 0.0.4

data/Rakefile

@@ -1,4 +1,3 @@
-require 'rubygems'
 require 'rake'
 
 begin

data/Readme.md

@@ -15,13 +15,9 @@ Usage
 
 All files you put in the `downloads` directory at the root of your sinatra app will be downloadable at `/downloads/your_file.ext` and it's torrent will be dynamically generated (and cached) at `/torrents/your_file.ext.torrent`. You will have trouble with larger files as it currently hashes as part of the request first time round. I'm planning on pushing this out to workers at some point. Not yet sure how I'm going to do that…
 
-**NB.** Files that take longer than 1s to hash will fail at the moment!
-
-The extension is in it's early stages at the moment, so many of the settings aren't adhered to, and there are some issues with the webseeding… however it *does* work.
-
 ### I want options!
 
-There needs to be a database of torrents and peers, this is taken care of by a database adapter. Currently I've written (a breally basic) one for active record, so many databases are supported. I'm still finding my way around the Sinatra extensions api, so this is how you specify your own ActiveRecord settings:
+There needs to be a database of torrents and peers, this is taken care of by a database adapter. Currently I've written (a really basic) one for active record, so many databases are supported as is, but you can write your own for others (eg. mongo). I'm still finding my way around the Sinatra extensions api, so this is how you specify your own ActiveRecord settings:
 
     require 'sinatra'
     require 'sinatra/torrent/activerecord'
@@ -31,6 +27,36 @@ There needs to be a database of torrents and peers, this is taken care of by a d
     }
     require 'sinatra/torrent'
 
+Rake
+----
+
+If a torrent takes longer than 1 second to generate on-the-fly, it'll be added to a queue for processing in a background task. If you require a special script in your Rakefile you'll be able to process the queue, add to it or pre-hash all the files in your download directory:
+
+	require 'rake'
+	
+	# These need to be the same adaptor and settings as your app, of course!
+	require 'sinatra/torrent/activerecord'
+	Sinatra::Torrent::Database.settings = {
+	  'adapter' => 'sqlite3',
+	  'database' => 'torrents.db'
+	}
+	require 'sinatra/torrent/hashing'
+	
+	# This line is optional, 'downloads' is the default
+	# If you've used `set :download_directory, 'files'` in your sinatra app, you need to do:
+	Sinatra::Torrent::DOWNLOAD_DIRECTORY = 'files'
+	
+	# The rest of your Rakefile
+
+* `rake hash:add filename.ext` will add `filename.ext` inside your download directory to the hash queue
+* `rake hash:queue` will process all the hash jobs in the queue (you may want to set up a cron job to run this)
+* `rake hash:all` will hash all files in the downloads directory right now, so none will be processed on-the-fly when the .torrent file is downloaded
+
+To Do
+-----
+
+* Execute a user-writen block of code when an on-the-fly torrent creation times out. This will allow people to trigger the background rake task immediately, rather than waiting on cron.
+
 Ummmm
 -----
 

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.0.4

data/lib/sinatra/torrent/activerecord.rb

@@ -32,7 +32,11 @@ module Sinatra
             end
         
             add_index :torrents, :infohash, :unique => true
+          end
+        end
         
+        unless Peer.table_exists?
+          ActiveRecord::Schema.define do
             create_table :peers do |table|
               table.string :torrent_infohash, :null => false
               table.string :peer_id
@@ -45,7 +49,19 @@ module Sinatra
               table.timestamps
             end
         
-            add_index :peers, [:peer_id,:torrent_infohash],:unique => true
+            add_index :peers, [:peer_id,:torrent_infohash], :unique => true
+          end
+        end
+        
+        unless DelayedJob.table_exists?
+          ActiveRecord::Schema.define do
+            create_table :delayed_jobs do |table|
+              table.string :filename, :null => false
+              
+              table.timestamp :created_at
+            end
+        
+            add_index :delayed_jobs, :filename, :unique => true
           end
         end
       end
@@ -129,15 +145,34 @@ module Sinatra
     
         peer.save
       end
+      
+      
+      # Registers a file to be hashed offline
+      def add_hashjob(filename)
+        DelayedJob.find_or_create_by_filename(filename)
+        # return eta in seconds, where possible
+        nil
+      end
+    
+      # Lists outstanding files to be hashed, by request date hopefully!
+      def list_hashjobs
+        DelayedJob.find(:all,:order => 'created_at DESC').collect{|dj| dj.filename}
+      end
+    
+      # Removes a file from the 'to be hashed' list
+      def remove_hashjob(filename)
+        DelayedJob.find_by_filename(filename).delete rescue false
+      end
   
       private
+      
       class Torrent < ActiveRecord::Base
         serialize :metadata, Hash
       end
   
-      class Peer < ActiveRecord::Base
-    
-      end
+      class Peer < ActiveRecord::Base; end
+      
+      class DelayedJob < ActiveRecord::Base; end
     end
   end
 end

data/lib/sinatra/torrent/hashing.rb

@@ -0,0 +1,83 @@
+require 'sinatra/torrent/helpers'
+
+namespace :hash do
+  begin
+    torrent_db = Sinatra::Torrent::Database.new
+  rescue NameError
+    raise RuntimeError, "Please require a sinatra-torrent database adapter in your rakefile."
+  end
+
+  task :add do
+    ARGV[1..-1].each do |rel_location|
+      if (File.exists?(File.join(Sinatra::Torrent::DOWNLOADS_DIRECTORY,rel_location)))
+        torrent_db.add_hashjob(rel_location)
+        $stdout.puts "added to queue: #{rel_location}"
+      else
+        $stderr.puts "#{rel_location} doesn't exist in the downloads directory"
+      end
+    end
+  end
+
+  task :all do
+    completed = 0
+    failed = 0
+    
+    Dir[File.join(Sinatra::Torrent::DOWNLOADS_DIRECTORY,'**')].each do |filename|
+      rel_location = filename[Sinatra::Torrent::DOWNLOADS_DIRECTORY.length+1..-1]
+      
+      begin
+        if !torrent_db.torrent_by_path_and_timestamp(rel_location,File.mtime(filename))
+          d = Sinatra::Torrent.create(filename)
+        
+          torrent_db.store_torrent(rel_location,File.mtime(filename),d['metadata'],d['infohash'])
+        
+          torrent_db.remove_hashjob(rel_location)
+          $stdout.puts "Hashed: #{rel_location}"
+          
+          completed += 1
+        else
+          $stdout.puts "Already hashed: #{rel_location}"
+        end
+      rescue
+        $stderr.puts "Not hashed: #{rel_location} (Unknown error)"
+        failed += 1
+      end
+    end
+    
+    $stdout.puts "#{completed} hash job#{(completed == 1) ? '' : 's'} completed sucessfully"
+    $stderr.puts "#{failed} failed!" unless failed == 0
+  end
+  
+  task :queue do
+    completed = 0
+    failed = 0
+    
+    torrent_db.list_hashjobs.each do |rel_location|
+      filename = File.join(Sinatra::Torrent::DOWNLOADS_DIRECTORY,rel_location)
+      begin
+        if torrent_db.torrent_by_path_and_timestamp(rel_location,File.mtime(filename))
+          torrent_db.remove_hashjob(rel_location)
+        else
+          d = Sinatra::Torrent.create(filename)
+        
+          torrent_db.store_torrent(rel_location,File.mtime(filename),d['metadata'],d['infohash'])
+        
+          completed += 1
+          torrent_db.remove_hashjob(rel_location)
+        end
+      rescue Errno::ENOENT
+        $stderr.puts "#{rel_location} no longer exists to be hashed. Removing job."
+        torrent_db.remove_hashjob(rel_location)
+        
+        failed += 1
+      rescue
+        $stderr.puts "Uncaught failure reason for #{rel_location}"
+        # TODO: trace?
+        failed += 1
+      end
+    end
+    
+    $stdout.puts "#{completed} queued hash job#{(completed == 1) ? '' : 's'} completed sucessfully"
+    $stderr.puts "#{failed} failed!" unless failed == 0
+  end
+end

data/lib/sinatra/torrent/helpers.rb

@@ -0,0 +1,38 @@
+require 'time'
+require 'digest/sha1'
+require 'bencode'
+
+module Sinatra
+  module Torrent
+    DOWNLOADS_DIRECTORY = 'downloads'
+    
+    def self.create(filename)
+      d = {
+        'metadata' => {
+          'created by' => "sinatra-torrent (#{File.read(File.expand_path(File.join(__FILE__,'..','..','..','..','VERSION'))).strip}) (http://github.com/jphastings/sinatra-torrent)",
+          'creation date' => Time.now.to_i,
+          'info' => {
+            'name' => File.basename(filename),
+            'length' => File.size(filename),
+            'piece length' => 2**10, # TODO: Choose reasonable piece size
+            'pieces' => ''
+          }
+        }
+      }
+
+      begin
+        file = open(filename,'r')
+
+        begin
+          d['metadata']['info']['pieces'] += Digest::SHA1.digest(file.read(d['metadata']['info']['piece length']))
+        end until file.eof?
+      ensure
+        file.close
+      end
+
+      d['infohash'] = Digest::SHA1.hexdigest(d['metadata']['info'].bencode)
+
+      return d
+    end
+  end
+end

data/lib/sinatra/torrent.rb

@@ -1,8 +1,6 @@
 require 'timeout'
-require 'time'
-require 'digest/sha1'
-require 'bencode'
 require 'sinatra/base'
+require 'sinatra/torrent/helpers'
 
 # This extension will serve up the contents of the specified folder as web seeded torrents.
 # Both webseed versions are supported (shad0w's and GetRight's) and there is an inbuilt tracker
@@ -16,7 +14,7 @@ module Sinatra
       # Putting the annouce URL of a tracker in here will use that tracker rather than the inbuilt one
       app.set :external_tracker, nil
       # Directory which holds all the files which will be provided as torrents
-      app.set :downloads_directory, File.dirname(__FILE__)+'downloads'
+      app.set :downloads_directory, File.join(File.dirname(__FILE__),Sinatra::Torrent::DOWNLOADS_DIRECTORY)
       # Mount point for the downloads directory
       app.set :downloads_mount, 'downloads'
       # Mount point for the torrents directory
@@ -41,38 +39,31 @@ module Sinatra
         filename = File.join(options.downloads_directory, rel_location)
         halt(404, "That file doesn't exist! #{filename}") unless File.exists?(filename)
         
-        if true #!(d = options.database_adapter.torrent_by_path_and_timestamp(filename,File.mtime(filename)))
-          
-          d = {
-            'metadata' => {
-              # TODO: Version?
-              'created by' => 'sinatra-torrent (0.0.1) (http://github.com/jphastings/sinatra-torrent)',
-              'creation date' => Time.now.to_i,
-              'info' => {
-                'name' => File.basename(rel_location),
-                'length' => File.size(filename),
-                'piece length' => 2**10, # TODO: Choose reasonable piece size
-                'pieces' => ''
-              }
-            }
-          }
+        if !(d = options.database_adapter.torrent_by_path_and_timestamp(filename,File.mtime(filename)))
           
           begin
-            file = open(filename,'r')
-            
             Timeout::timeout(1) do
-              begin
-                d['metadata']['info']['pieces'] += Digest::SHA1.digest(file.read(d['metadata']['info']['piece length']))
-              end until file.eof?
+              d = Sinatra::Torrent.create(filename)
             end
           rescue Timeout::Error
-            # TODO: Actually run it in the background!
-            halt(503,"This torrent is taking too long to build, we're [not currently supporting!] running it in the background. Please try again in a few minutes.")
-          ensure
-            file.close
+            eta = options.database_adapter.add_hashjob(filename)
+            
+            begin
+              wait = case (eta/60).floor
+              when 0
+                'under a minute'
+              when 1
+                'about a minute'
+              else
+                "about #{(eta/60).floor} minutes"
+              end
+            rescue NoMethodError
+              wait = "a short while"
+            end
+            
+            halt(503,"This torrent is taking too long to build, we're running it in the background. Please try again in #{wait}.")
           end
           
-          d['infohash'] = Digest::SHA1.hexdigest(d['metadata']['info'].bencode)
           options.database_adapter.store_torrent(filename,File.mtime(filename),d['metadata'],d['infohash'])
         end
         
@@ -128,8 +119,15 @@ module Sinatra
       end
 
 # INDEX PAGE
+
+      # TODO: Have a 'fallback' index view?
       app.get "/#{app.options.torrents_mount}/" do
-        haml :torrents_index,:locals => {:torrents => Dir.glob("#{options.downloads_directory}/**").collect {|f| f[options.downloads_directory.length+1..-1] } }
+        locals = {:torrents => (Dir.glob("#{options.downloads_directory}/**").collect {|f| f[options.downloads_directory.length+1..-1] } rescue [])}
+        begin
+          haml :torrents_index,:locals => locals
+        rescue Errno::ENOENT
+          "<ul>"<<locals[:torrents].collect{|t| "<li><a href=\"/#{options.torrents_mount}/#{t}.torrent\">#{t}</a></li>" }.join<<"</ul>"
+        end
       end
 
 # DATA
@@ -158,4 +156,4 @@ module Sinatra
   end
   
   register Torrent
-end
+end

data/sinatra-torrent.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{sinatra-torrent}
-  s.version = "0.0.3"
+  s.version = "0.0.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["JP Hastings-Spital"]
-  s.date = %q{2011-02-08}
+  s.date = %q{2011-02-09}
   s.description = %q{An extension to Sinatra which will allow you to run a webseeded torrent tracker of files in the folder you specify.}
   s.email = %q{jphastings@gmail.com}
   s.files = [
@@ -18,8 +18,9 @@ Gem::Specification.new do |s|
      "VERSION",
      "lib/sinatra/torrent.rb",
      "lib/sinatra/torrent/activerecord.rb",
-     "sinatra-torrent.gemspec",
-     "views/torrents_index.haml"
+     "lib/sinatra/torrent/hashing.rb",
+     "lib/sinatra/torrent/helpers.rb",
+     "sinatra-torrent.gemspec"
   ]
   s.homepage = %q{http://github.com/jphastings/sinatra-torrent}
   s.rdoc_options = ["--charset=UTF-8"]

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: sinatra-torrent
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 23
   prerelease: false
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - JP Hastings-Spital
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-08 00:00:00 +00:00
+date: 2011-02-09 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -48,8 +48,9 @@ files:
 - VERSION
 - lib/sinatra/torrent.rb
 - lib/sinatra/torrent/activerecord.rb
+- lib/sinatra/torrent/hashing.rb
+- lib/sinatra/torrent/helpers.rb
 - sinatra-torrent.gemspec
-- views/torrents_index.haml
 has_rdoc: true
 homepage: http://github.com/jphastings/sinatra-torrent
 licenses: []

data/views/torrents_index.haml

@@ -1,5 +0,0 @@
-%ul
-	-locals[:torrents].each do |torrent|
-		%li
-			%a{:href=>"/#{options.torrents_mount}/#{torrent}.torrent"}=torrent
-