murder 0.0.0.pre

data/doc/examples/Capfile

@@ -0,0 +1,28 @@
+# Capfile with example murder config
+
+require 'rubygems'
+require 'murder'
+
+set :application, 'example-app'
+
+set :scm, 'git'
+set :repository, 'ssh://git.example.com/my/app.git'
+set :copy_exclude, '.git'
+
+# use murder
+set :deploy_via, :murder
+
+# convenient hooks
+after 'deploy:setup', 'murder:distribute_files' # setup remote murder support files on all hosts
+before 'murder:start_seeding', 'murder:start_tracker' # let deploy manage the tracker as well
+after 'murder:stop_seeding', 'murder:stop_tracker'
+
+# define an app server role
+role :app, 'w01', 'w02', 'w03', 'w04', 'w05'
+
+# murder roles (optional. seeder/tracker will default to the first deployable server) 
+role :tracker, 'tracker01'
+role :seeder, 'seeder01'
+# peer role defaults to all deployable servers (all except :no_release => true)
+# you shouldn't need to set this if you are using the deploy strategy.
+#role :peer, 'w01', 'w02', 'w03', 'w04', 'w05'

data/lib/capistrano/recipes/deploy/strategy/murder.rb

@@ -0,0 +1,52 @@
+# Copyright 2010 Twitter, Inc.
+# Copyright 2010 Larry Gadea <lg@twitter.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'capistrano/recipes/deploy/strategy/copy'
+
+module Capistrano
+  module Deploy
+    module Strategy
+      class Murder < Capistrano::Deploy::Strategy::Copy
+        def upload(filename, remote_filename)
+          puts "Uploading release to seeder..."
+          configuration.upload(filename, remote_filename, :roles => :seeder)
+
+          ENV["tag"] = File.basename(filename)
+          ENV["path_is_file"] = "yes"
+          ENV["files_path"] = remote_filename
+          ENV['destination_path'] = remote_filename
+
+          puts "Creating torrent..."
+          murder.create_torrent
+
+          puts "Starting seeding..."
+          murder.start_seeding
+
+          puts "Peering..."
+          murder.peer
+
+          puts "Done. Killing seeding and all peering..."
+          murder.stop_peering
+          murder.stop_seeding
+
+          puts "Cleaning temp files..."
+          murder.clean_temp_files
+
+          puts "THANK YOU FOR USING MURDER, HAVE A NICE DAY"
+        end
+      end
+    end
+  end
+end

data/lib/murder.rb

@@ -0,0 +1,43 @@
+# Copyright 2010 Twitter, Inc.
+# Copyright 2010 Larry Gadea <lg@twitter.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Capistrano::Configuration.instance(:must_exist).load do
+
+  load_paths.push File.expand_path('../', __FILE__)
+
+  load 'murder/murder'
+  load 'murder/admin'
+
+  # no default defaults...
+  set :default_tag, ''
+  set :default_seeder_files_path, ''
+  set :default_destination_path, ''
+
+  # default remote dist path in app shared directory
+  set(:remote_murder_path) { "#{shared_path}/murder" }
+
+  # roles
+  excluded_roles = [:peer, :tracker, :seeder]
+
+  # get around the fact that find_servers does not work in role evaluation
+  # (it tries to evaluate all roles, leading to infinite recursion)
+  role :peer do
+    roles.reject{|k,v| excluded_roles.include? k }.values.map(&:servers).flatten.uniq.reject{|s| s.options[:no_release] }
+  end
+
+  role(:tracker) { roles[:peer].servers.first }
+  role(:seeder) { roles[:peer].servers.first }
+
+end

data/lib/murder/admin.rb

@@ -0,0 +1,47 @@
+# Copyright 2010 Twitter, Inc.
+# Copyright 2010 Larry Gadea <lg@twitter.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+namespace :murder do
+  task :distribute_files, :roles => [:tracker, :seeder, :peer] do
+    dist_path = File.expand_path('../../dist', __FILE__)
+
+    run "mkdir -p #{remote_murder_path}/"
+    run "[ $(find '#{remote_murder_path}/'* | wc -l ) -lt 1000 ] && rm -rf '#{remote_murder_path}/'* || ( echo 'Cowardly refusing to remove files! Check the remote_murder_path.' ; exit 1 )"
+
+    # TODO: Skip hidden (.*) files
+    # TODO: Specifyable tmp file
+    system "tar -c -z -C #{dist_path} -f /tmp/murder_dist.tgz ."
+    upload("/tmp/murder_dist.tgz", "/tmp/murder_dist.tgz", :via => :sftp)
+    run "tar xf /tmp/murder_dist.tgz -C #{remote_murder_path}"
+    run "rm /tmp/murder_dist.tgz"
+    system "rm /tmp/murder_dist.tgz"
+  end
+
+  task :start_tracker, :roles => :tracker do
+    run("screen -dms murder_tracker python #{remote_murder_path}/murder_tracker.py && sleep 0.2", :pty => true)
+  end
+
+  task :stop_tracker, :roles => :tracker do
+    run("pkill -f 'SCREEN.*murder_tracker.py'")
+  end
+
+  task :stop_all_seeding, :roles => :seeder do
+    run("pkill -f \"SCREEN.*seeder-\"")
+  end
+
+  task :stop_all_peering, :roles => :peer do
+    run("pkill -f \"murder_client.py peer\"")
+  end
+end

data/lib/murder/murder.rb

@@ -0,0 +1,121 @@
+# Copyright 2010 Twitter, Inc.
+# Copyright 2010 Larry Gadea <lg@twitter.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no default defaults...
+set :default_tag, ''
+set :default_seeder_files_path, ''
+set :default_destination_path, ''
+
+# default remote dist path in app shared directory
+set(:remote_murder_path) { "#{shared_path}/murder" }
+
+namespace :murder do
+  task :create_torrent, :roles => :seeder do
+    require_tag
+    if !(seeder_files_path = (default_seeder_files_path if default_seeder_files_path != "") || ENV['files_path'])
+      puts "You must specify a 'files_path' parameter with the directory on the seeder which contains the files to distribute"
+      exit(1)
+    end
+
+    if ENV['path_is_file']
+      run "cp \"#{seeder_files_path}\" #{filename}"
+    else
+      run "tar -c -z -C #{seeder_files_path}/ -f #{filename} --exclude \".git*\" ."
+    end
+
+    tracker = find_servers(:roles => :tracker).first
+    tracker_host = tracker.host
+    tracker_port = variables[:tracker_port] || '8998'
+
+    run "python #{remote_murder_path}/murder_make_torrent.py '#{filename}' #{tracker_host}:#{tracker_port} '#{filename}.torrent'"
+
+    download_torrent unless ENV['do_not_download_torrent']
+  end
+
+  task :download_torrent, :roles => :seeder do
+    require_tag
+    download("#{filename}.torrent", "#{filename}.torrent", :via => :scp)
+  end
+
+  task :start_seeding, :roles => :seeder do
+    require_tag
+    run "screen -dms 'seeder-#{tag}' python #{remote_murder_path}/murder_client.py seeder '#{filename}.torrent' '#{filename}' `host $HOSTNAME | awk '{print $4}'`"
+  end
+
+  task :stop_seeding, :roles => :seeder do
+    require_tag
+    run("pkill -f \"SCREEN.*seeder-#{tag}\"")
+  end
+
+  task :peer, :roles => :peer do
+    require_tag
+
+    if !(destination_path = (default_destination_path if default_destination_path != "") || ENV['destination_path'])
+      puts "You must specify a 'destination_path' parameter with the directory in which to place transferred (and extract) files. Note that inside this directory, a new directory named by the tag will be created. It is inside of this second diectory that the files which the torrent was created from will be placed. To not create this second directory, pass in parameter 'no_tag_directory=1'"
+      exit(1)
+    end
+
+    if !ENV['no_tag_directory'] && !ENV['path_is_file']
+      destination_path += "/#{tag}"
+    end
+
+    if !ENV['path_is_file']
+      run "mkdir -p #{destination_path}/"
+    end
+
+    if ENV['unsafe_please_delete']
+      run "rm -rf '#{destination_path}/'*"
+    end
+    if !ENV['no_tag_directory'] && !ENV['path_is_file']
+      run "find '#{destination_path}/'* >/dev/null 2>&1 && echo \"destination_path #{destination_path} on $HOSTNAME is not empty\" && exit 1 || exit 0"
+    end
+
+    upload("#{filename}.torrent", "#{filename}.torrent", :via => :scp)
+    run "/usr/bin/time -f %e python #{remote_murder_path}/murder_client.py peer '#{filename}.torrent' '#{filename}' `host $CAPISTRANO:HOST$ | awk '{print $4}'`"
+
+    if ENV['path_is_file']
+      run "cp #{filename} #{destination_path}"
+    else
+      run "tar xf #{filename} -C #{destination_path}"
+    end
+  end
+
+  task :stop_peering, :roles => :peer do
+    require_tag
+    run("pkill -f \"murder_client.py peer.*#{filename}\"")
+  end
+
+  task :clean_temp_files, :roles => [:peer, :seeder] do
+    require_tag
+    run "rm -f #{filename} #{filename}.torrent || exit 0"
+  end
+
+  ###
+
+  def require_tag
+    if !(temp_tag = (default_tag if default_tag != "") || ENV['tag'])
+      puts "You must specify a 'tag' parameter to identify the transfer"
+      exit(1)
+    end
+
+    if (temp_tag.include?("/"))
+      puts "Tag cannot contain a / character"
+      exit(1)
+    end
+
+    set :tag, temp_tag
+    set :filename, "/tmp/#{tag}.tgz"
+  end
+end

data/murder.gemspec

@@ -0,0 +1,101 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{murder}
+  s.version = "0.0.0.pre"
+
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Larry Gadea"]
+  s.date = %q{2010-03-15}
+  s.description = %q{Large scale server deploys using BitTorrent and the BitTornado library}
+  s.email = %q{lg@twitter.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README",
+     "Rakefile",
+     "VERSION",
+     "dist/BitTornado/BT1/Choker.py",
+     "dist/BitTornado/BT1/Connecter.py",
+     "dist/BitTornado/BT1/Downloader.py",
+     "dist/BitTornado/BT1/DownloaderFeedback.py",
+     "dist/BitTornado/BT1/Encrypter.py",
+     "dist/BitTornado/BT1/FileSelector.py",
+     "dist/BitTornado/BT1/Filter.py",
+     "dist/BitTornado/BT1/HTTPDownloader.py",
+     "dist/BitTornado/BT1/NatCheck.py",
+     "dist/BitTornado/BT1/PiecePicker.py",
+     "dist/BitTornado/BT1/Rerequester.py",
+     "dist/BitTornado/BT1/Statistics.py",
+     "dist/BitTornado/BT1/Storage.py",
+     "dist/BitTornado/BT1/StorageWrapper.py",
+     "dist/BitTornado/BT1/StreamCheck.py",
+     "dist/BitTornado/BT1/T2T.py",
+     "dist/BitTornado/BT1/Uploader.py",
+     "dist/BitTornado/BT1/__init__.py",
+     "dist/BitTornado/BT1/btformats.py",
+     "dist/BitTornado/BT1/fakeopen.py",
+     "dist/BitTornado/BT1/makemetafile.py",
+     "dist/BitTornado/BT1/track.py",
+     "dist/BitTornado/ConfigDir.py",
+     "dist/BitTornado/ConfigReader.py",
+     "dist/BitTornado/ConnChoice.py",
+     "dist/BitTornado/CreateIcons.py",
+     "dist/BitTornado/CurrentRateMeasure.py",
+     "dist/BitTornado/HTTPHandler.py",
+     "dist/BitTornado/PSYCO.py",
+     "dist/BitTornado/RateLimiter.py",
+     "dist/BitTornado/RateMeasure.py",
+     "dist/BitTornado/RawServer.py",
+     "dist/BitTornado/ServerPortHandler.py",
+     "dist/BitTornado/SocketHandler.py",
+     "dist/BitTornado/__init__.py",
+     "dist/BitTornado/bencode.py",
+     "dist/BitTornado/bitfield.py",
+     "dist/BitTornado/clock.py",
+     "dist/BitTornado/download_bt1.py",
+     "dist/BitTornado/inifile.py",
+     "dist/BitTornado/iprangeparse.py",
+     "dist/BitTornado/launchmanycore.py",
+     "dist/BitTornado/natpunch.py",
+     "dist/BitTornado/parseargs.py",
+     "dist/BitTornado/parsedir.py",
+     "dist/BitTornado/piecebuffer.py",
+     "dist/BitTornado/selectpoll.py",
+     "dist/BitTornado/subnetparse.py",
+     "dist/BitTornado/torrentlistparse.py",
+     "dist/BitTornado/zurllib.py",
+     "dist/murder_client.py",
+     "dist/murder_make_torrent.py",
+     "dist/murder_tracker.py",
+     "doc/examples/Capfile",
+     "lib/capistrano/recipes/deploy/strategy/murder.rb",
+     "lib/murder.rb",
+     "lib/murder/admin.rb",
+     "lib/murder/murder.rb",
+     "murder.gemspec"
+  ]
+  s.homepage = %q{http://github.com/lg/murder}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Large scale server deploys using BitTorrent and the BitTornado library}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification 
+name: murder
+version: !ruby/object:Gem::Version 
+  prerelease: true
+  segments: 
+  - 0
+  - 0
+  - 0
+  - pre
+  version: 0.0.0.pre
+platform: ruby
+authors: 
+- Larry Gadea
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-15 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Large scale server deploys using BitTorrent and the BitTornado library
+email: lg@twitter.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README
+files: 
+- .gitignore
+- LICENSE
+- README
+- Rakefile
+- VERSION
+- dist/BitTornado/BT1/Choker.py
+- dist/BitTornado/BT1/Connecter.py
+- dist/BitTornado/BT1/Downloader.py
+- dist/BitTornado/BT1/DownloaderFeedback.py
+- dist/BitTornado/BT1/Encrypter.py
+- dist/BitTornado/BT1/FileSelector.py
+- dist/BitTornado/BT1/Filter.py
+- dist/BitTornado/BT1/HTTPDownloader.py
+- dist/BitTornado/BT1/NatCheck.py
+- dist/BitTornado/BT1/PiecePicker.py
+- dist/BitTornado/BT1/Rerequester.py
+- dist/BitTornado/BT1/Statistics.py
+- dist/BitTornado/BT1/Storage.py
+- dist/BitTornado/BT1/StorageWrapper.py
+- dist/BitTornado/BT1/StreamCheck.py
+- dist/BitTornado/BT1/T2T.py
+- dist/BitTornado/BT1/Uploader.py
+- dist/BitTornado/BT1/__init__.py
+- dist/BitTornado/BT1/btformats.py
+- dist/BitTornado/BT1/fakeopen.py
+- dist/BitTornado/BT1/makemetafile.py
+- dist/BitTornado/BT1/track.py
+- dist/BitTornado/ConfigDir.py
+- dist/BitTornado/ConfigReader.py
+- dist/BitTornado/ConnChoice.py
+- dist/BitTornado/CreateIcons.py
+- dist/BitTornado/CurrentRateMeasure.py
+- dist/BitTornado/HTTPHandler.py
+- dist/BitTornado/PSYCO.py
+- dist/BitTornado/RateLimiter.py
+- dist/BitTornado/RateMeasure.py
+- dist/BitTornado/RawServer.py
+- dist/BitTornado/ServerPortHandler.py
+- dist/BitTornado/SocketHandler.py
+- dist/BitTornado/__init__.py
+- dist/BitTornado/bencode.py
+- dist/BitTornado/bitfield.py
+- dist/BitTornado/clock.py
+- dist/BitTornado/download_bt1.py
+- dist/BitTornado/inifile.py
+- dist/BitTornado/iprangeparse.py
+- dist/BitTornado/launchmanycore.py
+- dist/BitTornado/natpunch.py
+- dist/BitTornado/parseargs.py
+- dist/BitTornado/parsedir.py
+- dist/BitTornado/piecebuffer.py
+- dist/BitTornado/selectpoll.py
+- dist/BitTornado/subnetparse.py
+- dist/BitTornado/torrentlistparse.py
+- dist/BitTornado/zurllib.py
+- dist/murder_client.py
+- dist/murder_make_torrent.py
+- dist/murder_tracker.py
+- doc/examples/Capfile
+- lib/capistrano/recipes/deploy/strategy/murder.rb
+- lib/murder.rb
+- lib/murder/admin.rb
+- lib/murder/murder.rb
+- murder.gemspec
+has_rdoc: true
+homepage: http://github.com/lg/murder
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Large scale server deploys using BitTorrent and the BitTornado library
+test_files: []
+