git-media 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.gitignore +5 -0
- data/LICENSE +20 -0
- data/README.rdoc +87 -0
- data/Rakefile +48 -0
- data/VERSION +1 -0
- data/bin/git-media +8 -0
- data/git-media.gemspec +55 -0
- data/lib/git-media.rb +101 -0
- data/lib/git-media/clear.rb +23 -0
- data/lib/git-media/filter-clean.rb +38 -0
- data/lib/git-media/filter-smudge.rb +37 -0
- data/lib/git-media/status.rb +107 -0
- data/lib/git-media/sync.rb +47 -0
- data/lib/git-media/transport.rb +39 -0
- data/lib/git-media/transport/local.rb +50 -0
- data/lib/git-media/transport/s3.rb +55 -0
- data/lib/git-media/transport/scp.rb +68 -0
- metadata +82 -0
data/.document
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Scott Chacon
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
= git-media
|
2
|
+
|
3
|
+
GitMedia extension allows you to use Git with large media files
|
4
|
+
without storing the media in Git itself.
|
5
|
+
|
6
|
+
== Configuration
|
7
|
+
|
8
|
+
Setup the attributes filter settings.
|
9
|
+
|
10
|
+
(once after install)
|
11
|
+
$ git config filter.media.clean "git-media filter-clean"
|
12
|
+
$ git config filter.media.smudge "git-media filter-smudge"
|
13
|
+
|
14
|
+
Setup the .gitattributes file to map extensions to the filter.
|
15
|
+
|
16
|
+
(in repo - once)
|
17
|
+
$ echo "*.mov filter=media" > .gitattributes
|
18
|
+
|
19
|
+
Staging files with those extensions will automatically copy them to the
|
20
|
+
media buffer area (.git/media) until you run 'git media sync' wherein they
|
21
|
+
are uploaded. Checkouts that reference media you don't have yet will try to
|
22
|
+
be automatically downloaded, otherwise they are downloaded when you sync.
|
23
|
+
|
24
|
+
Next you need to configure git to tell it where you want to store the large files.
|
25
|
+
There are three options:
|
26
|
+
|
27
|
+
1. Storing remotely in Amazon's S3
|
28
|
+
2. Storing locally in a filesystem path
|
29
|
+
3. Storing remotely via SCP (should work with any SSH server)
|
30
|
+
|
31
|
+
Here are the relevant sections that should go either in ~/.gitconfig (for global settings)
|
32
|
+
or in clone/.git/config (for per-repo settings).
|
33
|
+
|
34
|
+
[git-media]
|
35
|
+
transport = <scp|local|s3>
|
36
|
+
|
37
|
+
# settings for scp transport
|
38
|
+
scpuser=<user>
|
39
|
+
scphost=<host>
|
40
|
+
scppath=<path_on_remote_server>
|
41
|
+
|
42
|
+
# settings for local transport
|
43
|
+
path=<local_filesystem_path>
|
44
|
+
|
45
|
+
# settings for s3 transport
|
46
|
+
s3bucket=<name_of_bucket>
|
47
|
+
s3key=<s3 access key>
|
48
|
+
s3secret=<s3 secret key>
|
49
|
+
|
50
|
+
== Usage
|
51
|
+
|
52
|
+
(in repo - repeatedly)
|
53
|
+
$ (hack, stage, commit)
|
54
|
+
$ git media sync
|
55
|
+
|
56
|
+
You can also check the status of your media files via
|
57
|
+
|
58
|
+
$ git media status
|
59
|
+
|
60
|
+
Which will show you files that are waiting to be uploaded and how much data
|
61
|
+
that is. If you want to upload & delete the local cache of media files, run:
|
62
|
+
|
63
|
+
$ git media clear
|
64
|
+
|
65
|
+
== Config Settings
|
66
|
+
|
67
|
+
$ git config --global media.auto-download false
|
68
|
+
|
69
|
+
|
70
|
+
== Installing
|
71
|
+
|
72
|
+
$ sudo gem install trollop
|
73
|
+
$ sudo gem install s3
|
74
|
+
$ sudo gem install right_aws
|
75
|
+
$ gem build git-media.gemspec
|
76
|
+
$ sudo gem install git-media-0.1.1.gem
|
77
|
+
|
78
|
+
== Notes for Windows
|
79
|
+
If installing on windows, you might run into a problem verifying certificates
|
80
|
+
for S3 or something. If that happens, modify
|
81
|
+
C:\Ruby191\lib\ruby\gems\1.9.1\gems\right_http_connection-1.2.4\lib\right_http_connection.rb
|
82
|
+
And add at line 310, right before "@http.start":
|
83
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
84
|
+
|
85
|
+
== Copyright
|
86
|
+
|
87
|
+
Copyright (c) 2009 Scott Chacon. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "git-media"
|
8
|
+
gem.summary = %Q{TODO}
|
9
|
+
gem.email = "schacon@gmail.com"
|
10
|
+
gem.homepage = "http://github.com/schacon/git-media"
|
11
|
+
gem.authors = ["Scott Chacon"]
|
12
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
13
|
+
end
|
14
|
+
|
15
|
+
rescue LoadError
|
16
|
+
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
17
|
+
end
|
18
|
+
|
19
|
+
require 'spec/rake/spectask'
|
20
|
+
Spec::Rake::SpecTask.new(:spec) do |spec|
|
21
|
+
spec.libs << 'lib' << 'spec'
|
22
|
+
spec.spec_files = FileList['spec/**/*_spec.rb']
|
23
|
+
end
|
24
|
+
|
25
|
+
Spec::Rake::SpecTask.new(:rcov) do |spec|
|
26
|
+
spec.libs << 'lib' << 'spec'
|
27
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
28
|
+
spec.rcov = true
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
task :default => :spec
|
33
|
+
|
34
|
+
require 'rake/rdoctask'
|
35
|
+
Rake::RDocTask.new do |rdoc|
|
36
|
+
if File.exist?('VERSION.yml')
|
37
|
+
config = YAML.load(File.read('VERSION.yml'))
|
38
|
+
version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
|
39
|
+
else
|
40
|
+
version = ""
|
41
|
+
end
|
42
|
+
|
43
|
+
rdoc.rdoc_dir = 'rdoc'
|
44
|
+
rdoc.title = "git-media #{version}"
|
45
|
+
rdoc.rdoc_files.include('README*')
|
46
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
47
|
+
end
|
48
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/bin/git-media
ADDED
data/git-media.gemspec
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{git-media}
|
5
|
+
s.version = "0.1.1"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Scott Chacon"]
|
9
|
+
s.date = %q{2009-06-10}
|
10
|
+
s.default_executable = %q{git-media}
|
11
|
+
s.email = %q{schacon@gmail.com}
|
12
|
+
s.executables = ["git-media"]
|
13
|
+
s.extra_rdoc_files = [
|
14
|
+
"LICENSE",
|
15
|
+
"README.rdoc"
|
16
|
+
]
|
17
|
+
s.files = [
|
18
|
+
".document",
|
19
|
+
".gitignore",
|
20
|
+
"LICENSE",
|
21
|
+
"README.rdoc",
|
22
|
+
"Rakefile",
|
23
|
+
"VERSION",
|
24
|
+
"bin/git-media",
|
25
|
+
"git-media.gemspec",
|
26
|
+
"lib/git-media/clear.rb",
|
27
|
+
"lib/git-media/filter-clean.rb",
|
28
|
+
"lib/git-media/filter-smudge.rb",
|
29
|
+
"lib/git-media/status.rb",
|
30
|
+
"lib/git-media/sync.rb",
|
31
|
+
"lib/git-media/transport",
|
32
|
+
"lib/git-media/transport/local.rb",
|
33
|
+
"lib/git-media/transport/s3.rb",
|
34
|
+
"lib/git-media/transport/scp.rb",
|
35
|
+
"lib/git-media/transport.rb",
|
36
|
+
"lib/git-media.rb"
|
37
|
+
]
|
38
|
+
s.has_rdoc = true
|
39
|
+
s.homepage = %q{http://github.com/schacon/git-media}
|
40
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
41
|
+
s.require_paths = ["lib"]
|
42
|
+
s.rubygems_version = %q{1.3.1}
|
43
|
+
s.summary = %q{"This is a summary! Stop yer whining"}
|
44
|
+
|
45
|
+
if s.respond_to? :specification_version then
|
46
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
47
|
+
s.specification_version = 2
|
48
|
+
|
49
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
50
|
+
else
|
51
|
+
end
|
52
|
+
else
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
data/lib/git-media.rb
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'trollop'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'git-media/transport/local'
|
4
|
+
require 'git-media/transport/s3'
|
5
|
+
require 'git-media/transport/scp'
|
6
|
+
|
7
|
+
module GitMedia
|
8
|
+
|
9
|
+
def self.get_media_buffer
|
10
|
+
@@git_dir ||= `git rev-parse --git-dir`.chomp
|
11
|
+
media_buffer = File.join(@@git_dir, 'media/objects')
|
12
|
+
FileUtils.mkdir_p(media_buffer) if !File.exist?(media_buffer)
|
13
|
+
return media_buffer
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.media_path(sha)
|
17
|
+
buf = self.get_media_buffer
|
18
|
+
File.join(buf, sha)
|
19
|
+
end
|
20
|
+
|
21
|
+
# TODO: select the proper transports based on settings
|
22
|
+
def self.get_push_transport
|
23
|
+
self.get_transport
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.get_transport
|
27
|
+
case `git config git-media.transport`.chomp
|
28
|
+
when ""
|
29
|
+
raise "git-media.transport not set"
|
30
|
+
when "scp"
|
31
|
+
user = `git config git-media.scpuser`.chomp
|
32
|
+
host = `git config git-media.scphost`.chomp
|
33
|
+
path = `git config git-media.scppath`.chomp
|
34
|
+
if user === ""
|
35
|
+
raise "git-media.scpuser not set for scp transport"
|
36
|
+
end
|
37
|
+
if host === ""
|
38
|
+
raise "git-media.scphost not set for scp transport"
|
39
|
+
end
|
40
|
+
if path === ""
|
41
|
+
raise "git-media.scppath not set for scp transport"
|
42
|
+
end
|
43
|
+
GitMedia::Transport::Scp.new(user, host, path)
|
44
|
+
|
45
|
+
when "local"
|
46
|
+
path = `git config git-media.localpath`.chomp
|
47
|
+
if path === ""
|
48
|
+
raise "git-media.localpath not set for local transport"
|
49
|
+
end
|
50
|
+
GitMedia::Transport::Local.new(path)
|
51
|
+
when "s3"
|
52
|
+
bucket = `git config git-media.s3bucket`.chomp
|
53
|
+
key = `git config git-media.s3key`.chomp
|
54
|
+
secret = `git config git-media.s3secret`.chomp
|
55
|
+
if bucket === ""
|
56
|
+
raise "git-media.s3bucket not set for s3 transport"
|
57
|
+
end
|
58
|
+
if key === ""
|
59
|
+
raise "git-media.s3key not set for s3 transport"
|
60
|
+
end
|
61
|
+
if secret === ""
|
62
|
+
raise "git-media.s3secret not set for s3 transport"
|
63
|
+
end
|
64
|
+
GitMedia::Transport::S3.new(bucket, key, secret)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.get_pull_transport
|
69
|
+
self.get_transport
|
70
|
+
end
|
71
|
+
|
72
|
+
module Application
|
73
|
+
def self.run!
|
74
|
+
|
75
|
+
cmd = ARGV.shift # get the subcommand
|
76
|
+
cmd_opts = case cmd
|
77
|
+
when "filter-clean" # parse delete options
|
78
|
+
require 'git-media/filter-clean'
|
79
|
+
GitMedia::FilterClean.run!
|
80
|
+
when "filter-smudge"
|
81
|
+
require 'git-media/filter-smudge'
|
82
|
+
GitMedia::FilterSmudge.run!
|
83
|
+
when "clear" # parse delete options
|
84
|
+
require 'git-media/clear'
|
85
|
+
GitMedia::Clear.run!
|
86
|
+
when "sync"
|
87
|
+
require 'git-media/sync'
|
88
|
+
GitMedia::Sync.run!
|
89
|
+
when 'status'
|
90
|
+
require 'git-media/status'
|
91
|
+
Trollop::options do
|
92
|
+
opt :force, "Force status"
|
93
|
+
end
|
94
|
+
GitMedia::Status.run!
|
95
|
+
else
|
96
|
+
raise "unknown media subcommand #{cmd.inspect}"
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'git-media/status'
|
2
|
+
|
3
|
+
module GitMedia
|
4
|
+
module Clear
|
5
|
+
|
6
|
+
def self.run!
|
7
|
+
@push = GitMedia.get_push_transport
|
8
|
+
self.clear_local_cache
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.clear_local_cache
|
12
|
+
# find files in media buffer and upload them
|
13
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
14
|
+
unpushed_files = @push.get_unpushed(all_cache)
|
15
|
+
pushed_files = all_cache - unpushed_files
|
16
|
+
pushed_files.each do |sha|
|
17
|
+
puts "removing " + sha[0, 8]
|
18
|
+
File.unlink(File.join(GitMedia.get_media_buffer, sha))
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'digest/sha1'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'tempfile'
|
4
|
+
|
5
|
+
module GitMedia
|
6
|
+
module FilterClean
|
7
|
+
|
8
|
+
def self.run!
|
9
|
+
# determine and initialize our media buffer directory
|
10
|
+
media_buffer = GitMedia.get_media_buffer
|
11
|
+
|
12
|
+
hashfunc = Digest::SHA1.new
|
13
|
+
start = Time.now
|
14
|
+
|
15
|
+
# TODO: read first 41 bytes and see if this is a stub
|
16
|
+
|
17
|
+
# read in buffered chunks of the data
|
18
|
+
# calculating the SHA and copying to a tempfile
|
19
|
+
tempfile = Tempfile.new('media')
|
20
|
+
while data = STDIN.read(4096)
|
21
|
+
hashfunc.update(data)
|
22
|
+
tempfile.write(data)
|
23
|
+
end
|
24
|
+
tempfile.close
|
25
|
+
|
26
|
+
# calculate and print the SHA of the data
|
27
|
+
puts hx = hashfunc.hexdigest
|
28
|
+
|
29
|
+
# move the tempfile to our media buffer area
|
30
|
+
media_file = File.join(media_buffer, hx)
|
31
|
+
FileUtils.mv(tempfile.path, media_file)
|
32
|
+
|
33
|
+
elapsed = Time.now - start
|
34
|
+
STDERR.puts('Saving media : ' + hx + ' : ' + elapsed.to_s)
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module GitMedia
|
2
|
+
module FilterSmudge
|
3
|
+
|
4
|
+
def self.run!
|
5
|
+
media_buffer = GitMedia.get_media_buffer
|
6
|
+
can_download = false # TODO: read this from config and implement
|
7
|
+
|
8
|
+
# read checksum size
|
9
|
+
sha = STDIN.read(41)
|
10
|
+
if STDIN.eof? && sha[40, 1] == "\n"
|
11
|
+
# this is a media file
|
12
|
+
media_file = File.join(media_buffer, sha.chomp)
|
13
|
+
if File.exists?(media_file)
|
14
|
+
STDERR.puts('recovering media : ' + sha)
|
15
|
+
File.open(media_file, 'r') do |f|
|
16
|
+
while data = f.read(4096) do
|
17
|
+
print data
|
18
|
+
end
|
19
|
+
end
|
20
|
+
else
|
21
|
+
# TODO: download file if not in the media buffer area
|
22
|
+
if !can_download
|
23
|
+
STDERR.puts('media missing, saving placeholder : ' + sha)
|
24
|
+
puts sha
|
25
|
+
end
|
26
|
+
end
|
27
|
+
else
|
28
|
+
# if more than 40 chars, just output
|
29
|
+
print sha
|
30
|
+
while data = STDIN.read(4096)
|
31
|
+
print data
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'pp'
|
2
|
+
|
3
|
+
module GitMedia
|
4
|
+
module Status
|
5
|
+
|
6
|
+
def self.run!
|
7
|
+
@push = GitMedia.get_push_transport
|
8
|
+
r = self.find_references
|
9
|
+
self.print_references(r)
|
10
|
+
r = self.local_cache_status
|
11
|
+
self.print_cache_status(r)
|
12
|
+
end
|
13
|
+
|
14
|
+
# find tree entries that are likely media references
|
15
|
+
def self.find_references
|
16
|
+
references = {:to_expand => [], :expanded => [], :deleted => []}
|
17
|
+
files = `git ls-tree -l -r HEAD`.split("\n")
|
18
|
+
files = files.map { |f| s = f.split("\t"); [s[0].split(' ').last, s[1]] }
|
19
|
+
files = files.select { |f| f[0] == '41' } # it's the right size
|
20
|
+
files.each do |tree_size, fname|
|
21
|
+
if size = File.size?(fname)
|
22
|
+
if size == tree_size.to_i
|
23
|
+
# TODO: read in the data and verify that it's a sha + newline
|
24
|
+
sha = File.read(fname).strip
|
25
|
+
if sha.length == 40
|
26
|
+
references[:to_expand] << [fname, sha]
|
27
|
+
end
|
28
|
+
else
|
29
|
+
references[:expanded] << fname
|
30
|
+
end
|
31
|
+
else
|
32
|
+
# file was deleted
|
33
|
+
references[:deleted] << fname
|
34
|
+
end
|
35
|
+
end
|
36
|
+
references
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.print_references(refs)
|
40
|
+
if refs[:to_expand].size > 0
|
41
|
+
puts "== Unexpanded Media =="
|
42
|
+
refs[:to_expand].each do |file, sha|
|
43
|
+
puts " " + sha[0, 8] + " " + file
|
44
|
+
end
|
45
|
+
puts
|
46
|
+
end
|
47
|
+
if refs[:expanded].size > 0
|
48
|
+
puts "== Expanded Media =="
|
49
|
+
refs[:expanded].each do |file|
|
50
|
+
size = File.size(file)
|
51
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + " #{file}"
|
52
|
+
end
|
53
|
+
puts
|
54
|
+
end
|
55
|
+
if refs[:deleted].size > 0
|
56
|
+
puts "== Deleted Media =="
|
57
|
+
refs[:deleted].each do |file|
|
58
|
+
puts " " + " #{file}"
|
59
|
+
end
|
60
|
+
puts
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.print_cache_status(refs)
|
65
|
+
if refs[:unpushed].size > 0
|
66
|
+
puts "== Unpushed Media =="
|
67
|
+
refs[:unpushed].each do |sha|
|
68
|
+
cache_file = GitMedia.media_path(sha)
|
69
|
+
size = File.size(cache_file)
|
70
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
|
71
|
+
end
|
72
|
+
puts
|
73
|
+
end
|
74
|
+
if refs[:pushed].size > 0
|
75
|
+
puts "== Already Pushed Media =="
|
76
|
+
refs[:pushed].each do |sha|
|
77
|
+
cache_file = GitMedia.media_path(sha)
|
78
|
+
size = File.size(cache_file)
|
79
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
|
80
|
+
end
|
81
|
+
puts
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.local_cache_status
|
86
|
+
# find files in media buffer and upload them
|
87
|
+
references = {:unpushed => [], :pushed => []}
|
88
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
89
|
+
unpushed_files = @push.get_unpushed(all_cache) || []
|
90
|
+
references[:unpushed] = unpushed_files
|
91
|
+
references[:pushed] = all_cache - unpushed_files rescue []
|
92
|
+
references
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
def self.to_human(size)
|
97
|
+
if size < 1024
|
98
|
+
return size.to_s + 'b'
|
99
|
+
elsif size < 1048576
|
100
|
+
return (size / 1024).to_s + 'k'
|
101
|
+
else
|
102
|
+
return (size / 1048576).to_s + 'm'
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# find files that are placeholders (41 char) and download them
|
2
|
+
# upload files in media buffer that are not in offsite bin
|
3
|
+
require 'git-media/status'
|
4
|
+
|
5
|
+
module GitMedia
|
6
|
+
module Sync
|
7
|
+
|
8
|
+
def self.run!
|
9
|
+
@push = GitMedia.get_push_transport
|
10
|
+
@pull = GitMedia.get_pull_transport
|
11
|
+
|
12
|
+
self.expand_references
|
13
|
+
self.upload_local_cache
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.expand_references
|
17
|
+
status = GitMedia::Status.find_references
|
18
|
+
status[:to_expand].each do |file, sha|
|
19
|
+
cache_file = GitMedia.media_path(sha)
|
20
|
+
if !File.exist?(cache_file)
|
21
|
+
puts "Downloading " + sha[0,8] + " : " + file
|
22
|
+
@pull.pull(file, sha)
|
23
|
+
end
|
24
|
+
|
25
|
+
puts "Expanding " + sha[0,8] + " : " + file
|
26
|
+
|
27
|
+
if File.exist?(cache_file)
|
28
|
+
FileUtils.cp(cache_file, file)
|
29
|
+
else
|
30
|
+
puts 'could not get media'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.upload_local_cache
|
36
|
+
# find files in media buffer and upload them
|
37
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
38
|
+
unpushed_files = @push.get_unpushed(all_cache)
|
39
|
+
unpushed_files.each do |sha|
|
40
|
+
puts 'uploading ' + sha[0, 8]
|
41
|
+
@push.push(sha)
|
42
|
+
end
|
43
|
+
# TODO: if --clean, remove them
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module GitMedia
|
2
|
+
module Transport
|
3
|
+
class Base
|
4
|
+
|
5
|
+
def pull(final_file, sha)
|
6
|
+
to_file = GitMedia.media_path(sha)
|
7
|
+
get_file(sha, to_file)
|
8
|
+
end
|
9
|
+
|
10
|
+
def push(sha)
|
11
|
+
from_file = GitMedia.media_path(sha)
|
12
|
+
put_file(sha, from_file)
|
13
|
+
end
|
14
|
+
|
15
|
+
## OVERWRITE ##
|
16
|
+
|
17
|
+
def read?
|
18
|
+
false
|
19
|
+
end
|
20
|
+
|
21
|
+
def write?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_file(sha, to_file)
|
26
|
+
false
|
27
|
+
end
|
28
|
+
|
29
|
+
def put_file(sha, to_file)
|
30
|
+
false
|
31
|
+
end
|
32
|
+
|
33
|
+
def get_unpushed(files)
|
34
|
+
files
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
|
3
|
+
# move large media to local bin
|
4
|
+
|
5
|
+
# git-media.transport local
|
6
|
+
# git-media.localpath /opt/media
|
7
|
+
|
8
|
+
module GitMedia
|
9
|
+
module Transport
|
10
|
+
class Local < Base
|
11
|
+
|
12
|
+
def initialize(path)
|
13
|
+
@path = path
|
14
|
+
end
|
15
|
+
|
16
|
+
def read?
|
17
|
+
File.exist?(@path)
|
18
|
+
end
|
19
|
+
|
20
|
+
def get_file(sha, to_file)
|
21
|
+
from_file = File.join(@path, sha)
|
22
|
+
if File.exists?(from_file)
|
23
|
+
FileUtils.cp(from_file, to_file)
|
24
|
+
return true
|
25
|
+
end
|
26
|
+
return false
|
27
|
+
end
|
28
|
+
|
29
|
+
def write?
|
30
|
+
File.exist?(@path)
|
31
|
+
end
|
32
|
+
|
33
|
+
def put_file(sha, from_file)
|
34
|
+
to_file = File.join(@path, sha)
|
35
|
+
if File.exists?(from_file)
|
36
|
+
FileUtils.cp(from_file, to_file)
|
37
|
+
return true
|
38
|
+
end
|
39
|
+
return false
|
40
|
+
end
|
41
|
+
|
42
|
+
def get_unpushed(files)
|
43
|
+
files.select do |f|
|
44
|
+
!File.exist?(File.join(@path, f))
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
require 'right_aws'
|
3
|
+
|
4
|
+
# git-media.transport s3
|
5
|
+
# git-media.s3bucket
|
6
|
+
# git-media.s3key
|
7
|
+
# git-media.s3secret
|
8
|
+
|
9
|
+
module GitMedia
|
10
|
+
module Transport
|
11
|
+
class S3 < Base
|
12
|
+
|
13
|
+
def initialize(bucket, access_key_id = nil, secret_access_key = nil)
|
14
|
+
@s3 = RightAws::S3Interface.new(access_key_id, secret_access_key,
|
15
|
+
{:multi_thread => true, :logger => Logger.new('/tmp/s3.log')})
|
16
|
+
@bucket = bucket
|
17
|
+
@buckets = @s3.list_all_my_buckets.map { |a| a[:name] }
|
18
|
+
if !@buckets.include?(bucket)
|
19
|
+
puts "Creating New Bucket"
|
20
|
+
if @s3.create_bucket(bucket)
|
21
|
+
@buckets << bucket
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def read?
|
27
|
+
@buckets.size > 0
|
28
|
+
end
|
29
|
+
|
30
|
+
def get_file(sha, to_file)
|
31
|
+
to = File.new(to_file, File::CREAT|File::RDWR)
|
32
|
+
@s3.get(@bucket, sha) do |chunk|
|
33
|
+
to.write(chunk)
|
34
|
+
end
|
35
|
+
to.close
|
36
|
+
end
|
37
|
+
|
38
|
+
def write?
|
39
|
+
@buckets.size > 0
|
40
|
+
end
|
41
|
+
|
42
|
+
def put_file(sha, from_file)
|
43
|
+
@s3.put(@bucket, sha, File.open(from_file))
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_unpushed(files)
|
47
|
+
keys = @s3.list_bucket(@bucket).map { |f| f[:key] }
|
48
|
+
files.select do |f|
|
49
|
+
!keys.include?(f)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
|
3
|
+
# move large media to remote server via SCP
|
4
|
+
|
5
|
+
# git-media.transport scp
|
6
|
+
# git-media.scpuser someuser
|
7
|
+
# git-media.scphost remoteserver.com
|
8
|
+
# git-media.scppath /opt/media
|
9
|
+
|
10
|
+
module GitMedia
|
11
|
+
module Transport
|
12
|
+
class Scp < Base
|
13
|
+
|
14
|
+
def initialize(user, host, path)
|
15
|
+
@user = user
|
16
|
+
@host = host
|
17
|
+
@path = path
|
18
|
+
end
|
19
|
+
|
20
|
+
def exist?(file)
|
21
|
+
if `ssh #{@user}@#{@host} [ -f "#{file}" ] && echo 1 || echo 0`.chomp == "1"
|
22
|
+
puts file + " exists"
|
23
|
+
return true
|
24
|
+
else
|
25
|
+
puts file + " doesn't exists"
|
26
|
+
return false
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def read?
|
31
|
+
return true
|
32
|
+
end
|
33
|
+
|
34
|
+
def get_file(sha, to_file)
|
35
|
+
from_file = @user+"@"+@host+":"+File.join(@path, sha)
|
36
|
+
`scp "#{from_file}" "#{to_file}"`
|
37
|
+
if $? == 0
|
38
|
+
puts sha+" downloaded"
|
39
|
+
return true
|
40
|
+
end
|
41
|
+
puts sha+" download fail"
|
42
|
+
return false
|
43
|
+
end
|
44
|
+
|
45
|
+
def write?
|
46
|
+
return true
|
47
|
+
end
|
48
|
+
|
49
|
+
def put_file(sha, from_file)
|
50
|
+
to_file = @user+"@"+@host+":"+File.join(@path, sha)
|
51
|
+
`scp "#{from_file}" "#{to_file}"`
|
52
|
+
if $? == 0
|
53
|
+
puts sha+" uploaded"
|
54
|
+
return true
|
55
|
+
end
|
56
|
+
puts sha+" upload fail"
|
57
|
+
return false
|
58
|
+
end
|
59
|
+
|
60
|
+
def get_unpushed(files)
|
61
|
+
files.select do |f|
|
62
|
+
!self.exist?(File.join(@path, f))
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
metadata
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: git-media
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 1
|
8
|
+
- 1
|
9
|
+
version: 0.1.1
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Scott Chacon
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2009-06-10 00:00:00 +02:00
|
18
|
+
default_executable: git-media
|
19
|
+
dependencies: []
|
20
|
+
|
21
|
+
description:
|
22
|
+
email: schacon@gmail.com
|
23
|
+
executables:
|
24
|
+
- git-media
|
25
|
+
extensions: []
|
26
|
+
|
27
|
+
extra_rdoc_files:
|
28
|
+
- LICENSE
|
29
|
+
- README.rdoc
|
30
|
+
files:
|
31
|
+
- .document
|
32
|
+
- .gitignore
|
33
|
+
- LICENSE
|
34
|
+
- README.rdoc
|
35
|
+
- Rakefile
|
36
|
+
- VERSION
|
37
|
+
- bin/git-media
|
38
|
+
- git-media.gemspec
|
39
|
+
- lib/git-media/clear.rb
|
40
|
+
- lib/git-media/filter-clean.rb
|
41
|
+
- lib/git-media/filter-smudge.rb
|
42
|
+
- lib/git-media/status.rb
|
43
|
+
- lib/git-media/sync.rb
|
44
|
+
- lib/git-media/transport/local.rb
|
45
|
+
- lib/git-media/transport/s3.rb
|
46
|
+
- lib/git-media/transport/scp.rb
|
47
|
+
- lib/git-media/transport.rb
|
48
|
+
- lib/git-media.rb
|
49
|
+
has_rdoc: true
|
50
|
+
homepage: http://github.com/schacon/git-media
|
51
|
+
licenses: []
|
52
|
+
|
53
|
+
post_install_message:
|
54
|
+
rdoc_options:
|
55
|
+
- --charset=UTF-8
|
56
|
+
require_paths:
|
57
|
+
- lib
|
58
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
59
|
+
none: false
|
60
|
+
requirements:
|
61
|
+
- - ">="
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
segments:
|
64
|
+
- 0
|
65
|
+
version: "0"
|
66
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
67
|
+
none: false
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
version: "0"
|
74
|
+
requirements: []
|
75
|
+
|
76
|
+
rubyforge_project:
|
77
|
+
rubygems_version: 1.3.7
|
78
|
+
signing_key:
|
79
|
+
specification_version: 2
|
80
|
+
summary: "\"This is a summary! Stop yer whining\""
|
81
|
+
test_files: []
|
82
|
+
|