notepunch-git-media 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.document +5 -0
- data/.gitignore +5 -0
- data/LICENSE +20 -0
- data/README.rdoc +100 -0
- data/Rakefile +41 -0
- data/VERSION +1 -0
- data/bin/git-media +8 -0
- data/lib/git-media.rb +131 -0
- data/lib/git-media/clear.rb +23 -0
- data/lib/git-media/filter-clean.rb +40 -0
- data/lib/git-media/filter-smudge.rb +38 -0
- data/lib/git-media/status.rb +107 -0
- data/lib/git-media/sync.rb +47 -0
- data/lib/git-media/transport.rb +39 -0
- data/lib/git-media/transport/atmos_client.rb +71 -0
- data/lib/git-media/transport/local.rb +50 -0
- data/lib/git-media/transport/s3.rb +55 -0
- data/lib/git-media/transport/scp.rb +80 -0
- data/notepunch-git-media.gemspec +56 -0
- metadata +65 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NmU1NjE4MTY5ZTU1ZDg0YTEzNjE3NmI5ZmZkYmYzNzQ4NzA4NjQ1Zg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
YjE5ODM1YzkyNmM4MTAzY2YwOTNhYWMwNDk1NWFmMTRjNDQwMjdmYw==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
NDJjODhmZmRmMDVmNzNhNTMzMWI5MGJmM2M2M2YzMGJhYTQyNjQ0NTQ0ZTcx
|
10
|
+
ZWMxYWQyOWJmNmU0M2RjMmYzZTU4N2UwMDdmYTgxNGYwZTlkNjc2MzBmNGE4
|
11
|
+
MGJhMWJjZmM4ZDY3M2FmYzkwZTZiOTllNTk3MzEzNWFlYTYwZDc=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
NmMwZmQyZmFlY2MyZThkYjcxZmYzZTYzNzliMjBkYWY5YWUzNTNmNTY5ZDZj
|
14
|
+
MDRkYWY3ZTkxYjRkNTQ4NTEyZjhmNzhiY2RiZTY1YmNkYjZiNjZlMmY5ZDc3
|
15
|
+
YjZjNTNhNDA3NjRkZGExOGU3ZTI3NjQxMTdkZjAxOWQ5OTAxNDk=
|
data/.document
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Scott Chacon
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
= git-media
|
2
|
+
|
3
|
+
GitMedia extension allows you to use Git with large media files
|
4
|
+
without storing the media in Git itself.
|
5
|
+
|
6
|
+
== Configuration
|
7
|
+
|
8
|
+
Setup the attributes filter settings.
|
9
|
+
|
10
|
+
(once after install)
|
11
|
+
$ git config filter.media.clean "git-media filter-clean"
|
12
|
+
$ git config filter.media.smudge "git-media filter-smudge"
|
13
|
+
|
14
|
+
Setup the .gitattributes file to map extensions to the filter.
|
15
|
+
|
16
|
+
(in repo - once)
|
17
|
+
$ echo "*.mov filter=media -crlf" > .gitattributes
|
18
|
+
|
19
|
+
Staging files with those extensions will automatically copy them to the
|
20
|
+
media buffer area (.git/media) until you run 'git media sync' wherein they
|
21
|
+
are uploaded. Checkouts that reference media you don't have yet will try to
|
22
|
+
be automatically downloaded, otherwise they are downloaded when you sync.
|
23
|
+
|
24
|
+
Next you need to configure git to tell it where you want to store the large files.
|
25
|
+
There are four options:
|
26
|
+
|
27
|
+
1. Storing remotely in Amazon's S3
|
28
|
+
2. Storing locally in a filesystem path
|
29
|
+
3. Storing remotely via SCP (should work with any SSH server)
|
30
|
+
4. Storing remotely in atmos
|
31
|
+
|
32
|
+
Here are the relevant sections that should go either in ~/.gitconfig (for global settings)
|
33
|
+
or in clone/.git/config (for per-repo settings).
|
34
|
+
|
35
|
+
[git-media]
|
36
|
+
transport = <scp|local|s3|atmos>
|
37
|
+
|
38
|
+
# settings for scp transport
|
39
|
+
scpuser=<user>
|
40
|
+
scphost=<host>
|
41
|
+
scppath=<path_on_remote_server>
|
42
|
+
|
43
|
+
# settings for local transport
|
44
|
+
path=<local_filesystem_path>
|
45
|
+
|
46
|
+
# settings for s3 transport
|
47
|
+
s3bucket=<name_of_bucket>
|
48
|
+
s3key=<s3 access key>
|
49
|
+
s3secret=<s3 secret key>
|
50
|
+
|
51
|
+
# settings for atmos transport
|
52
|
+
endpoint=<atmos server>
|
53
|
+
uid=<atmos_uid>
|
54
|
+
secret=<atmos secret key>
|
55
|
+
tag=<atmos object tag>
|
56
|
+
|
57
|
+
|
58
|
+
|
59
|
+
== Usage
|
60
|
+
|
61
|
+
(in repo - repeatedly)
|
62
|
+
$ (hack, stage, commit)
|
63
|
+
$ git media sync
|
64
|
+
|
65
|
+
You can also check the status of your media files via
|
66
|
+
|
67
|
+
$ git media status
|
68
|
+
|
69
|
+
Which will show you files that are waiting to be uploaded and how much data
|
70
|
+
that is. If you want to upload & delete the local cache of media files, run:
|
71
|
+
|
72
|
+
$ git media clear
|
73
|
+
|
74
|
+
== Config Settings
|
75
|
+
|
76
|
+
$ git config --global media.auto-download false
|
77
|
+
|
78
|
+
|
79
|
+
== Installing
|
80
|
+
|
81
|
+
$ sudo gem install trollop
|
82
|
+
$ sudo gem install s3
|
83
|
+
$ sudo gem install ruby-atmos-pure
|
84
|
+
$ sudo gem install right_aws
|
85
|
+
$ gem build git-media.gemspec
|
86
|
+
$ sudo gem install git-media-0.1.1.gem
|
87
|
+
|
88
|
+
== Notes for Windows
|
89
|
+
It is important to switch off git smart newline character support for media files.
|
90
|
+
Use "-crlf" switch in .gitattributes (for example "*.mov filter=media -crlf") or config option "core.autocrlf = false".
|
91
|
+
|
92
|
+
If installing on windows, you might run into a problem verifying certificates
|
93
|
+
for S3 or something. If that happens, modify
|
94
|
+
C:\Ruby191\lib\ruby\gems\1.9.1\gems\right_http_connection-1.2.4\lib\right_http_connection.rb
|
95
|
+
And add at line 310, right before "@http.start":
|
96
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
97
|
+
|
98
|
+
== Copyright
|
99
|
+
|
100
|
+
Copyright (c) 2009 Scott Chacon. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'bundler/gem_tasks'
|
6
|
+
rescue LoadError
|
7
|
+
end
|
8
|
+
|
9
|
+
begin
|
10
|
+
require 'spec/rake/spectask'
|
11
|
+
Spec::Rake::SpecTask.new(:spec) do |spec|
|
12
|
+
spec.libs << 'lib' << 'spec'
|
13
|
+
spec.spec_files = FileList['spec/**/*_spec.rb']
|
14
|
+
end
|
15
|
+
|
16
|
+
Spec::Rake::SpecTask.new(:rcov) do |spec|
|
17
|
+
spec.libs << 'lib' << 'spec'
|
18
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
19
|
+
spec.rcov = true
|
20
|
+
end
|
21
|
+
rescue LoadError
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
task :default => :spec
|
26
|
+
|
27
|
+
require 'rdoc/task'
|
28
|
+
RDoc::Task.new do |rdoc|
|
29
|
+
if File.exist?('VERSION.yml')
|
30
|
+
config = YAML.load(File.read('VERSION.yml'))
|
31
|
+
version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
|
32
|
+
else
|
33
|
+
version = ""
|
34
|
+
end
|
35
|
+
|
36
|
+
rdoc.rdoc_dir = 'rdoc'
|
37
|
+
rdoc.title = "git-media #{version}"
|
38
|
+
rdoc.rdoc_files.include('README*')
|
39
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
40
|
+
end
|
41
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.2
|
data/bin/git-media
ADDED
data/lib/git-media.rb
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
require 'trollop'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'git-media/transport/local'
|
4
|
+
require 'git-media/transport/s3'
|
5
|
+
require 'git-media/transport/scp'
|
6
|
+
|
7
|
+
module GitMedia
|
8
|
+
|
9
|
+
def self.get_media_buffer
|
10
|
+
@@git_dir ||= `git rev-parse --git-dir`.chomp
|
11
|
+
media_buffer = File.join(@@git_dir, 'media/objects')
|
12
|
+
FileUtils.mkdir_p(media_buffer) if !File.exist?(media_buffer)
|
13
|
+
return media_buffer
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.media_path(sha)
|
17
|
+
buf = self.get_media_buffer
|
18
|
+
File.join(buf, sha)
|
19
|
+
end
|
20
|
+
|
21
|
+
# TODO: select the proper transports based on settings
|
22
|
+
def self.get_push_transport
|
23
|
+
self.get_transport
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.get_transport
|
27
|
+
transport = `git config git-media.transport`.chomp
|
28
|
+
case transport
|
29
|
+
when ""
|
30
|
+
raise "git-media.transport not set"
|
31
|
+
when "scp"
|
32
|
+
user = `git config git-media.scpuser`.chomp
|
33
|
+
host = `git config git-media.scphost`.chomp
|
34
|
+
path = `git config git-media.scppath`.chomp
|
35
|
+
port = `git config git-media.scpport`.chomp
|
36
|
+
if user === ""
|
37
|
+
raise "git-media.scpuser not set for scp transport"
|
38
|
+
end
|
39
|
+
if host === ""
|
40
|
+
raise "git-media.scphost not set for scp transport"
|
41
|
+
end
|
42
|
+
if path === ""
|
43
|
+
raise "git-media.scppath not set for scp transport"
|
44
|
+
end
|
45
|
+
GitMedia::Transport::Scp.new(user, host, path, port)
|
46
|
+
|
47
|
+
when "local"
|
48
|
+
path = `git config git-media.localpath`.chomp
|
49
|
+
if path === ""
|
50
|
+
raise "git-media.localpath not set for local transport"
|
51
|
+
end
|
52
|
+
GitMedia::Transport::Local.new(path)
|
53
|
+
when "s3"
|
54
|
+
bucket = `git config git-media.s3bucket`.chomp
|
55
|
+
key = `git config git-media.s3key`.chomp
|
56
|
+
secret = `git config git-media.s3secret`.chomp
|
57
|
+
if bucket === ""
|
58
|
+
raise "git-media.s3bucket not set for s3 transport"
|
59
|
+
end
|
60
|
+
if key === ""
|
61
|
+
raise "git-media.s3key not set for s3 transport"
|
62
|
+
end
|
63
|
+
if secret === ""
|
64
|
+
raise "git-media.s3secret not set for s3 transport"
|
65
|
+
end
|
66
|
+
GitMedia::Transport::S3.new(bucket, key, secret)
|
67
|
+
when "atmos"
|
68
|
+
require 'git-media/transport/atmos_client'
|
69
|
+
endpoint = `git config git-media.endpoint`.chomp
|
70
|
+
uid = `git config git-media.uid`.chomp
|
71
|
+
secret = `git config git-media.secret`.chomp
|
72
|
+
tag = `git config git-media.tag`.chomp
|
73
|
+
|
74
|
+
if endpoint == ""
|
75
|
+
raise "git-media.endpoint not set for atmos transport"
|
76
|
+
end
|
77
|
+
|
78
|
+
if uid == ""
|
79
|
+
raise "git-media.uid not set for atmos transport"
|
80
|
+
end
|
81
|
+
|
82
|
+
if secret == ""
|
83
|
+
raise "git-media.secret not set for atmos transport"
|
84
|
+
end
|
85
|
+
GitMedia::Transport::AtmosClient.new(endpoint, uid, secret, tag)
|
86
|
+
else
|
87
|
+
raise "Invalid transport #{transport}"
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.get_pull_transport
|
92
|
+
self.get_transport
|
93
|
+
end
|
94
|
+
|
95
|
+
module Application
|
96
|
+
def self.run!
|
97
|
+
|
98
|
+
cmd = ARGV.shift # get the subcommand
|
99
|
+
cmd_opts = case cmd
|
100
|
+
when "filter-clean" # parse delete options
|
101
|
+
require 'git-media/filter-clean'
|
102
|
+
GitMedia::FilterClean.run!
|
103
|
+
when "filter-smudge"
|
104
|
+
require 'git-media/filter-smudge'
|
105
|
+
GitMedia::FilterSmudge.run!
|
106
|
+
when "clear" # parse delete options
|
107
|
+
require 'git-media/clear'
|
108
|
+
GitMedia::Clear.run!
|
109
|
+
when "sync"
|
110
|
+
require 'git-media/sync'
|
111
|
+
GitMedia::Sync.run!
|
112
|
+
when 'status'
|
113
|
+
require 'git-media/status'
|
114
|
+
Trollop::options do
|
115
|
+
opt :force, "Force status"
|
116
|
+
end
|
117
|
+
GitMedia::Status.run!
|
118
|
+
else
|
119
|
+
print <<EOF
|
120
|
+
usage: git media sync|status|clear
|
121
|
+
|
122
|
+
sync Sync files with remote server
|
123
|
+
status Show files that are waiting to be uploaded and file size
|
124
|
+
clear Upload and delete the local cache of media files
|
125
|
+
|
126
|
+
EOF
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'git-media/status'
|
2
|
+
|
3
|
+
module GitMedia
|
4
|
+
module Clear
|
5
|
+
|
6
|
+
def self.run!
|
7
|
+
@push = GitMedia.get_push_transport
|
8
|
+
self.clear_local_cache
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.clear_local_cache
|
12
|
+
# find files in media buffer and upload them
|
13
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
14
|
+
unpushed_files = @push.get_unpushed(all_cache)
|
15
|
+
pushed_files = all_cache - unpushed_files
|
16
|
+
pushed_files.each do |sha|
|
17
|
+
puts "removing " + sha[0, 8]
|
18
|
+
File.unlink(File.join(GitMedia.get_media_buffer, sha))
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'digest/sha1'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'tempfile'
|
4
|
+
|
5
|
+
module GitMedia
|
6
|
+
module FilterClean
|
7
|
+
|
8
|
+
def self.run!
|
9
|
+
# determine and initialize our media buffer directory
|
10
|
+
media_buffer = GitMedia.get_media_buffer
|
11
|
+
|
12
|
+
hashfunc = Digest::SHA1.new
|
13
|
+
start = Time.now
|
14
|
+
|
15
|
+
# TODO: read first 41 bytes and see if this is a stub
|
16
|
+
|
17
|
+
# read in buffered chunks of the data
|
18
|
+
# calculating the SHA and copying to a tempfile
|
19
|
+
tempfile = Tempfile.new('media')
|
20
|
+
while data = STDIN.read(4096)
|
21
|
+
hashfunc.update(data)
|
22
|
+
tempfile.write(data)
|
23
|
+
end
|
24
|
+
tempfile.close
|
25
|
+
|
26
|
+
# calculate and print the SHA of the data
|
27
|
+
STDOUT.print hx = hashfunc.hexdigest
|
28
|
+
STDOUT.binmode
|
29
|
+
STDOUT.write("\n")
|
30
|
+
|
31
|
+
# move the tempfile to our media buffer area
|
32
|
+
media_file = File.join(media_buffer, hx)
|
33
|
+
FileUtils.mv(tempfile.path, media_file)
|
34
|
+
|
35
|
+
elapsed = Time.now - start
|
36
|
+
STDERR.puts('Saving media : ' + hx + ' : ' + elapsed.to_s)
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module GitMedia
|
2
|
+
module FilterSmudge
|
3
|
+
|
4
|
+
def self.run!
|
5
|
+
media_buffer = GitMedia.get_media_buffer
|
6
|
+
can_download = false # TODO: read this from config and implement
|
7
|
+
|
8
|
+
# read checksum size
|
9
|
+
sha = STDIN.readline(64).strip # read no more than 64 bytes
|
10
|
+
if STDIN.eof? && sha.length == 40 && sha.match(/^[0-9a-fA-F]+$/) != nil
|
11
|
+
# this is a media file
|
12
|
+
media_file = File.join(media_buffer, sha.chomp)
|
13
|
+
if File.exists?(media_file)
|
14
|
+
STDERR.puts('recovering media : ' + sha)
|
15
|
+
File.open(media_file, 'r') do |f|
|
16
|
+
while data = f.read(4096) do
|
17
|
+
print data
|
18
|
+
end
|
19
|
+
end
|
20
|
+
else
|
21
|
+
# TODO: download file if not in the media buffer area
|
22
|
+
if !can_download
|
23
|
+
STDERR.puts('media missing, saving placeholder : ' + sha)
|
24
|
+
puts sha
|
25
|
+
end
|
26
|
+
end
|
27
|
+
else
|
28
|
+
# if it is not a 40 character long hash, just output
|
29
|
+
STDERR.puts('Unknown git-media file format')
|
30
|
+
print sha
|
31
|
+
while data = STDIN.read(4096)
|
32
|
+
print data
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'pp'
|
2
|
+
|
3
|
+
module GitMedia
|
4
|
+
module Status
|
5
|
+
|
6
|
+
def self.run!
|
7
|
+
@push = GitMedia.get_push_transport
|
8
|
+
r = self.find_references
|
9
|
+
self.print_references(r)
|
10
|
+
r = self.local_cache_status
|
11
|
+
self.print_cache_status(r)
|
12
|
+
end
|
13
|
+
|
14
|
+
# find tree entries that are likely media references
|
15
|
+
def self.find_references
|
16
|
+
references = {:to_expand => [], :expanded => [], :deleted => []}
|
17
|
+
files = `git ls-tree -lz -r HEAD | tr "\\000" \\\\n`.split("\n")
|
18
|
+
files = files.map { |f| s = f.split("\t"); [s[0].split(' ').last, s[1]] }
|
19
|
+
files = files.select { |f| f[0] == '41' } # it's the right size
|
20
|
+
files.each do |tree_size, fname|
|
21
|
+
if size = File.size?(fname)
|
22
|
+
if size == tree_size.to_i
|
23
|
+
# TODO: read in the data and verify that it's a sha + newline
|
24
|
+
sha = File.read(fname).strip
|
25
|
+
if sha.length == 40 && sha =~ /^[0-9a-f]+$/
|
26
|
+
references[:to_expand] << [fname, sha]
|
27
|
+
end
|
28
|
+
else
|
29
|
+
references[:expanded] << fname
|
30
|
+
end
|
31
|
+
else
|
32
|
+
# file was deleted
|
33
|
+
references[:deleted] << fname
|
34
|
+
end
|
35
|
+
end
|
36
|
+
references
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.print_references(refs)
|
40
|
+
if refs[:to_expand].size > 0
|
41
|
+
puts "== Unexpanded Media =="
|
42
|
+
refs[:to_expand].each do |file, sha|
|
43
|
+
puts " " + sha[0, 8] + " " + file
|
44
|
+
end
|
45
|
+
puts
|
46
|
+
end
|
47
|
+
if refs[:expanded].size > 0
|
48
|
+
puts "== Expanded Media =="
|
49
|
+
refs[:expanded].each do |file|
|
50
|
+
size = File.size(file)
|
51
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + " #{file}"
|
52
|
+
end
|
53
|
+
puts
|
54
|
+
end
|
55
|
+
if refs[:deleted].size > 0
|
56
|
+
puts "== Deleted Media =="
|
57
|
+
refs[:deleted].each do |file|
|
58
|
+
puts " " + " #{file}"
|
59
|
+
end
|
60
|
+
puts
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.print_cache_status(refs)
|
65
|
+
if refs[:unpushed].size > 0
|
66
|
+
puts "== Unpushed Media =="
|
67
|
+
refs[:unpushed].each do |sha|
|
68
|
+
cache_file = GitMedia.media_path(sha)
|
69
|
+
size = File.size(cache_file)
|
70
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
|
71
|
+
end
|
72
|
+
puts
|
73
|
+
end
|
74
|
+
if refs[:pushed].size > 0
|
75
|
+
puts "== Already Pushed Media =="
|
76
|
+
refs[:pushed].each do |sha|
|
77
|
+
cache_file = GitMedia.media_path(sha)
|
78
|
+
size = File.size(cache_file)
|
79
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
|
80
|
+
end
|
81
|
+
puts
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.local_cache_status
|
86
|
+
# find files in media buffer and upload them
|
87
|
+
references = {:unpushed => [], :pushed => []}
|
88
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
89
|
+
unpushed_files = @push.get_unpushed(all_cache) || []
|
90
|
+
references[:unpushed] = unpushed_files
|
91
|
+
references[:pushed] = all_cache - unpushed_files rescue []
|
92
|
+
references
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
def self.to_human(size)
|
97
|
+
if size < 1024
|
98
|
+
return size.to_s + 'b'
|
99
|
+
elsif size < 1048576
|
100
|
+
return (size / 1024).to_s + 'k'
|
101
|
+
else
|
102
|
+
return (size / 1048576).to_s + 'm'
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# find files that are placeholders (41 char) and download them
|
2
|
+
# upload files in media buffer that are not in offsite bin
|
3
|
+
require 'git-media/status'
|
4
|
+
|
5
|
+
module GitMedia
|
6
|
+
module Sync
|
7
|
+
|
8
|
+
def self.run!
|
9
|
+
@push = GitMedia.get_push_transport
|
10
|
+
@pull = GitMedia.get_pull_transport
|
11
|
+
|
12
|
+
self.expand_references
|
13
|
+
self.upload_local_cache
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.expand_references
|
17
|
+
status = GitMedia::Status.find_references
|
18
|
+
status[:to_expand].each do |file, sha|
|
19
|
+
cache_file = GitMedia.media_path(sha)
|
20
|
+
if !File.exist?(cache_file)
|
21
|
+
puts "Downloading " + sha[0,8] + " : " + file
|
22
|
+
@pull.pull(file, sha)
|
23
|
+
end
|
24
|
+
|
25
|
+
puts "Expanding " + sha[0,8] + " : " + file
|
26
|
+
|
27
|
+
if File.exist?(cache_file)
|
28
|
+
FileUtils.cp(cache_file, file)
|
29
|
+
else
|
30
|
+
puts 'could not get media'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.upload_local_cache
|
36
|
+
# find files in media buffer and upload them
|
37
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
38
|
+
unpushed_files = @push.get_unpushed(all_cache)
|
39
|
+
unpushed_files.each do |sha|
|
40
|
+
puts 'uploading ' + sha[0, 8]
|
41
|
+
@push.push(sha)
|
42
|
+
end
|
43
|
+
# TODO: if --clean, remove them
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module GitMedia
|
2
|
+
module Transport
|
3
|
+
class Base
|
4
|
+
|
5
|
+
def pull(final_file, sha)
|
6
|
+
to_file = GitMedia.media_path(sha)
|
7
|
+
get_file(sha, to_file)
|
8
|
+
end
|
9
|
+
|
10
|
+
def push(sha)
|
11
|
+
from_file = GitMedia.media_path(sha)
|
12
|
+
put_file(sha, from_file)
|
13
|
+
end
|
14
|
+
|
15
|
+
## OVERWRITE ##
|
16
|
+
|
17
|
+
def read?
|
18
|
+
false
|
19
|
+
end
|
20
|
+
|
21
|
+
def write?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_file(sha, to_file)
|
26
|
+
false
|
27
|
+
end
|
28
|
+
|
29
|
+
def put_file(sha, to_file)
|
30
|
+
false
|
31
|
+
end
|
32
|
+
|
33
|
+
def get_unpushed(files)
|
34
|
+
files
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
require 'atmos'
|
3
|
+
|
4
|
+
# git-media.transport atmos
|
5
|
+
# git-media.endpoint
|
6
|
+
# git-media.uid
|
7
|
+
# git-media.secret
|
8
|
+
# git-media.tag (optional)
|
9
|
+
|
10
|
+
module GitMedia
|
11
|
+
module Transport
|
12
|
+
class AtmosClient < Base
|
13
|
+
|
14
|
+
def initialize(endpoint, uid, secret, tag)
|
15
|
+
atmos_options = {
|
16
|
+
:url => endpoint,
|
17
|
+
:uid => uid,
|
18
|
+
:secret => secret
|
19
|
+
}
|
20
|
+
@tag = tag
|
21
|
+
@atmos_client = Atmos::Store.new(atmos_options)
|
22
|
+
end
|
23
|
+
|
24
|
+
def read?
|
25
|
+
reachable?
|
26
|
+
end
|
27
|
+
|
28
|
+
def get_file(sha, to_file)
|
29
|
+
dst_file = File.new(to_file, File::CREAT|File::RDWR)
|
30
|
+
@atmos_client.get(:namespace => sha).data_as_stream do |chunck|
|
31
|
+
dst_file.write(chunck)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def write
|
36
|
+
reachable?
|
37
|
+
end
|
38
|
+
|
39
|
+
def put_file(sha, from_file)
|
40
|
+
src_file = File.open(from_file)
|
41
|
+
obj_conf = {:data => src_file, :length => File.size(from_file), :namespace => sha}
|
42
|
+
obj_conf[:listable_metadata] = {@tag => true} if @tag
|
43
|
+
@atmos_client.create(obj_conf)
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_unpushed(files)
|
47
|
+
unpushed = []
|
48
|
+
files.each do |file|
|
49
|
+
begin
|
50
|
+
@atmos_client.get(:namespace => file)
|
51
|
+
rescue Atmos::Exceptions::AtmosException
|
52
|
+
unpushed << file
|
53
|
+
end
|
54
|
+
end
|
55
|
+
unpushed
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
# dummy function to test connectivity to atmos
|
60
|
+
def reachable?
|
61
|
+
@atmos_client.server_version
|
62
|
+
true
|
63
|
+
rescue
|
64
|
+
false
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
|
3
|
+
# move large media to local bin
|
4
|
+
|
5
|
+
# git-media.transport local
|
6
|
+
# git-media.localpath /opt/media
|
7
|
+
|
8
|
+
module GitMedia
|
9
|
+
module Transport
|
10
|
+
class Local < Base
|
11
|
+
|
12
|
+
def initialize(path)
|
13
|
+
@path = path
|
14
|
+
end
|
15
|
+
|
16
|
+
def read?
|
17
|
+
File.exist?(@path)
|
18
|
+
end
|
19
|
+
|
20
|
+
def get_file(sha, to_file)
|
21
|
+
from_file = File.join(@path, sha)
|
22
|
+
if File.exists?(from_file)
|
23
|
+
FileUtils.cp(from_file, to_file)
|
24
|
+
return true
|
25
|
+
end
|
26
|
+
return false
|
27
|
+
end
|
28
|
+
|
29
|
+
def write?
|
30
|
+
File.exist?(@path)
|
31
|
+
end
|
32
|
+
|
33
|
+
def put_file(sha, from_file)
|
34
|
+
to_file = File.join(@path, sha)
|
35
|
+
if File.exists?(from_file)
|
36
|
+
FileUtils.cp(from_file, to_file)
|
37
|
+
return true
|
38
|
+
end
|
39
|
+
return false
|
40
|
+
end
|
41
|
+
|
42
|
+
def get_unpushed(files)
|
43
|
+
files.select do |f|
|
44
|
+
!File.exist?(File.join(@path, f))
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
require 'right_aws'
|
3
|
+
|
4
|
+
# git-media.transport s3
|
5
|
+
# git-media.s3bucket
|
6
|
+
# git-media.s3key
|
7
|
+
# git-media.s3secret
|
8
|
+
|
9
|
+
module GitMedia
|
10
|
+
module Transport
|
11
|
+
class S3 < Base
|
12
|
+
|
13
|
+
def initialize(bucket, access_key_id = nil, secret_access_key = nil)
|
14
|
+
@s3 = RightAws::S3Interface.new(access_key_id, secret_access_key,
|
15
|
+
{:multi_thread => true, :logger => Logger.new('/tmp/s3.log')})
|
16
|
+
@bucket = bucket
|
17
|
+
@buckets = @s3.list_all_my_buckets.map { |a| a[:name] }
|
18
|
+
if !@buckets.include?(bucket)
|
19
|
+
puts "Creating New Bucket"
|
20
|
+
if @s3.create_bucket(bucket)
|
21
|
+
@buckets << bucket
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def read?
|
27
|
+
@buckets.size > 0
|
28
|
+
end
|
29
|
+
|
30
|
+
def get_file(sha, to_file)
|
31
|
+
to = File.new(to_file, File::CREAT|File::RDWR)
|
32
|
+
@s3.get(@bucket, sha) do |chunk|
|
33
|
+
to.write(chunk)
|
34
|
+
end
|
35
|
+
to.close
|
36
|
+
end
|
37
|
+
|
38
|
+
def write?
|
39
|
+
@buckets.size > 0
|
40
|
+
end
|
41
|
+
|
42
|
+
def put_file(sha, from_file)
|
43
|
+
@s3.put(@bucket, sha, File.open(from_file))
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_unpushed(files)
|
47
|
+
keys = @s3.list_bucket(@bucket).map { |f| f[:key] }
|
48
|
+
files.select do |f|
|
49
|
+
!keys.include?(f)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
|
3
|
+
# move large media to remote server via SCP
|
4
|
+
|
5
|
+
# git-media.transport scp
|
6
|
+
# git-media.scpuser someuser
|
7
|
+
# git-media.scphost remoteserver.com
|
8
|
+
# git-media.scppath /opt/media
|
9
|
+
|
10
|
+
module GitMedia
|
11
|
+
module Transport
|
12
|
+
class Scp < Base
|
13
|
+
|
14
|
+
def initialize(user, host, path, port)
|
15
|
+
@user = user
|
16
|
+
@host = host
|
17
|
+
@path = path
|
18
|
+
unless port === ""
|
19
|
+
@sshport = "-p#{port}"
|
20
|
+
end
|
21
|
+
unless port === ""
|
22
|
+
@scpport = "-P#{port}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def exist?(file)
|
27
|
+
@file = file
|
28
|
+
|
29
|
+
if Integer(result) == 1
|
30
|
+
puts file + " exists"
|
31
|
+
return true
|
32
|
+
else
|
33
|
+
puts file + " doesn't exists"
|
34
|
+
return false
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def result
|
39
|
+
`ssh #{@user}@#{@host} #{@sshport} [ -f "#{@file}" ] && echo 1 || echo 0`
|
40
|
+
end
|
41
|
+
|
42
|
+
def read?
|
43
|
+
return true
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_file(sha, to_file)
|
47
|
+
from_file = @user+"@"+@host+":"+File.join(@path, sha)
|
48
|
+
`scp #{@scpport} "#{from_file}" "#{to_file}"`
|
49
|
+
if $? == 0
|
50
|
+
puts sha+" downloaded"
|
51
|
+
return true
|
52
|
+
end
|
53
|
+
puts sha+" download fail"
|
54
|
+
return false
|
55
|
+
end
|
56
|
+
|
57
|
+
def write?
|
58
|
+
return true
|
59
|
+
end
|
60
|
+
|
61
|
+
def put_file(sha, from_file)
|
62
|
+
to_file = @user+"@"+@host+":"+File.join(@path, sha)
|
63
|
+
`scp #{@scpport} "#{from_file}" "#{to_file}"`
|
64
|
+
if $? == 0
|
65
|
+
puts sha+" uploaded"
|
66
|
+
return true
|
67
|
+
end
|
68
|
+
puts sha+" upload fail"
|
69
|
+
return false
|
70
|
+
end
|
71
|
+
|
72
|
+
def get_unpushed(files)
|
73
|
+
files.select do |f|
|
74
|
+
!self.exist?(File.join(@path, f))
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{notepunch-git-media}
|
5
|
+
s.version = "0.1.3"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Scott Chacon"]
|
9
|
+
s.date = %q{2009-06-10}
|
10
|
+
s.default_executable = %q{git-media}
|
11
|
+
s.email = %q{schacon@gmail.com}
|
12
|
+
s.executables = ["git-media"]
|
13
|
+
s.extra_rdoc_files = [
|
14
|
+
"LICENSE",
|
15
|
+
"README.rdoc"
|
16
|
+
]
|
17
|
+
s.files = [
|
18
|
+
".document",
|
19
|
+
".gitignore",
|
20
|
+
"LICENSE",
|
21
|
+
"README.rdoc",
|
22
|
+
"Rakefile",
|
23
|
+
"VERSION",
|
24
|
+
"bin/git-media",
|
25
|
+
"notepunch-git-media.gemspec",
|
26
|
+
"lib/git-media/clear.rb",
|
27
|
+
"lib/git-media/filter-clean.rb",
|
28
|
+
"lib/git-media/filter-smudge.rb",
|
29
|
+
"lib/git-media/status.rb",
|
30
|
+
"lib/git-media/sync.rb",
|
31
|
+
"lib/git-media/transport",
|
32
|
+
"lib/git-media/transport/local.rb",
|
33
|
+
"lib/git-media/transport/s3.rb",
|
34
|
+
"lib/git-media/transport/atmos_client.rb",
|
35
|
+
"lib/git-media/transport/scp.rb",
|
36
|
+
"lib/git-media/transport.rb",
|
37
|
+
"lib/git-media.rb"
|
38
|
+
]
|
39
|
+
s.has_rdoc = true
|
40
|
+
s.homepage = %q{http://github.com/schacon/git-media}
|
41
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
42
|
+
s.require_paths = ["lib"]
|
43
|
+
s.rubygems_version = %q{1.3.1}
|
44
|
+
s.summary = %q{"This is a summary! Stop yer whining"}
|
45
|
+
|
46
|
+
if s.respond_to? :specification_version then
|
47
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
48
|
+
s.specification_version = 2
|
49
|
+
|
50
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
51
|
+
else
|
52
|
+
end
|
53
|
+
else
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
metadata
ADDED
@@ -0,0 +1,65 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: notepunch-git-media
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Scott Chacon
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2009-06-10 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description:
|
14
|
+
email: schacon@gmail.com
|
15
|
+
executables:
|
16
|
+
- git-media
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files:
|
19
|
+
- LICENSE
|
20
|
+
- README.rdoc
|
21
|
+
files:
|
22
|
+
- .document
|
23
|
+
- .gitignore
|
24
|
+
- LICENSE
|
25
|
+
- README.rdoc
|
26
|
+
- Rakefile
|
27
|
+
- VERSION
|
28
|
+
- bin/git-media
|
29
|
+
- notepunch-git-media.gemspec
|
30
|
+
- lib/git-media/clear.rb
|
31
|
+
- lib/git-media/filter-clean.rb
|
32
|
+
- lib/git-media/filter-smudge.rb
|
33
|
+
- lib/git-media/status.rb
|
34
|
+
- lib/git-media/sync.rb
|
35
|
+
- lib/git-media/transport/local.rb
|
36
|
+
- lib/git-media/transport/s3.rb
|
37
|
+
- lib/git-media/transport/atmos_client.rb
|
38
|
+
- lib/git-media/transport/scp.rb
|
39
|
+
- lib/git-media/transport.rb
|
40
|
+
- lib/git-media.rb
|
41
|
+
homepage: http://github.com/schacon/git-media
|
42
|
+
licenses: []
|
43
|
+
metadata: {}
|
44
|
+
post_install_message:
|
45
|
+
rdoc_options:
|
46
|
+
- --charset=UTF-8
|
47
|
+
require_paths:
|
48
|
+
- lib
|
49
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
55
|
+
requirements:
|
56
|
+
- - ! '>='
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: '0'
|
59
|
+
requirements: []
|
60
|
+
rubyforge_project:
|
61
|
+
rubygems_version: 2.0.0
|
62
|
+
signing_key:
|
63
|
+
specification_version: 2
|
64
|
+
summary: ! '"This is a summary! Stop yer whining"'
|
65
|
+
test_files: []
|