notepunch-git-media 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.document +5 -0
- data/.gitignore +5 -0
- data/LICENSE +20 -0
- data/README.rdoc +100 -0
- data/Rakefile +41 -0
- data/VERSION +1 -0
- data/bin/git-media +8 -0
- data/lib/git-media.rb +131 -0
- data/lib/git-media/clear.rb +23 -0
- data/lib/git-media/filter-clean.rb +40 -0
- data/lib/git-media/filter-smudge.rb +38 -0
- data/lib/git-media/status.rb +107 -0
- data/lib/git-media/sync.rb +47 -0
- data/lib/git-media/transport.rb +39 -0
- data/lib/git-media/transport/atmos_client.rb +71 -0
- data/lib/git-media/transport/local.rb +50 -0
- data/lib/git-media/transport/s3.rb +55 -0
- data/lib/git-media/transport/scp.rb +80 -0
- data/notepunch-git-media.gemspec +56 -0
- metadata +65 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NmU1NjE4MTY5ZTU1ZDg0YTEzNjE3NmI5ZmZkYmYzNzQ4NzA4NjQ1Zg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
YjE5ODM1YzkyNmM4MTAzY2YwOTNhYWMwNDk1NWFmMTRjNDQwMjdmYw==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
NDJjODhmZmRmMDVmNzNhNTMzMWI5MGJmM2M2M2YzMGJhYTQyNjQ0NTQ0ZTcx
|
10
|
+
ZWMxYWQyOWJmNmU0M2RjMmYzZTU4N2UwMDdmYTgxNGYwZTlkNjc2MzBmNGE4
|
11
|
+
MGJhMWJjZmM4ZDY3M2FmYzkwZTZiOTllNTk3MzEzNWFlYTYwZDc=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
NmMwZmQyZmFlY2MyZThkYjcxZmYzZTYzNzliMjBkYWY5YWUzNTNmNTY5ZDZj
|
14
|
+
MDRkYWY3ZTkxYjRkNTQ4NTEyZjhmNzhiY2RiZTY1YmNkYjZiNjZlMmY5ZDc3
|
15
|
+
YjZjNTNhNDA3NjRkZGExOGU3ZTI3NjQxMTdkZjAxOWQ5OTAxNDk=
|
data/.document
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Scott Chacon
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
= git-media
|
2
|
+
|
3
|
+
GitMedia extension allows you to use Git with large media files
|
4
|
+
without storing the media in Git itself.
|
5
|
+
|
6
|
+
== Configuration
|
7
|
+
|
8
|
+
Setup the attributes filter settings.
|
9
|
+
|
10
|
+
(once after install)
|
11
|
+
$ git config filter.media.clean "git-media filter-clean"
|
12
|
+
$ git config filter.media.smudge "git-media filter-smudge"
|
13
|
+
|
14
|
+
Setup the .gitattributes file to map extensions to the filter.
|
15
|
+
|
16
|
+
(in repo - once)
|
17
|
+
$ echo "*.mov filter=media -crlf" > .gitattributes
|
18
|
+
|
19
|
+
Staging files with those extensions will automatically copy them to the
|
20
|
+
media buffer area (.git/media) until you run 'git media sync' wherein they
|
21
|
+
are uploaded. Checkouts that reference media you don't have yet will try to
|
22
|
+
be automatically downloaded, otherwise they are downloaded when you sync.
|
23
|
+
|
24
|
+
Next you need to configure git to tell it where you want to store the large files.
|
25
|
+
There are four options:
|
26
|
+
|
27
|
+
1. Storing remotely in Amazon's S3
|
28
|
+
2. Storing locally in a filesystem path
|
29
|
+
3. Storing remotely via SCP (should work with any SSH server)
|
30
|
+
4. Storing remotely in atmos
|
31
|
+
|
32
|
+
Here are the relevant sections that should go either in ~/.gitconfig (for global settings)
|
33
|
+
or in clone/.git/config (for per-repo settings).
|
34
|
+
|
35
|
+
[git-media]
|
36
|
+
transport = <scp|local|s3|atmos>
|
37
|
+
|
38
|
+
# settings for scp transport
|
39
|
+
scpuser=<user>
|
40
|
+
scphost=<host>
|
41
|
+
scppath=<path_on_remote_server>
|
42
|
+
|
43
|
+
# settings for local transport
|
44
|
+
path=<local_filesystem_path>
|
45
|
+
|
46
|
+
# settings for s3 transport
|
47
|
+
s3bucket=<name_of_bucket>
|
48
|
+
s3key=<s3 access key>
|
49
|
+
s3secret=<s3 secret key>
|
50
|
+
|
51
|
+
# settings for atmos transport
|
52
|
+
endpoint=<atmos server>
|
53
|
+
uid=<atmos_uid>
|
54
|
+
secret=<atmos secret key>
|
55
|
+
tag=<atmos object tag>
|
56
|
+
|
57
|
+
|
58
|
+
|
59
|
+
== Usage
|
60
|
+
|
61
|
+
(in repo - repeatedly)
|
62
|
+
$ (hack, stage, commit)
|
63
|
+
$ git media sync
|
64
|
+
|
65
|
+
You can also check the status of your media files via
|
66
|
+
|
67
|
+
$ git media status
|
68
|
+
|
69
|
+
Which will show you files that are waiting to be uploaded and how much data
|
70
|
+
that is. If you want to upload & delete the local cache of media files, run:
|
71
|
+
|
72
|
+
$ git media clear
|
73
|
+
|
74
|
+
== Config Settings
|
75
|
+
|
76
|
+
$ git config --global media.auto-download false
|
77
|
+
|
78
|
+
|
79
|
+
== Installing
|
80
|
+
|
81
|
+
$ sudo gem install trollop
|
82
|
+
$ sudo gem install s3
|
83
|
+
$ sudo gem install ruby-atmos-pure
|
84
|
+
$ sudo gem install right_aws
|
85
|
+
$ gem build git-media.gemspec
|
86
|
+
$ sudo gem install git-media-0.1.1.gem
|
87
|
+
|
88
|
+
== Notes for Windows
|
89
|
+
It is important to switch off git smart newline character support for media files.
|
90
|
+
Use "-crlf" switch in .gitattributes (for example "*.mov filter=media -crlf") or config option "core.autocrlf = false".
|
91
|
+
|
92
|
+
If installing on windows, you might run into a problem verifying certificates
|
93
|
+
for S3 or something. If that happens, modify
|
94
|
+
C:\Ruby191\lib\ruby\gems\1.9.1\gems\right_http_connection-1.2.4\lib\right_http_connection.rb
|
95
|
+
And add at line 310, right before "@http.start":
|
96
|
+
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
97
|
+
|
98
|
+
== Copyright
|
99
|
+
|
100
|
+
Copyright (c) 2009 Scott Chacon. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'bundler/gem_tasks'
|
6
|
+
rescue LoadError
|
7
|
+
end
|
8
|
+
|
9
|
+
begin
|
10
|
+
require 'spec/rake/spectask'
|
11
|
+
Spec::Rake::SpecTask.new(:spec) do |spec|
|
12
|
+
spec.libs << 'lib' << 'spec'
|
13
|
+
spec.spec_files = FileList['spec/**/*_spec.rb']
|
14
|
+
end
|
15
|
+
|
16
|
+
Spec::Rake::SpecTask.new(:rcov) do |spec|
|
17
|
+
spec.libs << 'lib' << 'spec'
|
18
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
19
|
+
spec.rcov = true
|
20
|
+
end
|
21
|
+
rescue LoadError
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
task :default => :spec
|
26
|
+
|
27
|
+
require 'rdoc/task'
|
28
|
+
RDoc::Task.new do |rdoc|
|
29
|
+
if File.exist?('VERSION.yml')
|
30
|
+
config = YAML.load(File.read('VERSION.yml'))
|
31
|
+
version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
|
32
|
+
else
|
33
|
+
version = ""
|
34
|
+
end
|
35
|
+
|
36
|
+
rdoc.rdoc_dir = 'rdoc'
|
37
|
+
rdoc.title = "git-media #{version}"
|
38
|
+
rdoc.rdoc_files.include('README*')
|
39
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
40
|
+
end
|
41
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.2
|
data/bin/git-media
ADDED
data/lib/git-media.rb
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
require 'trollop'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'git-media/transport/local'
|
4
|
+
require 'git-media/transport/s3'
|
5
|
+
require 'git-media/transport/scp'
|
6
|
+
|
7
|
+
module GitMedia
|
8
|
+
|
9
|
+
def self.get_media_buffer
|
10
|
+
@@git_dir ||= `git rev-parse --git-dir`.chomp
|
11
|
+
media_buffer = File.join(@@git_dir, 'media/objects')
|
12
|
+
FileUtils.mkdir_p(media_buffer) if !File.exist?(media_buffer)
|
13
|
+
return media_buffer
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.media_path(sha)
|
17
|
+
buf = self.get_media_buffer
|
18
|
+
File.join(buf, sha)
|
19
|
+
end
|
20
|
+
|
21
|
+
# TODO: select the proper transports based on settings
|
22
|
+
def self.get_push_transport
|
23
|
+
self.get_transport
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.get_transport
|
27
|
+
transport = `git config git-media.transport`.chomp
|
28
|
+
case transport
|
29
|
+
when ""
|
30
|
+
raise "git-media.transport not set"
|
31
|
+
when "scp"
|
32
|
+
user = `git config git-media.scpuser`.chomp
|
33
|
+
host = `git config git-media.scphost`.chomp
|
34
|
+
path = `git config git-media.scppath`.chomp
|
35
|
+
port = `git config git-media.scpport`.chomp
|
36
|
+
if user === ""
|
37
|
+
raise "git-media.scpuser not set for scp transport"
|
38
|
+
end
|
39
|
+
if host === ""
|
40
|
+
raise "git-media.scphost not set for scp transport"
|
41
|
+
end
|
42
|
+
if path === ""
|
43
|
+
raise "git-media.scppath not set for scp transport"
|
44
|
+
end
|
45
|
+
GitMedia::Transport::Scp.new(user, host, path, port)
|
46
|
+
|
47
|
+
when "local"
|
48
|
+
path = `git config git-media.localpath`.chomp
|
49
|
+
if path === ""
|
50
|
+
raise "git-media.localpath not set for local transport"
|
51
|
+
end
|
52
|
+
GitMedia::Transport::Local.new(path)
|
53
|
+
when "s3"
|
54
|
+
bucket = `git config git-media.s3bucket`.chomp
|
55
|
+
key = `git config git-media.s3key`.chomp
|
56
|
+
secret = `git config git-media.s3secret`.chomp
|
57
|
+
if bucket === ""
|
58
|
+
raise "git-media.s3bucket not set for s3 transport"
|
59
|
+
end
|
60
|
+
if key === ""
|
61
|
+
raise "git-media.s3key not set for s3 transport"
|
62
|
+
end
|
63
|
+
if secret === ""
|
64
|
+
raise "git-media.s3secret not set for s3 transport"
|
65
|
+
end
|
66
|
+
GitMedia::Transport::S3.new(bucket, key, secret)
|
67
|
+
when "atmos"
|
68
|
+
require 'git-media/transport/atmos_client'
|
69
|
+
endpoint = `git config git-media.endpoint`.chomp
|
70
|
+
uid = `git config git-media.uid`.chomp
|
71
|
+
secret = `git config git-media.secret`.chomp
|
72
|
+
tag = `git config git-media.tag`.chomp
|
73
|
+
|
74
|
+
if endpoint == ""
|
75
|
+
raise "git-media.endpoint not set for atmos transport"
|
76
|
+
end
|
77
|
+
|
78
|
+
if uid == ""
|
79
|
+
raise "git-media.uid not set for atmos transport"
|
80
|
+
end
|
81
|
+
|
82
|
+
if secret == ""
|
83
|
+
raise "git-media.secret not set for atmos transport"
|
84
|
+
end
|
85
|
+
GitMedia::Transport::AtmosClient.new(endpoint, uid, secret, tag)
|
86
|
+
else
|
87
|
+
raise "Invalid transport #{transport}"
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.get_pull_transport
|
92
|
+
self.get_transport
|
93
|
+
end
|
94
|
+
|
95
|
+
module Application
|
96
|
+
def self.run!
|
97
|
+
|
98
|
+
cmd = ARGV.shift # get the subcommand
|
99
|
+
cmd_opts = case cmd
|
100
|
+
when "filter-clean" # parse delete options
|
101
|
+
require 'git-media/filter-clean'
|
102
|
+
GitMedia::FilterClean.run!
|
103
|
+
when "filter-smudge"
|
104
|
+
require 'git-media/filter-smudge'
|
105
|
+
GitMedia::FilterSmudge.run!
|
106
|
+
when "clear" # parse delete options
|
107
|
+
require 'git-media/clear'
|
108
|
+
GitMedia::Clear.run!
|
109
|
+
when "sync"
|
110
|
+
require 'git-media/sync'
|
111
|
+
GitMedia::Sync.run!
|
112
|
+
when 'status'
|
113
|
+
require 'git-media/status'
|
114
|
+
Trollop::options do
|
115
|
+
opt :force, "Force status"
|
116
|
+
end
|
117
|
+
GitMedia::Status.run!
|
118
|
+
else
|
119
|
+
print <<EOF
|
120
|
+
usage: git media sync|status|clear
|
121
|
+
|
122
|
+
sync Sync files with remote server
|
123
|
+
status Show files that are waiting to be uploaded and file size
|
124
|
+
clear Upload and delete the local cache of media files
|
125
|
+
|
126
|
+
EOF
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'git-media/status'
|
2
|
+
|
3
|
+
module GitMedia
|
4
|
+
module Clear
|
5
|
+
|
6
|
+
def self.run!
|
7
|
+
@push = GitMedia.get_push_transport
|
8
|
+
self.clear_local_cache
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.clear_local_cache
|
12
|
+
# find files in media buffer and upload them
|
13
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
14
|
+
unpushed_files = @push.get_unpushed(all_cache)
|
15
|
+
pushed_files = all_cache - unpushed_files
|
16
|
+
pushed_files.each do |sha|
|
17
|
+
puts "removing " + sha[0, 8]
|
18
|
+
File.unlink(File.join(GitMedia.get_media_buffer, sha))
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'digest/sha1'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'tempfile'
|
4
|
+
|
5
|
+
module GitMedia
|
6
|
+
module FilterClean
|
7
|
+
|
8
|
+
def self.run!
|
9
|
+
# determine and initialize our media buffer directory
|
10
|
+
media_buffer = GitMedia.get_media_buffer
|
11
|
+
|
12
|
+
hashfunc = Digest::SHA1.new
|
13
|
+
start = Time.now
|
14
|
+
|
15
|
+
# TODO: read first 41 bytes and see if this is a stub
|
16
|
+
|
17
|
+
# read in buffered chunks of the data
|
18
|
+
# calculating the SHA and copying to a tempfile
|
19
|
+
tempfile = Tempfile.new('media')
|
20
|
+
while data = STDIN.read(4096)
|
21
|
+
hashfunc.update(data)
|
22
|
+
tempfile.write(data)
|
23
|
+
end
|
24
|
+
tempfile.close
|
25
|
+
|
26
|
+
# calculate and print the SHA of the data
|
27
|
+
STDOUT.print hx = hashfunc.hexdigest
|
28
|
+
STDOUT.binmode
|
29
|
+
STDOUT.write("\n")
|
30
|
+
|
31
|
+
# move the tempfile to our media buffer area
|
32
|
+
media_file = File.join(media_buffer, hx)
|
33
|
+
FileUtils.mv(tempfile.path, media_file)
|
34
|
+
|
35
|
+
elapsed = Time.now - start
|
36
|
+
STDERR.puts('Saving media : ' + hx + ' : ' + elapsed.to_s)
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module GitMedia
|
2
|
+
module FilterSmudge
|
3
|
+
|
4
|
+
def self.run!
|
5
|
+
media_buffer = GitMedia.get_media_buffer
|
6
|
+
can_download = false # TODO: read this from config and implement
|
7
|
+
|
8
|
+
# read checksum size
|
9
|
+
sha = STDIN.readline(64).strip # read no more than 64 bytes
|
10
|
+
if STDIN.eof? && sha.length == 40 && sha.match(/^[0-9a-fA-F]+$/) != nil
|
11
|
+
# this is a media file
|
12
|
+
media_file = File.join(media_buffer, sha.chomp)
|
13
|
+
if File.exists?(media_file)
|
14
|
+
STDERR.puts('recovering media : ' + sha)
|
15
|
+
File.open(media_file, 'r') do |f|
|
16
|
+
while data = f.read(4096) do
|
17
|
+
print data
|
18
|
+
end
|
19
|
+
end
|
20
|
+
else
|
21
|
+
# TODO: download file if not in the media buffer area
|
22
|
+
if !can_download
|
23
|
+
STDERR.puts('media missing, saving placeholder : ' + sha)
|
24
|
+
puts sha
|
25
|
+
end
|
26
|
+
end
|
27
|
+
else
|
28
|
+
# if it is not a 40 character long hash, just output
|
29
|
+
STDERR.puts('Unknown git-media file format')
|
30
|
+
print sha
|
31
|
+
while data = STDIN.read(4096)
|
32
|
+
print data
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'pp'
|
2
|
+
|
3
|
+
module GitMedia
|
4
|
+
module Status
|
5
|
+
|
6
|
+
def self.run!
|
7
|
+
@push = GitMedia.get_push_transport
|
8
|
+
r = self.find_references
|
9
|
+
self.print_references(r)
|
10
|
+
r = self.local_cache_status
|
11
|
+
self.print_cache_status(r)
|
12
|
+
end
|
13
|
+
|
14
|
+
# find tree entries that are likely media references
|
15
|
+
def self.find_references
|
16
|
+
references = {:to_expand => [], :expanded => [], :deleted => []}
|
17
|
+
files = `git ls-tree -lz -r HEAD | tr "\\000" \\\\n`.split("\n")
|
18
|
+
files = files.map { |f| s = f.split("\t"); [s[0].split(' ').last, s[1]] }
|
19
|
+
files = files.select { |f| f[0] == '41' } # it's the right size
|
20
|
+
files.each do |tree_size, fname|
|
21
|
+
if size = File.size?(fname)
|
22
|
+
if size == tree_size.to_i
|
23
|
+
# TODO: read in the data and verify that it's a sha + newline
|
24
|
+
sha = File.read(fname).strip
|
25
|
+
if sha.length == 40 && sha =~ /^[0-9a-f]+$/
|
26
|
+
references[:to_expand] << [fname, sha]
|
27
|
+
end
|
28
|
+
else
|
29
|
+
references[:expanded] << fname
|
30
|
+
end
|
31
|
+
else
|
32
|
+
# file was deleted
|
33
|
+
references[:deleted] << fname
|
34
|
+
end
|
35
|
+
end
|
36
|
+
references
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.print_references(refs)
|
40
|
+
if refs[:to_expand].size > 0
|
41
|
+
puts "== Unexpanded Media =="
|
42
|
+
refs[:to_expand].each do |file, sha|
|
43
|
+
puts " " + sha[0, 8] + " " + file
|
44
|
+
end
|
45
|
+
puts
|
46
|
+
end
|
47
|
+
if refs[:expanded].size > 0
|
48
|
+
puts "== Expanded Media =="
|
49
|
+
refs[:expanded].each do |file|
|
50
|
+
size = File.size(file)
|
51
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + " #{file}"
|
52
|
+
end
|
53
|
+
puts
|
54
|
+
end
|
55
|
+
if refs[:deleted].size > 0
|
56
|
+
puts "== Deleted Media =="
|
57
|
+
refs[:deleted].each do |file|
|
58
|
+
puts " " + " #{file}"
|
59
|
+
end
|
60
|
+
puts
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.print_cache_status(refs)
|
65
|
+
if refs[:unpushed].size > 0
|
66
|
+
puts "== Unpushed Media =="
|
67
|
+
refs[:unpushed].each do |sha|
|
68
|
+
cache_file = GitMedia.media_path(sha)
|
69
|
+
size = File.size(cache_file)
|
70
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
|
71
|
+
end
|
72
|
+
puts
|
73
|
+
end
|
74
|
+
if refs[:pushed].size > 0
|
75
|
+
puts "== Already Pushed Media =="
|
76
|
+
refs[:pushed].each do |sha|
|
77
|
+
cache_file = GitMedia.media_path(sha)
|
78
|
+
size = File.size(cache_file)
|
79
|
+
puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
|
80
|
+
end
|
81
|
+
puts
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.local_cache_status
|
86
|
+
# find files in media buffer and upload them
|
87
|
+
references = {:unpushed => [], :pushed => []}
|
88
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
89
|
+
unpushed_files = @push.get_unpushed(all_cache) || []
|
90
|
+
references[:unpushed] = unpushed_files
|
91
|
+
references[:pushed] = all_cache - unpushed_files rescue []
|
92
|
+
references
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
def self.to_human(size)
|
97
|
+
if size < 1024
|
98
|
+
return size.to_s + 'b'
|
99
|
+
elsif size < 1048576
|
100
|
+
return (size / 1024).to_s + 'k'
|
101
|
+
else
|
102
|
+
return (size / 1048576).to_s + 'm'
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# find files that are placeholders (41 char) and download them
|
2
|
+
# upload files in media buffer that are not in offsite bin
|
3
|
+
require 'git-media/status'
|
4
|
+
|
5
|
+
module GitMedia
|
6
|
+
module Sync
|
7
|
+
|
8
|
+
def self.run!
|
9
|
+
@push = GitMedia.get_push_transport
|
10
|
+
@pull = GitMedia.get_pull_transport
|
11
|
+
|
12
|
+
self.expand_references
|
13
|
+
self.upload_local_cache
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.expand_references
|
17
|
+
status = GitMedia::Status.find_references
|
18
|
+
status[:to_expand].each do |file, sha|
|
19
|
+
cache_file = GitMedia.media_path(sha)
|
20
|
+
if !File.exist?(cache_file)
|
21
|
+
puts "Downloading " + sha[0,8] + " : " + file
|
22
|
+
@pull.pull(file, sha)
|
23
|
+
end
|
24
|
+
|
25
|
+
puts "Expanding " + sha[0,8] + " : " + file
|
26
|
+
|
27
|
+
if File.exist?(cache_file)
|
28
|
+
FileUtils.cp(cache_file, file)
|
29
|
+
else
|
30
|
+
puts 'could not get media'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.upload_local_cache
|
36
|
+
# find files in media buffer and upload them
|
37
|
+
all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
|
38
|
+
unpushed_files = @push.get_unpushed(all_cache)
|
39
|
+
unpushed_files.each do |sha|
|
40
|
+
puts 'uploading ' + sha[0, 8]
|
41
|
+
@push.push(sha)
|
42
|
+
end
|
43
|
+
# TODO: if --clean, remove them
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module GitMedia
|
2
|
+
module Transport
|
3
|
+
class Base
|
4
|
+
|
5
|
+
def pull(final_file, sha)
|
6
|
+
to_file = GitMedia.media_path(sha)
|
7
|
+
get_file(sha, to_file)
|
8
|
+
end
|
9
|
+
|
10
|
+
def push(sha)
|
11
|
+
from_file = GitMedia.media_path(sha)
|
12
|
+
put_file(sha, from_file)
|
13
|
+
end
|
14
|
+
|
15
|
+
## OVERWRITE ##
|
16
|
+
|
17
|
+
def read?
|
18
|
+
false
|
19
|
+
end
|
20
|
+
|
21
|
+
def write?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_file(sha, to_file)
|
26
|
+
false
|
27
|
+
end
|
28
|
+
|
29
|
+
def put_file(sha, to_file)
|
30
|
+
false
|
31
|
+
end
|
32
|
+
|
33
|
+
def get_unpushed(files)
|
34
|
+
files
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
require 'atmos'
|
3
|
+
|
4
|
+
# git-media.transport atmos
|
5
|
+
# git-media.endpoint
|
6
|
+
# git-media.uid
|
7
|
+
# git-media.secret
|
8
|
+
# git-media.tag (optional)
|
9
|
+
|
10
|
+
module GitMedia
|
11
|
+
module Transport
|
12
|
+
class AtmosClient < Base
|
13
|
+
|
14
|
+
def initialize(endpoint, uid, secret, tag)
|
15
|
+
atmos_options = {
|
16
|
+
:url => endpoint,
|
17
|
+
:uid => uid,
|
18
|
+
:secret => secret
|
19
|
+
}
|
20
|
+
@tag = tag
|
21
|
+
@atmos_client = Atmos::Store.new(atmos_options)
|
22
|
+
end
|
23
|
+
|
24
|
+
def read?
|
25
|
+
reachable?
|
26
|
+
end
|
27
|
+
|
28
|
+
def get_file(sha, to_file)
|
29
|
+
dst_file = File.new(to_file, File::CREAT|File::RDWR)
|
30
|
+
@atmos_client.get(:namespace => sha).data_as_stream do |chunck|
|
31
|
+
dst_file.write(chunck)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def write
|
36
|
+
reachable?
|
37
|
+
end
|
38
|
+
|
39
|
+
def put_file(sha, from_file)
|
40
|
+
src_file = File.open(from_file)
|
41
|
+
obj_conf = {:data => src_file, :length => File.size(from_file), :namespace => sha}
|
42
|
+
obj_conf[:listable_metadata] = {@tag => true} if @tag
|
43
|
+
@atmos_client.create(obj_conf)
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_unpushed(files)
|
47
|
+
unpushed = []
|
48
|
+
files.each do |file|
|
49
|
+
begin
|
50
|
+
@atmos_client.get(:namespace => file)
|
51
|
+
rescue Atmos::Exceptions::AtmosException
|
52
|
+
unpushed << file
|
53
|
+
end
|
54
|
+
end
|
55
|
+
unpushed
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
# dummy function to test connectivity to atmos
|
60
|
+
def reachable?
|
61
|
+
@atmos_client.server_version
|
62
|
+
true
|
63
|
+
rescue
|
64
|
+
false
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
|
3
|
+
# move large media to local bin
|
4
|
+
|
5
|
+
# git-media.transport local
|
6
|
+
# git-media.localpath /opt/media
|
7
|
+
|
8
|
+
module GitMedia
|
9
|
+
module Transport
|
10
|
+
class Local < Base
|
11
|
+
|
12
|
+
def initialize(path)
|
13
|
+
@path = path
|
14
|
+
end
|
15
|
+
|
16
|
+
def read?
|
17
|
+
File.exist?(@path)
|
18
|
+
end
|
19
|
+
|
20
|
+
def get_file(sha, to_file)
|
21
|
+
from_file = File.join(@path, sha)
|
22
|
+
if File.exists?(from_file)
|
23
|
+
FileUtils.cp(from_file, to_file)
|
24
|
+
return true
|
25
|
+
end
|
26
|
+
return false
|
27
|
+
end
|
28
|
+
|
29
|
+
def write?
|
30
|
+
File.exist?(@path)
|
31
|
+
end
|
32
|
+
|
33
|
+
def put_file(sha, from_file)
|
34
|
+
to_file = File.join(@path, sha)
|
35
|
+
if File.exists?(from_file)
|
36
|
+
FileUtils.cp(from_file, to_file)
|
37
|
+
return true
|
38
|
+
end
|
39
|
+
return false
|
40
|
+
end
|
41
|
+
|
42
|
+
def get_unpushed(files)
|
43
|
+
files.select do |f|
|
44
|
+
!File.exist?(File.join(@path, f))
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
require 'right_aws'
|
3
|
+
|
4
|
+
# git-media.transport s3
|
5
|
+
# git-media.s3bucket
|
6
|
+
# git-media.s3key
|
7
|
+
# git-media.s3secret
|
8
|
+
|
9
|
+
module GitMedia
|
10
|
+
module Transport
|
11
|
+
class S3 < Base
|
12
|
+
|
13
|
+
def initialize(bucket, access_key_id = nil, secret_access_key = nil)
|
14
|
+
@s3 = RightAws::S3Interface.new(access_key_id, secret_access_key,
|
15
|
+
{:multi_thread => true, :logger => Logger.new('/tmp/s3.log')})
|
16
|
+
@bucket = bucket
|
17
|
+
@buckets = @s3.list_all_my_buckets.map { |a| a[:name] }
|
18
|
+
if !@buckets.include?(bucket)
|
19
|
+
puts "Creating New Bucket"
|
20
|
+
if @s3.create_bucket(bucket)
|
21
|
+
@buckets << bucket
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def read?
|
27
|
+
@buckets.size > 0
|
28
|
+
end
|
29
|
+
|
30
|
+
def get_file(sha, to_file)
|
31
|
+
to = File.new(to_file, File::CREAT|File::RDWR)
|
32
|
+
@s3.get(@bucket, sha) do |chunk|
|
33
|
+
to.write(chunk)
|
34
|
+
end
|
35
|
+
to.close
|
36
|
+
end
|
37
|
+
|
38
|
+
def write?
|
39
|
+
@buckets.size > 0
|
40
|
+
end
|
41
|
+
|
42
|
+
def put_file(sha, from_file)
|
43
|
+
@s3.put(@bucket, sha, File.open(from_file))
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_unpushed(files)
|
47
|
+
keys = @s3.list_bucket(@bucket).map { |f| f[:key] }
|
48
|
+
files.select do |f|
|
49
|
+
!keys.include?(f)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'git-media/transport'
|
2
|
+
|
3
|
+
# move large media to remote server via SCP
|
4
|
+
|
5
|
+
# git-media.transport scp
|
6
|
+
# git-media.scpuser someuser
|
7
|
+
# git-media.scphost remoteserver.com
|
8
|
+
# git-media.scppath /opt/media
|
9
|
+
|
10
|
+
module GitMedia
|
11
|
+
module Transport
|
12
|
+
class Scp < Base
|
13
|
+
|
14
|
+
def initialize(user, host, path, port)
|
15
|
+
@user = user
|
16
|
+
@host = host
|
17
|
+
@path = path
|
18
|
+
unless port === ""
|
19
|
+
@sshport = "-p#{port}"
|
20
|
+
end
|
21
|
+
unless port === ""
|
22
|
+
@scpport = "-P#{port}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def exist?(file)
|
27
|
+
@file = file
|
28
|
+
|
29
|
+
if Integer(result) == 1
|
30
|
+
puts file + " exists"
|
31
|
+
return true
|
32
|
+
else
|
33
|
+
puts file + " doesn't exists"
|
34
|
+
return false
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def result
|
39
|
+
`ssh #{@user}@#{@host} #{@sshport} [ -f "#{@file}" ] && echo 1 || echo 0`
|
40
|
+
end
|
41
|
+
|
42
|
+
def read?
|
43
|
+
return true
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_file(sha, to_file)
|
47
|
+
from_file = @user+"@"+@host+":"+File.join(@path, sha)
|
48
|
+
`scp #{@scpport} "#{from_file}" "#{to_file}"`
|
49
|
+
if $? == 0
|
50
|
+
puts sha+" downloaded"
|
51
|
+
return true
|
52
|
+
end
|
53
|
+
puts sha+" download fail"
|
54
|
+
return false
|
55
|
+
end
|
56
|
+
|
57
|
+
def write?
|
58
|
+
return true
|
59
|
+
end
|
60
|
+
|
61
|
+
def put_file(sha, from_file)
|
62
|
+
to_file = @user+"@"+@host+":"+File.join(@path, sha)
|
63
|
+
`scp #{@scpport} "#{from_file}" "#{to_file}"`
|
64
|
+
if $? == 0
|
65
|
+
puts sha+" uploaded"
|
66
|
+
return true
|
67
|
+
end
|
68
|
+
puts sha+" upload fail"
|
69
|
+
return false
|
70
|
+
end
|
71
|
+
|
72
|
+
def get_unpushed(files)
|
73
|
+
files.select do |f|
|
74
|
+
!self.exist?(File.join(@path, f))
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{notepunch-git-media}
|
5
|
+
s.version = "0.1.3"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Scott Chacon"]
|
9
|
+
s.date = %q{2009-06-10}
|
10
|
+
s.default_executable = %q{git-media}
|
11
|
+
s.email = %q{schacon@gmail.com}
|
12
|
+
s.executables = ["git-media"]
|
13
|
+
s.extra_rdoc_files = [
|
14
|
+
"LICENSE",
|
15
|
+
"README.rdoc"
|
16
|
+
]
|
17
|
+
s.files = [
|
18
|
+
".document",
|
19
|
+
".gitignore",
|
20
|
+
"LICENSE",
|
21
|
+
"README.rdoc",
|
22
|
+
"Rakefile",
|
23
|
+
"VERSION",
|
24
|
+
"bin/git-media",
|
25
|
+
"notepunch-git-media.gemspec",
|
26
|
+
"lib/git-media/clear.rb",
|
27
|
+
"lib/git-media/filter-clean.rb",
|
28
|
+
"lib/git-media/filter-smudge.rb",
|
29
|
+
"lib/git-media/status.rb",
|
30
|
+
"lib/git-media/sync.rb",
|
31
|
+
"lib/git-media/transport",
|
32
|
+
"lib/git-media/transport/local.rb",
|
33
|
+
"lib/git-media/transport/s3.rb",
|
34
|
+
"lib/git-media/transport/atmos_client.rb",
|
35
|
+
"lib/git-media/transport/scp.rb",
|
36
|
+
"lib/git-media/transport.rb",
|
37
|
+
"lib/git-media.rb"
|
38
|
+
]
|
39
|
+
s.has_rdoc = true
|
40
|
+
s.homepage = %q{http://github.com/schacon/git-media}
|
41
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
42
|
+
s.require_paths = ["lib"]
|
43
|
+
s.rubygems_version = %q{1.3.1}
|
44
|
+
s.summary = %q{"This is a summary! Stop yer whining"}
|
45
|
+
|
46
|
+
if s.respond_to? :specification_version then
|
47
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
48
|
+
s.specification_version = 2
|
49
|
+
|
50
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
51
|
+
else
|
52
|
+
end
|
53
|
+
else
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
metadata
ADDED
@@ -0,0 +1,65 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: notepunch-git-media
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Scott Chacon
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2009-06-10 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description:
|
14
|
+
email: schacon@gmail.com
|
15
|
+
executables:
|
16
|
+
- git-media
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files:
|
19
|
+
- LICENSE
|
20
|
+
- README.rdoc
|
21
|
+
files:
|
22
|
+
- .document
|
23
|
+
- .gitignore
|
24
|
+
- LICENSE
|
25
|
+
- README.rdoc
|
26
|
+
- Rakefile
|
27
|
+
- VERSION
|
28
|
+
- bin/git-media
|
29
|
+
- notepunch-git-media.gemspec
|
30
|
+
- lib/git-media/clear.rb
|
31
|
+
- lib/git-media/filter-clean.rb
|
32
|
+
- lib/git-media/filter-smudge.rb
|
33
|
+
- lib/git-media/status.rb
|
34
|
+
- lib/git-media/sync.rb
|
35
|
+
- lib/git-media/transport/local.rb
|
36
|
+
- lib/git-media/transport/s3.rb
|
37
|
+
- lib/git-media/transport/atmos_client.rb
|
38
|
+
- lib/git-media/transport/scp.rb
|
39
|
+
- lib/git-media/transport.rb
|
40
|
+
- lib/git-media.rb
|
41
|
+
homepage: http://github.com/schacon/git-media
|
42
|
+
licenses: []
|
43
|
+
metadata: {}
|
44
|
+
post_install_message:
|
45
|
+
rdoc_options:
|
46
|
+
- --charset=UTF-8
|
47
|
+
require_paths:
|
48
|
+
- lib
|
49
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
55
|
+
requirements:
|
56
|
+
- - ! '>='
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: '0'
|
59
|
+
requirements: []
|
60
|
+
rubyforge_project:
|
61
|
+
rubygems_version: 2.0.0
|
62
|
+
signing_key:
|
63
|
+
specification_version: 2
|
64
|
+
summary: ! '"This is a summary! Stop yer whining"'
|
65
|
+
test_files: []
|