ibg 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,14 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.so
12
+ *.o
13
+ *.a
14
+ mkmf.log
data/.travis.yml ADDED
@@ -0,0 +1,7 @@
1
+ language: ruby
2
+ cache: bundler
3
+ rvm:
4
+ - 1.9.3
5
+ - 2.0.0
6
+ - 2.1.5
7
+ - 2.2.0
data/Dockerfile ADDED
@@ -0,0 +1,47 @@
1
+ FROM ubuntu:14.04.1
2
+
3
+ RUN apt-get update
4
+ RUN apt-get upgrade -y
5
+ RUN apt-get install -y wget curl vim
6
+ RUN apt-get install -y redis-tools git ruby
7
+ RUN gem install redis
8
+ RUN gem install sinatra
9
+ RUN gem install haml
10
+ RUN apt-get install -y make ruby-dev g++
11
+ RUN gem install aws-sdk thin
12
+ RUN git clone https://github.com/ohac/iceberg.git
13
+ RUN \
14
+ mkdir -p /iceberg/public/css && \
15
+ mkdir -p /iceberg/public/js && \
16
+ mkdir -p /iceberg/public/fonts && \
17
+ cd /iceberg/public/css && \
18
+ curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css && \
19
+ curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap-theme.min.css && \
20
+ cd /iceberg/public/js && \
21
+ curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js && \
22
+ cd /iceberg/public/fonts && \
23
+ curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff && \
24
+ curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf
25
+ EXPOSE 4567
26
+ WORKDIR /iceberg
27
+ CMD rackup -p 4567 -o 0.0.0.0
28
+
29
+ # Build:
30
+ # docker build -t ohac/iceberg .
31
+ #
32
+ # Run (standalone):
33
+ # docker run --name redis -d \
34
+ # -v /somewhere/redis:/data redis redis-server --appendonly yes
35
+ # docker run --name iceberg -d --link redis:db -p 4567:4567 \
36
+ # -v /somewhere/iceberg:/root/.iceberg ohac/iceberg
37
+ #
38
+ # Run (with nginx):
39
+ # (setup default.conf, cert.pem and cert.key)
40
+ # docker run --name bb -d -t -p 80:80 -p 443:443 busybox
41
+ # docker run --name rd -d --net container:bb \
42
+ # -v $PWD/tmp/redis:/data redis redis-server --appendonly yes
43
+ # docker run --name nx -d --net container:bb \
44
+ # -v $PWD/examples/docker/default.conf:/etc/nginx/conf.d/default.conf:ro \
45
+ # -v $PWD/examples/docker/ssl:/data:ro nginx
46
+ # docker run --name ib -d --net container:bb \
47
+ # -v $PWD/tmp/iceberg:/.iceberg -v $PWD/tmp/aws:/.aws:ro ohac/iceberg
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2015 OHASHI Hideya
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,72 @@
1
+ # Iceberg
2
+
3
+ [![Build Status](https://travis-ci.org/ohac/iceberg.svg?branch=master)](https://travis-ci.org/ohac/iceberg)
4
+
5
+ Demo site: https://box.sighash.info/
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'ibg'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install ibg
22
+
23
+ ## Usage
24
+
25
+ Server
26
+ $ bundle exec rackup -s thin -p 4567
27
+ or
28
+ $ bundle exec bin/iceberg-web
29
+
30
+ Client
31
+ $ echo a > a
32
+ $ sha1sum a
33
+ 3f786850e387550fdab836ed7e6dc881de23001b a
34
+ $ curl -F file=@a -F tripkey=a http://localhost:4567/api/v1/upload
35
+ {"digest":"3f786850e387550fdab836ed7e6dc881de23001b","encdigest":"5618577aa4e6dc87931719ea26afbd7e886cb4e2","tripcode":"hvfkN.qlp.zh"}
36
+
37
+ Server
38
+ $ base64 ~/.iceberg/download/5618577aa4e6dc87931719ea26afbd7e886cb4e2
39
+ Zaf76KVeOWrs4UEWGqSReg==
40
+
41
+ Client
42
+ $ echo a | openssl enc -aes-128-cbc -K 3f786850e387550fdab836ed7e6dc881 -iv e387550fdab836ed7e6dc881de23001b -p -base64 -nosalt
43
+ key=3F786850E387550FDAB836ED7E6DC881
44
+ iv =E387550FDAB836ED7E6DC881DE23001B
45
+ Zaf76KVeOWrs4UEWGqSReg==
46
+ $ curl -s http://localhost:4567/api/v1/download/5618577aa4e6dc87931719ea26afbd7e886cb4e2 | base64
47
+ Zaf76KVeOWrs4UEWGqSReg==
48
+ $ curl -s http://localhost:4567/api/v1/recentfiles
49
+ {"recentfiles":["5618577aa4e6dc87931719ea26afbd7e886cb4e2"],"filemax":200}
50
+ $ curl -s http://localhost:4567/api/v1/tripcodelist
51
+ {"tripcodelist":["hvfkN.qlp.zh"]}
52
+
53
+ ## Contributing
54
+
55
+ 1. Fork it ( https://github.com/ohac/iceberg/fork )
56
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
57
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
58
+ 4. Push to the branch (`git push origin my-new-feature`)
59
+ 5. Create a new Pull Request
60
+
61
+ ## Copyright and license
62
+
63
+ * Iceberg
64
+ * The MIT License (MIT)
65
+ * Copyright (c) 2014-2014 OHASHI Hideya
66
+ * jQuery
67
+ * The MIT License (MIT)
68
+ * Twitter Bootstrap
69
+ * The MIT License (MIT)
70
+ * Copyright (c) 2011-2014 Twitter, Inc
71
+ * Bootstrap FileStyle
72
+ * The MIT License (MIT)
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ desc 'Run unit tests'
3
+ task :test do
4
+ sh 'ruby -I lib test/test_entry.rb'
5
+ end
6
+ task :default => :test
data/bin/iceberg-sync ADDED
@@ -0,0 +1,137 @@
1
+ #!/usr/bin/ruby
2
+ # coding: utf-8
3
+ require 'ibg'
4
+ require 'net/https'
5
+ require 'json'
6
+ require 'fileutils'
7
+
8
+ class MultiReader
9
+
10
+ def initialize
11
+ @readers = []
12
+ @cur = 0
13
+ end
14
+
15
+ def add(reader)
16
+ @readers << reader
17
+ end
18
+
19
+ def read(size = nil)
20
+ rv = ''
21
+ loop do
22
+ reader = @readers[@cur]
23
+ break unless reader
24
+ s = reader.read(size) # TODO S3
25
+ if s
26
+ rv += s
27
+ if size
28
+ size -= s.size
29
+ break if size <= 0
30
+ end
31
+ else
32
+ @cur += 1
33
+ end
34
+ end
35
+ rv.size > 0 ? rv : nil
36
+ end
37
+
38
+ end
39
+
40
+ class FileHub
41
+
42
+ def initialize(uri)
43
+ @uri = URI.parse(uri)
44
+ end
45
+
46
+ def gethttp
47
+ http = Net::HTTP.new(@uri.host, @uri.port)
48
+ if @uri.port == 443
49
+ http.use_ssl = true
50
+ #http.ca_file = 'cert.pem' # TODO
51
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
52
+ http.verify_depth = 5
53
+ end
54
+ http
55
+ end
56
+
57
+ def recentfiles
58
+ req = Net::HTTP::Get.new('/api/v1/recentfiles')
59
+ gethttp.start do |http|
60
+ res = http.request(req)
61
+ body = res.body
62
+ JSON.parse(body)
63
+ end
64
+ end
65
+
66
+ def download(encdigest, file)
67
+ begin
68
+ req = Net::HTTP::Get.new("/api/v1/download/#{encdigest}")
69
+ gethttp.start do |http|
70
+ http.request(req) do |res|
71
+ res.read_body do |chunk|
72
+ file.write(chunk) # TODO S3
73
+ end
74
+ end
75
+ end
76
+ # TODO check sha1sum
77
+ rescue
78
+ file.close rescue nil
79
+ file.delete
80
+ end
81
+ end
82
+
83
+ def uploadraw(file)
84
+ gethttp.start do |http|
85
+ encdigest = File.basename(file.key)
86
+ req = Net::HTTP::Post.new('/api/v1/uploadraw')
87
+ boundary = (0...50).map { (65 + rand(26)).chr }.join
88
+ req.set_content_type("multipart/form-data; boundary=#{boundary}")
89
+ body1 = StringIO.new(<<EOF)
90
+ --#{boundary}\r
91
+ Content-Disposition: form-data; name="file"; filename="#{encdigest}"\r
92
+ Content-Type: application/octet-stream\r
93
+ Content-Transfer-Encoding: binary\r
94
+ \r
95
+ EOF
96
+ body3 = StringIO.new(<<EOF)
97
+ --#{boundary}--\r
98
+ \r
99
+
100
+ EOF
101
+ mr = MultiReader.new
102
+ mr.add(body1)
103
+ mr.add(file)
104
+ mr.add(body3)
105
+ req.body_stream = mr
106
+ req.content_length = body1.size + file.content_length + body3.size
107
+ res = http.request(req)
108
+ body = res.body
109
+ JSON.parse(body)
110
+ end
111
+ end
112
+
113
+ end
114
+
115
+ b = Iceberg::Storage.new
116
+ hub = FileHub.new('https://box.sighash.info') # TODO
117
+ json = hub.recentfiles()
118
+ recentfiles = json['recentfiles']
119
+ recentfiles.each do |encdigest|
120
+ file = b.getobject(encdigest)
121
+ next if file.exists?
122
+ puts "download: #{encdigest}"
123
+ hub.download(encdigest, file)
124
+ file.close rescue nil # TODO
125
+ end
126
+ dir = b.dir
127
+ dir.each do |filename|
128
+ next unless filename.size == 40
129
+ unless recentfiles.index(filename)
130
+ puts "upload: #{filename}"
131
+ # TODO check server side file
132
+ # TODO check sha1sum before upload
133
+ file = b.getobject(filename)
134
+ json = hub.uploadraw(file)
135
+ file.close rescue nil # TODO
136
+ end
137
+ end
data/bin/iceberg-web ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/ruby
2
+ require 'ibg'
3
+
4
+ module Iceberg
5
+ class WebApp
6
+ run!
7
+ end
8
+ end
data/config.ru ADDED
@@ -0,0 +1,2 @@
1
+ require 'ibg'
2
+ run Iceberg::WebApp
@@ -0,0 +1,38 @@
1
+ upstream localhost {
2
+ ip_hash;
3
+ server 127.0.0.1:4567;
4
+ #server 127.0.0.1:9292;
5
+ }
6
+
7
+ server {
8
+ server_name localhost;
9
+ proxy_set_header Host $host;
10
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
11
+ proxy_set_header X-Forwarded-Host $host;
12
+ proxy_set_header X-Forwarded-Server $host;
13
+ proxy_set_header X-Real-IP $remote_addr;
14
+ location / {
15
+ set $do_not_cache 0;
16
+ proxy_no_cache $do_not_cache;
17
+ proxy_cache_bypass $do_not_cache;
18
+ proxy_pass http://localhost/;
19
+ }
20
+ }
21
+
22
+ server {
23
+ listen 443 default ssl;
24
+ ssl on;
25
+ server_name localhost;
26
+ ssl_certificate /etc/nginx/cert.pem;
27
+ ssl_certificate_key /etc/nginx/cert.key;
28
+ location / {
29
+ set $do_not_cache 0;
30
+ proxy_no_cache $do_not_cache;
31
+ proxy_cache_bypass $do_not_cache;
32
+ proxy_pass http://localhost/;
33
+ }
34
+ location /dbg/ {
35
+ root /usr/share/nginx/html;
36
+ index index.html index.htm;
37
+ }
38
+ }
@@ -0,0 +1,3 @@
1
+ [default]
2
+ aws_access_key_id = AAAAAAAAAAAAAAAAAAAA
3
+ aws_secret_access_key = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
data/ibg.gemspec ADDED
@@ -0,0 +1,27 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'ibg/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "ibg"
8
+ spec.version = Iceberg::VERSION
9
+ spec.authors = ["OHASHI Hideya"]
10
+ spec.email = ["ohachige@gmail.com"]
11
+ spec.summary = %q{Decenterized file storage}
12
+ spec.description = %q{Decenterized file storage}
13
+ spec.homepage = "https://box.sighash.info/"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_development_dependency "bundler", "~> 1.7"
22
+ spec.add_development_dependency "rake", "~> 10.0"
23
+ spec.add_dependency 'sinatra'
24
+ spec.add_dependency 'haml'
25
+ spec.add_dependency 'redis'
26
+ spec.add_dependency 'aws-sdk'
27
+ end
@@ -0,0 +1,96 @@
1
+ # coding: utf-8
2
+ require 'aws-sdk'
3
+
4
+ module Iceberg
5
+
6
+ class Storage
7
+
8
+ def initialize
9
+ @bucketname = SETTING['local']['s3bucket']
10
+ if @bucketname
11
+ s3 = AWS::S3.new
12
+ @bucket = s3.buckets[@bucketname]
13
+ end
14
+ end
15
+
16
+ def getobject(name)
17
+ if @bucketname
18
+ @bucket.objects['iceberg/' + name]
19
+ else
20
+ FileObject.new(name)
21
+ end
22
+ end
23
+
24
+ def dir
25
+ if @bucketname
26
+ s3 = AWS::S3.new
27
+ s3.buckets
28
+ else
29
+ path = SETTING['local']['download']
30
+ File.directory?(path) ? Dir.new(SETTING['local']['download']) : []
31
+ end
32
+ end
33
+
34
+ def create
35
+ if @bucketname
36
+ s3 = AWS::S3.new
37
+ @bucket = s3.buckets.create(@bucketname)
38
+ else
39
+ path = SETTING['local']['download']
40
+ FileUtils.mkdir_p(path)
41
+ end
42
+ end
43
+
44
+ end
45
+
46
+ class FileObject
47
+
48
+ def initialize(key)
49
+ download = SETTING['local']['download']
50
+ @key = key
51
+ @path = File.join(download, key)
52
+ @content_length = File.exist?(@path) ? File.size(@path) : nil
53
+ end
54
+
55
+ attr_reader :content_length, :key
56
+
57
+ def exists?
58
+ @content_length != nil
59
+ end
60
+
61
+ # TODO S3 bin/icebergsync.rb
62
+ def write(data)
63
+ @fd = File.open(@path, 'wb') unless @fd
64
+ @fd.write(data)
65
+ end
66
+
67
+ # TODO S3 bin/icebergsync.rb
68
+ def read(size = nil)
69
+ if block_given?
70
+ close
71
+ File.open(@path, 'rb') do |fd|
72
+ loop do
73
+ data = fd.read(32 * 1024) # TODO
74
+ break unless data
75
+ yield(data)
76
+ end
77
+ end
78
+ else
79
+ @fd = File.open(@path, 'rb') unless @fd
80
+ @fd.read(size)
81
+ end
82
+ end
83
+
84
+ def close
85
+ return unless @fd
86
+ @fd.close
87
+ @fd = nil
88
+ end
89
+
90
+ def delete
91
+ FileUtils.rm_f(@path)
92
+ end
93
+
94
+ end
95
+
96
+ end
@@ -0,0 +1,3 @@
1
+ module Iceberg
2
+ VERSION = "0.0.3"
3
+ end