ibg 0.0.3

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+cache: bundler
+rvm:
+- 1.9.3
+- 2.0.0
+- 2.1.5
+- 2.2.0

data/Dockerfile

@@ -0,0 +1,47 @@
+FROM ubuntu:14.04.1
+
+RUN apt-get update
+RUN apt-get upgrade -y
+RUN apt-get install -y wget curl vim
+RUN apt-get install -y redis-tools git ruby
+RUN gem install redis
+RUN gem install sinatra
+RUN gem install haml
+RUN apt-get install -y make ruby-dev g++
+RUN gem install aws-sdk thin
+RUN git clone https://github.com/ohac/iceberg.git
+RUN \
+  mkdir -p /iceberg/public/css && \
+  mkdir -p /iceberg/public/js && \
+  mkdir -p /iceberg/public/fonts && \
+  cd /iceberg/public/css && \
+  curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css && \
+  curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap-theme.min.css && \
+  cd /iceberg/public/js && \
+  curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js && \
+  cd /iceberg/public/fonts && \
+  curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff && \
+  curl -sO https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf
+EXPOSE 4567
+WORKDIR /iceberg
+CMD rackup -p 4567 -o 0.0.0.0
+
+# Build:
+# docker build -t ohac/iceberg .
+#
+# Run (standalone):
+# docker run --name redis -d \
+#   -v /somewhere/redis:/data redis redis-server --appendonly yes
+# docker run --name iceberg -d --link redis:db -p 4567:4567 \
+#   -v /somewhere/iceberg:/root/.iceberg ohac/iceberg
+#
+# Run (with nginx):
+# (setup default.conf, cert.pem and cert.key)
+# docker run --name bb -d -t -p 80:80 -p 443:443 busybox
+# docker run --name rd -d --net container:bb \
+#   -v $PWD/tmp/redis:/data redis redis-server --appendonly yes
+# docker run --name nx -d --net container:bb \
+#   -v $PWD/examples/docker/default.conf:/etc/nginx/conf.d/default.conf:ro \
+#   -v $PWD/examples/docker/ssl:/data:ro nginx
+# docker run --name ib -d --net container:bb \
+#   -v $PWD/tmp/iceberg:/.iceberg -v $PWD/tmp/aws:/.aws:ro ohac/iceberg

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 OHASHI Hideya
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,72 @@
+# Iceberg
+
+[![Build Status](https://travis-ci.org/ohac/iceberg.svg?branch=master)](https://travis-ci.org/ohac/iceberg)
+
+Demo site: https://box.sighash.info/
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ibg'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ibg
+
+## Usage
+
+    Server
+    $ bundle exec rackup -s thin -p 4567
+    or
+    $ bundle exec bin/iceberg-web
+
+    Client
+    $ echo a > a
+    $ sha1sum a
+    3f786850e387550fdab836ed7e6dc881de23001b  a
+    $ curl -F file=@a -F tripkey=a http://localhost:4567/api/v1/upload
+    {"digest":"3f786850e387550fdab836ed7e6dc881de23001b","encdigest":"5618577aa4e6dc87931719ea26afbd7e886cb4e2","tripcode":"hvfkN.qlp.zh"}
+
+    Server
+    $ base64 ~/.iceberg/download/5618577aa4e6dc87931719ea26afbd7e886cb4e2 
+    Zaf76KVeOWrs4UEWGqSReg==
+
+    Client
+    $ echo a | openssl enc -aes-128-cbc -K 3f786850e387550fdab836ed7e6dc881 -iv e387550fdab836ed7e6dc881de23001b -p -base64 -nosalt
+    key=3F786850E387550FDAB836ED7E6DC881
+    iv =E387550FDAB836ED7E6DC881DE23001B
+    Zaf76KVeOWrs4UEWGqSReg==
+    $ curl -s http://localhost:4567/api/v1/download/5618577aa4e6dc87931719ea26afbd7e886cb4e2 | base64
+    Zaf76KVeOWrs4UEWGqSReg==
+    $ curl -s http://localhost:4567/api/v1/recentfiles
+    {"recentfiles":["5618577aa4e6dc87931719ea26afbd7e886cb4e2"],"filemax":200}
+    $ curl -s http://localhost:4567/api/v1/tripcodelist
+    {"tripcodelist":["hvfkN.qlp.zh"]}
+
+## Contributing
+
+1. Fork it ( https://github.com/ohac/iceberg/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Copyright and license
+
+* Iceberg
+  * The MIT License (MIT)
+  * Copyright (c) 2014-2014 OHASHI Hideya
+* jQuery
+  * The MIT License (MIT)
+* Twitter Bootstrap
+  * The MIT License (MIT)
+  * Copyright (c) 2011-2014 Twitter, Inc
+* Bootstrap FileStyle
+  * The MIT License (MIT)

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+desc 'Run unit tests'
+task :test do
+    sh 'ruby -I lib test/test_entry.rb'
+end
+task :default => :test

data/bin/iceberg-sync

@@ -0,0 +1,137 @@
+#!/usr/bin/ruby
+# coding: utf-8
+require 'ibg'
+require 'net/https'
+require 'json'
+require 'fileutils'
+
+class MultiReader
+
+  def initialize
+    @readers = []
+    @cur = 0
+  end
+
+  def add(reader)
+    @readers << reader
+  end
+
+  def read(size = nil)
+    rv = ''
+    loop do
+      reader = @readers[@cur]
+      break unless reader
+      s = reader.read(size) # TODO S3
+      if s
+        rv += s
+        if size
+          size -= s.size
+          break if size <= 0
+        end
+      else
+        @cur += 1
+      end
+    end
+    rv.size > 0 ? rv : nil
+  end
+
+end
+
+class FileHub
+
+  def initialize(uri)
+    @uri = URI.parse(uri)
+  end
+
+  def gethttp
+    http = Net::HTTP.new(@uri.host, @uri.port)
+    if @uri.port == 443
+      http.use_ssl = true
+      #http.ca_file = 'cert.pem' # TODO
+      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      http.verify_depth = 5
+    end
+    http
+  end
+
+  def recentfiles
+    req = Net::HTTP::Get.new('/api/v1/recentfiles')
+    gethttp.start do |http|
+      res = http.request(req)
+      body = res.body
+      JSON.parse(body)
+    end
+  end
+
+  def download(encdigest, file)
+    begin
+      req = Net::HTTP::Get.new("/api/v1/download/#{encdigest}")
+      gethttp.start do |http|
+        http.request(req) do |res|
+          res.read_body do |chunk|
+            file.write(chunk) # TODO S3
+          end
+        end
+      end
+      # TODO check sha1sum
+    rescue
+      file.close rescue nil
+      file.delete
+    end
+  end
+
+  def uploadraw(file)
+    gethttp.start do |http|
+      encdigest = File.basename(file.key)
+      req = Net::HTTP::Post.new('/api/v1/uploadraw')
+      boundary = (0...50).map { (65 + rand(26)).chr }.join
+      req.set_content_type("multipart/form-data; boundary=#{boundary}")
+      body1 = StringIO.new(<<EOF)
+--#{boundary}\r
+Content-Disposition: form-data; name="file"; filename="#{encdigest}"\r
+Content-Type: application/octet-stream\r
+Content-Transfer-Encoding: binary\r
+\r
+EOF
+      body3 = StringIO.new(<<EOF)
+--#{boundary}--\r
+\r
+
+EOF
+      mr = MultiReader.new
+      mr.add(body1)
+      mr.add(file)
+      mr.add(body3)
+      req.body_stream = mr
+      req.content_length = body1.size + file.content_length + body3.size
+      res = http.request(req)
+      body = res.body
+      JSON.parse(body)
+    end
+  end
+
+end
+
+b = Iceberg::Storage.new
+hub = FileHub.new('https://box.sighash.info') # TODO
+json = hub.recentfiles()
+recentfiles = json['recentfiles']
+recentfiles.each do |encdigest|
+  file = b.getobject(encdigest)
+  next if file.exists?
+  puts "download: #{encdigest}"
+  hub.download(encdigest, file)
+  file.close rescue nil # TODO
+end
+dir = b.dir
+dir.each do |filename|
+  next unless filename.size == 40
+  unless recentfiles.index(filename)
+    puts "upload: #{filename}"
+    # TODO check server side file
+    # TODO check sha1sum before upload
+    file = b.getobject(filename)
+    json = hub.uploadraw(file)
+    file.close rescue nil # TODO
+  end
+end

data/bin/iceberg-web

@@ -0,0 +1,8 @@
+#!/usr/bin/ruby
+require 'ibg'
+
+module Iceberg
+  class WebApp
+    run!
+  end
+end

data/config.ru

@@ -0,0 +1,2 @@
+require 'ibg'
+run Iceberg::WebApp

data/examples/docker/default.conf

@@ -0,0 +1,38 @@
+upstream localhost {
+    ip_hash;
+    server 127.0.0.1:4567;
+    #server 127.0.0.1:9292;
+}
+
+server {
+    server_name localhost;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Host $host;
+    proxy_set_header X-Forwarded-Server $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    location / {
+        set $do_not_cache 0;
+        proxy_no_cache $do_not_cache;
+        proxy_cache_bypass $do_not_cache;
+        proxy_pass http://localhost/;
+    }
+}
+
+server {
+    listen 443 default ssl;
+    ssl on;
+    server_name localhost;
+    ssl_certificate /etc/nginx/cert.pem;
+    ssl_certificate_key /etc/nginx/cert.key;
+    location / {
+        set $do_not_cache 0;
+        proxy_no_cache $do_not_cache;
+        proxy_cache_bypass $do_not_cache;
+        proxy_pass http://localhost/;
+    }
+    location /dbg/ {
+        root /usr/share/nginx/html;
+        index index.html index.htm;
+    }
+}

data/examples/s3/credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = AAAAAAAAAAAAAAAAAAAA
+aws_secret_access_key = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa

data/ibg.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ibg/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ibg"
+  spec.version       = Iceberg::VERSION
+  spec.authors       = ["OHASHI Hideya"]
+  spec.email         = ["ohachige@gmail.com"]
+  spec.summary       = %q{Decenterized file storage}
+  spec.description   = %q{Decenterized file storage}
+  spec.homepage      = "https://box.sighash.info/"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_dependency 'sinatra'
+  spec.add_dependency 'haml'
+  spec.add_dependency 'redis'
+  spec.add_dependency 'aws-sdk'
+end

data/lib/ibg/storage.rb

@@ -0,0 +1,96 @@
+# coding: utf-8
+require 'aws-sdk'
+
+module Iceberg
+
+  class Storage
+
+    def initialize
+      @bucketname = SETTING['local']['s3bucket']
+      if @bucketname
+        s3 = AWS::S3.new
+        @bucket = s3.buckets[@bucketname]
+      end
+    end
+
+    def getobject(name)
+      if @bucketname
+        @bucket.objects['iceberg/' + name]
+      else
+        FileObject.new(name)
+      end
+    end
+
+    def dir
+      if @bucketname
+        s3 = AWS::S3.new
+        s3.buckets
+      else
+        path = SETTING['local']['download']
+        File.directory?(path) ? Dir.new(SETTING['local']['download']) : []
+      end
+    end
+
+    def create
+      if @bucketname
+        s3 = AWS::S3.new
+        @bucket = s3.buckets.create(@bucketname)
+      else
+        path = SETTING['local']['download']
+        FileUtils.mkdir_p(path)
+      end
+    end
+
+  end
+
+  class FileObject
+
+    def initialize(key)
+      download = SETTING['local']['download']
+      @key = key
+      @path = File.join(download, key)
+      @content_length = File.exist?(@path) ? File.size(@path) : nil
+    end
+
+    attr_reader :content_length, :key
+
+    def exists?
+      @content_length != nil
+    end
+
+    # TODO S3 bin/icebergsync.rb
+    def write(data)
+      @fd = File.open(@path, 'wb') unless @fd
+      @fd.write(data)
+    end
+
+    # TODO S3 bin/icebergsync.rb
+    def read(size = nil)
+      if block_given?
+        close
+        File.open(@path, 'rb') do |fd|
+          loop do
+            data = fd.read(32 * 1024) # TODO
+            break unless data
+            yield(data)
+          end
+        end
+      else
+        @fd = File.open(@path, 'rb') unless @fd
+        @fd.read(size)
+      end
+    end
+
+    def close
+      return unless @fd
+      @fd.close
+      @fd = nil
+    end
+
+    def delete
+      FileUtils.rm_f(@path)
+    end
+
+  end
+
+end

data/lib/ibg/version.rb

@@ -0,0 +1,3 @@
+module Iceberg
+  VERSION = "0.0.3"
+end