blobsterix 0.0.9

data/.gitignore

@@ -0,0 +1,27 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.rbenv-gemsets
+.ruby-gemset
+.ruby-version
+
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+contents/
+cache/
+typescript
+*~
+.idea
+spec.log

data/CHANGELOG.txt

@@ -0,0 +1,13 @@
+= 0.0.7
+ * now uses blobaccess to access all data in the transformation manager
+   cache and storage are now hidden and are used in the background accordingly. allows the system to prevent unneeded caching
+= 0.0.6
+ * renaming maxsize to resizemax
+ * event listener now accepts hashes
+ * adding events for API calls
+ * adding Blobsterix.event hook
+= 0.0.5
+ * adding maxsize
+= 0.0.4
+ * The first rc
+

data/Gemfile

@@ -0,0 +1,16 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in blob_server.gemspec
+gemspec
+
+gem 'simplecov', :require => false, :group => :test
+gem 'rack-test', :group => :test
+gem 'em-http-request', :group => :test
+
+group :development do 
+	gem "capistrano", "2.15.4"
+	gem "capistrano-ext", "1.2.1"
+	gem "rvm-capistrano", "~> 1.5.1"
+	gem 'whenever', '=0.8.2', :require => false
+	gem "railsless-deploy", :require => false
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Daniel Sudmann
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,122 @@
+# Blobsterix
+
+
+# IMPORTANT
+Use at least ruby-2.0.0-p451 to ensure thread safety (reference: https://github.com/eventmachine/eventmachine/issues/457)
+
+
+Blobsterix is a s3 compliant content server that supports on the fly transformations for its data. Images for example can be scaled, rotated and so on before delivered to the client. The system also supports encryption and transformation signing for the non s3 interface functions. The s3 interface itself is unsecured and should not be visible to the outside.
+
+# Install
+
+* Checkout the repository
+
+* Setup binary dependencies
+  * Webp
+    - Download precompiled binaries from google: https://developers.google.com/speed/webp/download
+    - OR apt-get install webp (on newer ubuntu this should work)
+    - OR make sure you have a cwbep script that behaves the right way
+  * MiniMagick
+    sudo apt-get install imagemagick
+  * VIPS(deprecated)
+    sudo apt-get install libjpeg-dev libpng-dev libtiff-dev libvips-dev
+  * lightpd config(not needed): 
+    sudo apt-get install libpcre3-dev libbz2-dev libglib2.0-dev
+  * ASCII Art (optional)
+    sudo apt-get install jp2a
+
+* Setup ruby
+  - bundle install
+
+# Usage
+
+To install blobsterix simply do a:
+
+  * gem install blobsterix
+
+This will install blobsterix with all its dependencies. After that you have to install the binary dependecies which are written above. Those are needed for the transformation pipeline not for the server itself. If you have your own transformations you might not need thoose.
+
+Now the server is basicly ready to run with 
+
+  * blobsterix server
+
+This will run the server in the current directory which will create a contents and a cache folder uppon the first request that needs them. If this is enough you are basicly done and you can use the server. When you need more control over the server you will have to create a server configuration. For this process there are generators shipped with blobsterix.
+
+# HTTP interface
+
+Now the server has two http interface and a status page.
+
+## Status Page
+
+The status page can be retrived via 
+
+  * GET /status
+  * GET /status.json
+  * GET /status.xml
+
+The first returns a "beautifull" html page. The other two return the status data as json or xml as expected.
+
+## S3 interface
+
+The s3 interface works like the aws system but without the encryption and security parameters. Because of this you shouldn't route GET, PUT, POST requests to the outside.
+Check the amazon s3 specs for more information. It only supports the REST api with the following actions:
+
+  * list buckets
+  * list bucket entries
+  * get file
+  * head file
+  * create bucket
+  * delete bucket
+  * put file into bucket
+  * delete file from bucket
+
+## Blob interface
+
+The blob interface is reachable at 
+
+  * GET /blob/v1/
+
+It only supports GET and HEAD requests and the url is constructed as this:
+
+  * /blob/v1/$trafo.$bucket/$file
+
+The transformation is expected in the following format:
+
+  * $trafoname_$value,$trafoname_$value,...
+
+The trafo is optional the bucket and file are not. The blob interface does not allow listing of buckets or files inside buckets.
+
+
+# Config setup
+
+To setup a special server change into and empty directory:
+
+  * mkdir special-blobsterix
+  * blobsterix generate init special-blobsterix
+
+This will copy a base template into special-blobsterix. Now you can checkout the generated config files for documentation on how to change things.
+
+# Custom Transformation
+
+The biggest strength of the system is it supports custom transformations without much coding required. To use a custom transformation you first have to setup a configuration as explained in the step before. Then you can use transformation generator to create a new transformation with
+
+  * blobsterix generate transformator CropToFace
+
+This will generate a new transforamtion in the transformators subdirectory of your server configuration. The transformation has standard parameters set which already make the transformation work. There are five functions:
+
+  * name
+  * is_format?
+  * input_type
+  * output_type
+  * transform
+
+The name function is used to match a parameter in the transformation string to a specific transformator. If it can not a transformator with the given name a dummy transformator is used which does nothing. The is_format? function tells the transformation manager if this transformator is an output format which should override a requested format send by the browser. For example images requested by a chrome browser are requested as webp and if in the transformation chain is no transformator with is_format? == true then in the end the transformation manager will try to find a transformation which will convert the image to webp. Now the input_type and output_type function do what they say. They tell the system what kind of file the transformator accepts and what kind of file it produces. With this information the transformation chain is build. So incase those function return wrong values the transformation might fail horribly. The last function is the one with all the magic. It transforms an input file to an output file. Now those files are passed as paths and are saved in the tmp folder or where ever ruby decides to save TempFiles. Those are outmaticly cleaned up after the transformation finishes. The value parameter is the second part of the transformation: resize_50
+Now the 50 would be passed as value with type string. The transformation has to take care of parsing the parameter and ensuring its not doing any harm to the system. Another thing to keep in mind is that the transform functions are called in a different thread than the rest of the server.
+
+# Custom Storage
+
+Blobsterix also supports different kinds of backend storages. For this purpose you can again use a generator to create template storage system with:
+
+  * blobsterix generate storage MyBackend
+
+This will again generate a working empty storage that doesn't allow saving or retriving of data. To see a working storage implementation checkout the FileSystem storage in the source code. Now you will see that all data there is returned as a BlobMetaData or better in the FileSystem storage as a FileSystemMetaData. As long as your implementation supports the interface everything should work like expected.

data/Rakefile

@@ -0,0 +1,13 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+
+desc "Simple cov"
+task :simplecov do
+  require 'simplecov'
+  SimpleCov.start do 
+    command_name "Tests"
+  end
+  require "blobsterix"
+  `rspec`
+end

data/bin/blobsterix

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+
+require "goliath/runner"
+require "fileutils"
+
+require "blobsterix"
+
+module ArgParser
+  def parse(args)
+    args[1..-1].each{|command|
+      @param ||= command if command != "generate" and @type
+      @type ||= command if command != "generate"
+    }
+  end
+end
+
+class Generator
+  include ArgParser
+  def initialize(args)
+    parse(args)
+  end
+
+  def run()
+    puts "Doing #{@type} with #{@param} in #{Dir.pwd}"
+
+    init(@param||"") if @type == "init"
+    transformation(@param||"NewTrafo") if @type == "transformator"
+    storage(@param||"Special") if @type == "storage"
+  end
+
+  def init(folder)
+    puts "Init new Blobsterix Server"
+
+    files_to_write = Dir.glob(File.join(BLOBSTERIX_GEM_DIR, "templates/app/**/*"))
+    check_files = files_to_write.map{|file|file.gsub(File.join(BLOBSTERIX_GEM_DIR, "templates/app/"), "")}
+
+    check_files.each_with_index {|file, index|
+      if !Dir.exist?(File.dirname(Blobsterix.root.join(folder, file)))
+        puts "Create folder: #{File.dirname(Blobsterix.root.join(folder, file))}"
+        FileUtils.mkdir_p(File.dirname(Blobsterix.root.join(folder, file)))
+      end
+      if !File.directory?(files_to_write[index]) && !File.exist?(Blobsterix.root.join(folder, file))
+        puts "Create file: #{Blobsterix.root.join(folder, file)}"
+        FileUtils.cp(files_to_write[index], Blobsterix.root.join(folder, file))
+      end
+    }
+  end
+
+  def storage(name)
+    storages_dir = Blobsterix.root.join("storages")
+    if File.exist?(File.join(storages_dir, "#{name.downcase}_storage.rb"))
+      puts "Storage #{name} at #{File.join(storages_dir, "#{name.downcase}_transformation.rb")} already exists. cancel"
+      return
+    end
+    FileUtils.mkdir_p(storages_dir)
+    new_transformator = File.read(File.join(BLOBSTERIX_GEM_DIR, "templates/storage_template.rb"))
+    new_transformator = new_transformator.gsub("$ClassName", name.capitalize)
+    new_transformator = new_transformator.gsub("$url_name", name.downcase)
+    File.write(File.join(storages_dir, "#{name.downcase}_storage.rb"), new_transformator)
+  end
+
+  def transformation(name)
+    trafo_dir = Blobsterix.root.join("transformators")
+    if File.exist?(File.join(trafo_dir, "#{name.downcase}_transformation.rb"))
+      puts "Transformator #{name} at #{File.join(trafo_dir, "#{name.downcase}_transformation.rb")} already exists. cancel"
+      return
+    end
+    FileUtils.mkdir_p(trafo_dir)
+    new_transformator = File.read(File.join(BLOBSTERIX_GEM_DIR, "templates/transformation_template.rb"))
+    new_transformator = new_transformator.gsub("$ClassName", name.capitalize)
+    new_transformator = new_transformator.gsub("$url_name", name.downcase)
+    File.write(File.join(trafo_dir, "#{name.downcase}_transformation.rb"), new_transformator)
+  end
+end
+class BlobsterixRunner
+  include ArgParser
+  include Blobsterix::ConfigLoader
+
+  def initialize(args)
+    parse(args)
+  end
+
+  def run_command()
+    begin
+      require "pry"
+
+      require_storages
+      require_config
+      require_transformators
+
+      Kernel.eval(@type)
+      exit
+    rescue LoadError => e
+      warn "=> Unable to load pry: #{e}"
+    end
+  end
+
+  def run_console()
+    begin
+      require "pry"
+
+      require_storages
+      require_config
+      require_transformators
+
+      Pry.start
+      exit
+    rescue LoadError => e
+      warn "=> Unable to load pry: #{e}"
+    end
+  end
+
+  def run_blobsterix()
+    require_storages
+    require_config
+    require_transformators
+
+    runner = Goliath::Runner.new(ARGV, nil)
+    runner.logger=Blobsterix.logger
+    runner.port = Blobsterix.port if Blobsterix.respond_to?(:port)
+    runner.address = Blobsterix.address if Blobsterix.respond_to?(:address)
+
+    Goliath.env = Blobsterix.env if Blobsterix.respond_to?(:env)
+
+    runner.api = Blobsterix::Service.new
+    runner.app = Goliath::Rack::Builder.build(Blobsterix::Service, runner.api)
+
+    puts "Now starting Blobsterix #{Blobsterix::VERSION}...."
+    runner.run
+  end
+end
+
+puts "Blobsterix #{Blobsterix::VERSION}"
+if ARGV.length == 0
+  puts "Usage: "
+  puts "\tserver (goliath server options) \n\t\tStarts blobsterix. When a blobsterix config is present in the executing folder it \n\t\twill be used otherwise blobsterix will run with standard options."
+  puts "\tconsole \n\t\tStarts blobsterix console and loads app config"
+  puts "\tgenerate (generator) (param) Used to generate blobsterix server files"
+  puts "\t\t init (foldername) - Creates a new blobsterix project. When a foldername is supplied it will be created in that folder."
+  puts "\t\t transformator (TrafoName) - Creates a new blobsterix transformator with the given name or just new as name."
+  puts "\t\t storage (StorageName) - Creates a new blobsterix storage with the given name or just new as name."
+  exit
+end
+if ARGV[0] == "server"
+  BlobsterixRunner.new(ARGV).run_blobsterix
+elsif ARGV[0] == "console"
+  BlobsterixRunner.new(ARGV).run_console
+elsif ARGV[0] == "run"
+  BlobsterixRunner.new(ARGV).run_command
+elsif ARGV[0] == "generate"
+  Generator.new(ARGV).run
+end

data/bin/test

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require 'blob_server'
+
+trafoManager = Blobsterix::Transformations::TransformationManager.new.add("ImageTransformation")
+
+bucket = "test"
+id = "droid2.png"
+type = "image/webp"
+trafo = "rotate_20,shrink_2"
+etag = "1fea74d710de911a9fd2807250765a20"
+
+#trafo = trafoManager.create(:bucket => bucket, :id => id, :type => type, :trafo => trafo)
+#trafo.run
+
+trafoManager.create(:bucket => bucket, :id => id, :type => type, :trafo => trafo).run
+trafoManager.create(:bucket => bucket, :id => id, :type => type, :trafo => trafo).run
+
+
+#f = File.open('contents/test/droid2.png')
+#blob = f.read
+
+##image = Magick::Image.from_blob(blob)
+#ilist = Magick::ImageList.new.from_blob(blob)
+##ilist << image[0]
+#ilist.background_color = "none"
+#ilist.rotate(20).write("out.png")

data/blobsterix.gemspec

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/blobsterix/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.required_ruby_version = '>= 2.0.0.p451'
+  gem.authors       = ["Daniel Sudmann"]
+  gem.email         = ["suddani@googlemail.com"]
+  gem.description   = "BlobServer"
+  gem.summary       = "BlobServer"
+  gem.homepage      = "http://experteer.com"
+  gem.license       = "MIT License"
+
+  gem.files         = `git ls-files`.split($\).select{|filename|
+    if filename.match(/^(\"contents).*/)
+      false
+    else
+      true
+    end
+  }
+  gem.executables   = ["blobsterix"]#gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "blobsterix"
+  gem.require_paths = ["lib"]
+  gem.version       = Blobsterix::VERSION
+
+
+  gem.add_dependency "json",        "~> 1.8.1"
+  gem.add_dependency "goliath",     "~> 1.0.3"
+  gem.add_dependency "journey",     "~> 1.0.4"
+  gem.add_dependency "nokogiri",    "~> 1.6.1"
+  gem.add_dependency "ruby-webp",   "~> 0.1.0"
+  gem.add_dependency "mini_magick", "~> 3.5.0"
+
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "rspec-core"
+  gem.add_development_dependency "rspec-expectations"
+  gem.add_development_dependency "rspec-mocks"
+  gem.add_development_dependency "pry"
+end

data/config/lighttpd.conf

@@ -0,0 +1,50 @@
+#lightpd config: sudo apt-get install libpcre3-dev libbz2-dev libglib2.0-dev
+#run: lighttpd -m /home/dsudmann/workspace/libs/lighttpd-1.4.33/src/.libs/ -D -f workspace/blobserver/config/lighttpd.conf
+#run: lighttpd -m /home/dsudmann/workspace/libs/lighttpd-1.5.0/src/.libs/ -D -f workspace/blobserver/config/lighttpd.conf
+
+server.document-root = "/home/dsudmann/workspace/experteer/blobsterix/contents/" 
+
+server.port = 3000
+
+server.modules += ( 
+#"mod_fastcgi", 
+#"mod_proxy", 
+"mod_proxy_core", "mod_proxy_backend_http", "mod_proxy_backend_fastcgi", "mod_staticfile", "mod_setenv" )
+
+mimetype.assign = (
+  ".html" => "text/html", 
+  ".txt" => "text/plain",
+  ".jpg" => "image/jpeg",
+  ".png" => "image/png" 
+)
+
+#$HTTP["url"] =~ "(^/blob/)" {   
+#  #proxy.server  = ( "" => ("" => ( "host" => "127.0.0.1", "port" => 9000 ))) 
+#  proxy.server = (""    => (
+#  			( "host" => "127.0.0.1", "port" => 9000),
+#  			( "host" => "127.0.0.1", "port" => 9001)
+#  		))
+#}
+
+$HTTP["url"] =~ "^/blob/" {
+		setenv.add-request-header = (
+		      "X-File" => "yes"
+		)
+        proxy-core.backends = ( 
+        "127.0.0.1:9000"
+        , "127.0.0.1:9001"
+        , "127.0.0.1:9002"
+        #, "127.0.0.1:9003"
+        #, "127.0.0.1:9004"
+        #, "127.0.0.1:9005"
+        )
+        #proxy-core.balancer = "round-robin"
+        proxy-core.protocol = "http" 
+        proxy-core.max-pool-size = 20
+        proxy-core.allow-x-sendfile = "enable"
+        #proxy-core.rewrite-request = (
+        #	"X-File" => (".*" => "YES" )
+        #   "_pathinfo" => ( "^/trac(/.*)" => "$1" ),
+        #   "_scriptname" => ( "^(/trac/)" => "$1" )
+        #)
+      }