borium 0.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    N2VlNWI2YzExMDA2NWVmODJmYzdiMTZmM2VkNjRhYzA5OWI3YmFkYg==
+  data.tar.gz: !binary |-
+    NjJjYWVhZmYxYmI0MGM0YzMzOTI0OWFmYTVlMGE1MWFkY2UyYWI2Mg==
+SHA512:
+  metadata.gz: !binary |-
+    NDVmY2NmZmViNmYxMTk0YWEzZjA5NTUxY2Y4N2ViMjI4MTk5ZTY1N2FjMjEx
+    NmQ4ODUxZTRlNTE3OTA5MzYzMjkwNjE5OWU2NTYxMWQ2YjJjNGM1YmRlMDEw
+    YjYzZWU4M2IyOGMwNjM1ZWJlYWFiNWFkNWQwY2MzZjZhMjg1MjU=
+  data.tar.gz: !binary |-
+    YTMxYjY2NjU3ZDhlNjAxMzk2NzYzMGY1MjUzNGM5ZTAyMDZlZWY5YmRiYzA4
+    YzE2ZWZiZWViZDAwMjJhMmFjNjMzOWM5YzBkMTRmMzUxYjYzZmQ5MzM3ZTRl
+    MmUzMzA4NTZhMGQwMjNiODEwZDlhY2M3OTQzYTk2ZGQwYzQ5ZjQ=

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.ruby-gemset

@@ -0,0 +1 @@
+borium

data/.ruby-version

@@ -0,0 +1 @@
+ruby-1.9.3-p484

data/Gemfile

@@ -0,0 +1,14 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+gem 'tokyocabinet', '1.29.1'
+gem 'json', '1.8.1'
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "bundler", "1.6.2"
+  gem "jeweler", "2.0.1"
+end

data/Gemfile.lock

@@ -0,0 +1,59 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.3.6)
+    builder (3.2.2)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    git (1.2.6)
+    github_api (0.11.3)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.1)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 1.2)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.0)
+      oauth2
+    hashie (2.1.1)
+    highline (1.6.21)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.1)
+    jwt (0.1.13)
+      multi_json (>= 1.5)
+    mini_portile (0.6.0)
+    multi_json (1.10.0)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.2.1)
+      mini_portile (= 0.6.0)
+    oauth2 (0.9.3)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 0.1.8)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rake (10.3.2)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    thread_safe (0.3.3)
+    tokyocabinet (1.29.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (= 1.6.2)
+  jeweler (= 2.0.1)
+  json (= 1.8.1)
+  tokyocabinet (= 1.29.1)

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2014 Freshout
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,20 @@
+= borium
+
+Borium is a distributed super-queue engine, capable of storing many different kinds of queues (fifo).
+
+Borium is tcp, and it keeps everything in queue, even if the machine crashes or is turned off it will continue right where it was at as soon as it gets restarted.
+
+Borium Receives requests as text, and responds json, this way many different clients and workers will know how to interpret the data
+
+Borium ensures that a requested item in queue will be given to only one client once.
+
+Strategy is that you have two kinds of applications interacting with Borium: 1) The ones that put something in the queue. 2) The ones that get something out of the queue.
+
+Contributors: Kazuyoshi Tlacaelel, Joaquin Benitez, Ben Beltran.
+
+
+== Copyright
+
+Copyright (c) 2014 Freshout. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "borium"
+  gem.homepage = "http://github.com/freshout-dev/borium"
+  gem.license = "MIT"
+  gem.summary = %Q{Borium - Distributed background engine.}
+  gem.description = %Q{Borium is a distributed super-queue engine, capable of storing many different kinds of queues (fifo).}
+  gem.email = "kazu.dev@gmail.com"
+  gem.authors = ["kazuyoshi tlacaelel"]
+  gem.executables << 'borium'
+  # dependencies defined in Gemfile
+end
+
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['test'].execute
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "borium #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/bin/borium

@@ -0,0 +1,156 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'tokyocabinet'
+require 'socket'
+require 'digest/md5'
+require 'json'
+require 'logger'
+require 'fileutils'
+
+include TokyoCabinet
+
+app = {
+
+  :databases       => { },
+  :eot             => "\004",
+  :must_stop       => false,
+  :name            => 'borium',
+  :nothing_counter => 0,
+
+  :status => {
+    :error      => 'error',      # get
+    :found      => 'found',      # get
+    :nothing    => 'nothing',    # get
+    :stored     => 'stored',     # put
+    :unparsable => 'unparsable'  # get
+  },
+
+  :config => {
+    'connections' => 1000,
+    'host'        => 'localhost',
+    'log'         => 'STDOUT',
+    'port'        => 8200,
+    'storage'     => '/tmp/borium/'
+  }
+
+} 
+
+configuration_file = "/etc/#{app[:name]}/configuration.json"
+
+if File.exists?(configuration_file)
+  config = JSON.parse(File.read(configuration_file))
+else
+  config = app[:config]
+end
+
+if File.exists?(config['log'])
+  logger = Logger.new(config['log'])
+else
+  logger = Logger.new(STDOUT)
+end
+
+terminate = Proc.new do
+  puts 'Bye.'
+  app[:must_stop] = true
+  logger.warn 'Going down.'
+  app[:databases].each do |key, hdb|
+    logger.warn "syncronize hdb: #{key}"
+    hdb.close
+  end
+  exit 0
+end
+
+trap('TERM', &terminate)
+trap('INT',  &terminate)
+trap('QUIT', &terminate)
+
+def generate_key
+  Digest::MD5.hexdigest(
+    Time.now.to_f.to_s + '-' + (1..100).to_a.shuffle.first.to_s
+  )
+end
+
+FileUtils.mkdir_p(config['storage'])
+server = TCPServer.open config['host'], config['port']
+server.listen(config['connections'])
+
+logger.info 'Server started'
+
+config.each do |key, value|
+  logger.info "CONFIG: #{key} => #{value}"
+end
+
+loop do
+  break if app[:must_stop]
+
+  client = server.accept
+  data   = ''
+
+  while line = client.gets
+    data << line
+    break if line.include?(app[:eot])
+  end
+
+  chunks   = data.split(':')
+  response = {}
+
+  # respond with error if "job_type" was not given.
+  if chunks.size < 3
+    response[:status] = app[:status][:unparsable]
+    begin
+      client.puts response.to_json
+    rescue => e
+      logger.warn 'Client closed the connection.'
+    end
+    client.close
+    logger.error "wrong request: #{chunks.inspect}"
+    next
+  end
+
+  request  = chunks.shift
+  job_type = chunks.shift
+
+  unless app[:databases][job_type]
+    app[:databases][job_type] = HDB::new
+    app[:databases][job_type].open(
+      "#{config['storage']}/#{job_type}.job.tch",
+      HDB::OCREAT | HDB::OWRITER
+    )
+  end
+
+  db = app[:databases][job_type]
+
+  if (request == 'put')
+    db.put(generate_key, chunks.join('').gsub(app[:eot], '').gsub(/\n$/, ''))
+    response[:status] = app[:status][:stored]
+  else
+    db.iterinit
+    key = db.iternext
+    job = db.get(key)
+    if job
+      response[:status] = app[:status][:found]
+      response[:job] = job
+      db.out(key)
+    else
+      response[:status] = app[:status][:nothing]
+    end
+  end
+  result = response.to_json
+  begin
+    client.puts(result)
+  rescue => e
+    logger.warn 'Client closed the connection.'
+  end
+  client.close
+  if response[:status] == app[:status][:nothing]
+    app[:nothing_counter] += 1 
+    if app[:nothing_counter] > 5000
+      logger.info "ok #{request} #{job_type} #{db.rnum.to_s} 5K => #{result}"
+      app[:nothing_counter] = 0
+    end
+  else
+    logger.info "ok #{request} #{job_type} #{db.rnum.to_s} #{result}"
+  end
+end
+

data/lib/borium.rb

@@ -0,0 +1,63 @@
+require 'rubygems'
+require 'json'
+require 'socket'
+
+class Borium
+
+  # @visibility private
+  EOT = "\004"
+
+  # @visibility private
+  DEFAULT_CONFIG = {
+    'connections' => 1000,
+    'host'        => 'localhost',
+    'log'         => 'STDOUT',
+    'port'        => 8200,
+    'storage'     => '/tmp/borium/'
+  }
+
+  def self.get type
+    _request "get:#{type}:"
+  end
+
+  def self.put type, job
+    _request "put:#{type}:#{job}"
+  end
+
+  private
+
+  def self._config
+    @_config ||= _resolve_configuration
+  end
+
+  def self._resolve_configuration
+    configuration_file = '/etc/borium/configuration.json'
+    return DEFAULT_CONFIG unless File.exists?(configuration_file)
+    JSON.parse(File.read(configuration_file))
+  end
+
+  def self._request query
+    counter = 0
+    begin
+      counter += 1
+      socket = ::TCPSocket.new _config['host'], _config['port']
+      socket.puts query + Borium::EOT
+      data = ''
+      while line = socket.gets
+        break unless line
+        data << line
+      end
+      socket.close
+      return ::JSON.parse(data)
+    rescue => error
+      STDERR.puts error.message
+      if counter > 10
+        raise error
+      else
+        sleep 1
+        retry
+      end
+    end
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,34 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'borium'
+
+class Test::Unit::TestCase
+end

data/test/test_borium.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestBorium < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: borium
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- kazuyoshi tlacaelel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: tokyocabinet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.29.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.29.1
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.2
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+description: Borium is a distributed super-queue engine, capable of storing many different
+  kinds of queues (fifo).
+email: kazu.dev@gmail.com
+executables:
+- borium
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .document
+- .ruby-gemset
+- .ruby-version
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/borium
+- lib/borium.rb
+- test/helper.rb
+- test/test_borium.rb
+homepage: http://github.com/freshout-dev/borium
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Borium - Distributed background engine.
+test_files: []