ls4 0.9.0

data/ChangeLog

@@ -0,0 +1,9 @@
+
+== 2011-03-20 version 0.9.0
+
+Dedicated to the revival of Japan, with
+my heartfelt condolences to the victims
+of most disastrous earthquake on record.
+
+* Initial release.
+

data/NOTICE

@@ -0,0 +1,8 @@
+MessagePack-RPC for Ruby is developed by FURUHASHI Sadayuki, licensed under
+Apache License, Version 2.0. The original software and related information
+is available at http://msgpack.org/.
+
+MessagePack is developed by FURUHASHI Sadayuki, licensed under Apache License,
+Version 2.0. The original software and related information is available at
+http://msgpack.org/.
+

data/README.rdoc

@@ -0,0 +1,61 @@
+= LS4, a Large-Scale Simple Storage System
+
+LS4 is a distributed storage system that can store large data like photos, music or movies.
+
+[Scalability] Grows storage capacity and I/O throughput dynamically without any modification of applications.
+[Availability] Prevents whole system falt from partial hardware crashes by failing-over within very small downtime.
+[Versioning] Takes back old data by creation time or version name of the stored objects.
+[Continuous snapshot] Supported based on time-based versioning.
+[Geo-redundancy] Supports replication of objects across multiple datacenters.
+[Direct data transfer] Reduces local network traffic and load of front-end servers by cooperating with proxy servers.
+[Traffic offloading] Enhances performance of data servers by offloading traffics to native web servers.
+
+
+== Data model
+
+LS4 stores a set of objects identified by a key. Each object consists with data and attributes, and can have multiple versions.
+
+    key                        object
+                      data                  attributes
+               +-----------------+---------------------------------+       ---+
+  "image1" =>  |  "HTJ PNG ..."  |  { type:png, model:NEX-5 }      |--+       | each object can
+               +-----------------+---------------------------------+  |--+    | have multiple
+                 +-----------------+----------------------------------+  |    | versoins
+                    +----------------+-----------------------------------+    |
+                                                                           ---+
+               +-----------------+---------------------------------+
+    key    =>  |  bytes .......  |  { key:value, key:value, ... }  |--+
+               +-----------------+---------------------------------+  |--+
+                 +-----------------+----------------------------------+  |
+                    +----------------+-----------------------------------+
+  
+    ...    =>  ...
+
+For more details, see {LS4 Documents}[http://ls4.sourceforge.net/doc/].
+
+
+== Quick Start
+
+    $ gem install ls4
+    $ ls4-standalone -s ./data -h 18080 &
+    
+    $ curl -X POST -d 'data=value1&attrs={"test":"attr"}' http://localhost:18080/data/key1
+    
+    $ curl -X GET http://localhost:18080/data/key1
+    value1
+    
+    $ curl -X GET http://localhost:18080/attrs/key1
+    {"test":"attr"}
+
+
+
+Project web site::  http://ls4.sourceforge.net/
+Documents:: http://ls4.sourceforge.net/doc/
+Documents (Japanese):: http://ls4.sourceforge.net/doc/ja/
+Source repository:: http://github.com/ls4
+Author:: FURUHASHI Sadayuki
+Copyright:: (c) 2010-2011 FURUHASHI Sadayuki
+License:: AGPL
+
+See also NOTICE[https://github.com/ls4/ls4/blob/master/NOTICE] file.
+

data/bin/ls4-cs

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/cs'

data/bin/ls4-ds

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/ds'

data/bin/ls4-gw

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/gw'

data/bin/ls4-standalone

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/standalone'

data/bin/ls4cmd

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/cmd'

data/bin/ls4ctl

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/ctl'

data/bin/ls4rpc

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/rpc'

data/bin/ls4stat

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/stat'

data/bin/ls4top

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'ls4/command/top'

data/lib/ls4/command/cmd.rb

@@ -0,0 +1,241 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+require 'msgpack/rpc'
+require 'json'
+require 'pp'
+
+def usage
+	puts "Usage: #{File.basename($0)} <cs address[:port]> <command> [options]"
+	puts "command:"
+	puts "   get <key>                           get data and attributes"
+	puts "   gett <time> <key>                   get data and attributes using the time"
+	puts "   getv <vname> <key>                  get data and attributes using the version name"
+	puts "   get_data <key>                      get data"
+	puts "   gett_data <time> <key>              get data using the time"
+	puts "   getv_data <vname> <key>             get data using the version name"
+	puts "   get_attrs <key>                     get attributes"
+	puts "   gett_attrs <time> <key>             get attributes using the time"
+	puts "   getv_attrs <vname> <key>            get attributes using the version name"
+	puts "   read <key> <offset> <size>          get data with the offset and the size"
+	puts "   readt <time> <key> <offset> <size>  get data with the offset and the size using version time"
+	puts "   readv <vname> <key> <offset> <size> get data with the offset and the size using version name"
+	puts "   add <key> <data> <json>             set data and attributes"
+	puts "   addv <vname> <key> <data> <json>    set data and attributes with version name"
+	puts "   add_data <key> <data>               set data"
+	puts "   addv_data <vname> <key> <data>      set data with version name"
+	puts "   update_attrs <key> <json>           update attributes"
+	puts "   delete <key>                        delete the data and attributes"
+	puts "   deletet <time> <key>                delete the data and attributes using the time"
+	puts "   deletev <vname> <key>               delete the data and attributes using the version name"
+	puts "   remove <key>                        remove the data and attributes"
+	exit 1
+end
+
+if ARGV.length < 2
+	usage
+end
+
+$net = MessagePack::RPC::SessionPool.new
+addr = ARGV.shift
+host, port = addr.split(':', 2)
+port = port.to_i
+port = 18800 if port == 0
+$addr = [host,port]#Address.new(host, port)
+
+cmd = ARGV.shift
+
+def cmd_args(n)
+	if n < 0
+		return ARGV
+	end
+	usage if ARGV.length != n
+	ARGV.map! {|ar| ar == '-' ? $stdin.read : ar }
+	if n == 1
+		ARGV[0]
+	else
+		ARGV
+	end
+end
+
+def call(klass, *args)
+	start = Time.now
+
+	s = $net.get_session(*$addr)
+	s.timeout = 20
+	result = s.call(*args)
+	if klass && result
+		result = klass.new.from_msgpack(result)
+	end
+
+	finish = Time.now
+	$stderr.puts "#{finish - start} sec."
+
+	result
+end
+
+case cmd
+when 'get'
+	key = cmd_args(1)
+	data, attrs = call(nil, :get, key)
+	if data
+		puts attrs.to_json
+		$stdout.write data
+	else
+		$stderr.puts "nil"
+	end
+
+when 'gett'
+	vtime, key = cmd_args(2)
+	data, attrs = call(nil, :gett, vtime.to_i, key)
+	if data
+		puts attrs.to_json
+		$stdout.write data
+	else
+		$stderr.puts "nil"
+	end
+
+when 'getv'
+	vname, key = cmd_args(2)
+	data, attrs = call(nil, :getv, vname, key)
+	if data
+		puts attrs.to_json
+		$stdout.write data
+	else
+		$stderr.puts "nil"
+	end
+
+when 'get_data'
+	key = cmd_args(1)
+	data = call(nil, :get_data, key)
+	if data
+		$stderr.puts "#{data.size} bytes"
+		$stdout.write data
+	else
+		$stderr.puts "nil"
+	end
+
+when 'gett_data'
+	vtime, key = cmd_args(2)
+	data = call(nil, :gett_data, vtime.to_i, key)
+	if data
+		$stderr.puts "#{data.size} bytes"
+		$stdout.write data
+	else
+		$stderr.puts "nil"
+	end
+
+when 'getv_data'
+	vname, key = cmd_args(2)
+	data = call(nil, :getv_data, vname, key)
+	if data
+		$stderr.puts "#{data.size} bytes"
+		$stdout.write data
+	else
+		$stderr.puts "nil"
+	end
+
+when 'get_attrs'
+	key = cmd_args(1)
+	attrs = call(nil, :get_attrs, key)
+	if attrs
+		puts attrs.to_json
+	else
+		puts "nil"
+	end
+
+when 'gett_attrs'
+	vtime, key = cmd_args(2)
+	attrs = call(nil, :gett_attrs, vtime.to_i, key)
+	if attrs
+		puts attrs.to_json
+	else
+		puts "nil"
+	end
+
+when 'getv_attrs'
+	vname, key = cmd_args(2)
+	attrs = call(nil, :getv_attrs, vname, key)
+	if attrs
+		puts attrs.to_json
+	else
+		puts "nil"
+	end
+
+when 'read'
+	key, offset, size = cmd_args(3)
+	data = call(nil, :read, key, offset.to_i, size.to_i)
+	$stderr.puts "#{data.size} bytes"
+	$stdout.write data
+
+when 'readt'
+	vtime, key, offset, size = cmd_args(4)
+	data = call(nil, :readt, vtime.to_i, key, offset.to_i, size.to_i)
+	$stderr.puts "#{data.size} bytes"
+	$stdout.write data
+
+when 'readt'
+	vname, key, offset, size = cmd_args(4)
+	data = call(nil, :readv, vname.to_i, key, offset.to_i, size.to_i)
+	$stderr.puts "#{data.size} bytes"
+	$stdout.write data
+
+when 'add'
+	key, data, json = cmd_args(3)
+	attrs = JSON.parse(json)
+	pp call(nil, :add, key, data, attrs)
+
+when 'addv'
+	vname, key, data, json = cmd_args(4)
+	attrs = JSON.parse(json)
+	pp call(nil, :addv, vname, key, data, attrs)
+
+when 'add_data'
+	key, data = cmd_args(2)
+	pp call(nil, :add_data, key, data)
+
+when 'addv_data'
+	vname, key, data = cmd_args(3)
+	pp call(nil, :addv_data, vname, key, data)
+
+when 'update_attrs'
+	key, json = cmd_args(2)
+	attrs = JSON.parse(json)
+	pp call(nil, :update_attrs, key, attrs)
+
+when 'delete'
+	key = cmd_args(1)
+	pp call(nil, :delete, key)
+
+when 'deletet'
+	vtime, key = cmd_args(1)
+	pp call(nil, :deletet, vtime.to_i, key)
+
+when 'deletev'
+	vname, key = cmd_args(2)
+	pp call(nil, :deletev, vname, key)
+
+when 'remove'
+	key = cmd_args(1)
+	pp call(nil, :remove, key)
+
+else
+	$stderr.puts "unknown command #{cmd}"
+	$stderr.puts ""
+	usage
+end
+

data/lib/ls4/command/cs.rb

@@ -0,0 +1,190 @@
+#
+#  LS4
+#  Copyright (C) 2010-2011  FURUHASHI Sadayuki
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as
+#  published by the Free Software Foundation, either version 3 of the
+#  License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+require 'msgpack/rpc'
+require 'digest/md5'
+require 'digest/sha1'
+require 'csv'
+require 'fileutils'
+require 'ls4/lib/cclog'
+require 'ls4/lib/ebus'
+require 'ls4/lib/vbcode'
+require 'ls4/logic/tsv_data'
+require 'ls4/logic/weight'
+require 'ls4/logic/fault_detector'
+require 'ls4/logic/membership'
+require 'ls4/logic/node'
+require 'ls4/logic/okey'
+require 'ls4/service/base'
+require 'ls4/service/bus'
+require 'ls4/service/process'
+require 'ls4/service/rpc'
+require 'ls4/service/rpc_cs'
+require 'ls4/service/stat'
+require 'ls4/service/stat_cs'
+require 'ls4/service/config'
+require 'ls4/service/config_cs'
+require 'ls4/service/sync'
+require 'ls4/service/heartbeat'
+require 'ls4/service/membership'
+require 'ls4/service/weight'
+require 'ls4/service/balance'
+require 'ls4/service/master_select'
+require 'ls4/service/mds'
+require 'ls4/service/mds_cache'
+require 'ls4/service/log'
+require 'ls4/default'
+require 'ls4/version'
+require 'optparse'
+
+include LS4
+
+conf = CSConfigService.init
+
+op = OptionParser.new
+
+(class<<self;self;end).module_eval do
+	define_method(:usage) do |msg|
+		puts op.to_s
+		puts "error: #{msg}" if msg
+		exit 1
+	end
+end
+
+store_path = nil
+
+listen_host = '0.0.0.0'
+listen_port = CS_DEFAULT_PORT
+
+op.on('-p', '--port PORT', "listen port") do |addr|
+	if addr.include?(':')
+		listen_host, listen_port = addr.split(':',2)
+		listen_port = listen_port.to_i
+		listen_port = CS_DEFAULT_PORT if listen_port == 0
+	else
+		listen_port = addr.to_i
+	end
+end
+
+op.on('-l', '--listen HOST', "listen address") do |addr|
+	if addr.include?(':')
+		host, port = addr.split(':',2)
+		port = port.to_i
+		port = CS_DEFAULT_PORT if port == 0
+		listen_host = host
+		listen_port = port
+	else
+		listen_host = addr
+	end
+end
+
+op.on('-m', '--mds EXPR', "address of metadata servers (required)") do |s|
+	conf.mds_uri = s
+end
+
+op.on('-M', '--mds-cache EXPR', "address of metadata cache servers") do |s|
+	conf.mds_cache_uri = s
+end
+
+op.on('-s', '--store PATH', "path to default directory to store various data (required for production use)") do |path|
+	store_path = path
+end
+
+op.on('--fault_store PATH', "path to fault status file") do |path|
+	conf.fault_path = path
+end
+
+op.on('--membership_store PATH', "path to membership status file") do |path|
+	conf.membership_path = path
+end
+
+op.on('--weight_store PATH', "path to weight status file") do |path|
+	conf.weight_path = path
+end
+
+op.on('-o', '--log PATH') do |path|
+	conf.log_path = path
+end
+
+op.on('-v', '--verbose', "show debug messages", TrueClass) do |b|
+	$log.level = 1 if b
+end
+
+op.on('--trace', "show debug and trace messages", TrueClass) do |b|
+	$log.level = 0 if b
+end
+
+op.on('--color-log', "force to enable color log", TrueClass) do |b|
+	$log.enable_color
+end
+
+
+begin
+	op.parse!(ARGV)
+
+	if ARGV.length != 0
+		raise "unknown option: #{ARGV[0].dump}"
+	end
+
+	unless conf.mds_uri
+		raise "--mds option is required"
+	end
+
+	if store_path
+		FileUtils.mkdir_p(store_path)
+	end
+
+	if !conf.fault_path && store_path
+		conf.fault_path = File.join(store_path, "fault")
+	end
+
+	if !conf.membership_path && store_path
+		conf.membership_path = File.join(store_path, "membership")
+	end
+
+	if !conf.weight_path && store_path
+		conf.weight_path = File.join(store_path, "weight")
+	end
+
+rescue
+	usage $!.to_s
+end
+
+
+ProcessService.init
+LogService.open!
+SyncServerService.init
+HeartbeatServerService.init
+MembershipManagerService.init
+WeightManagerService.init
+MDSConfigService.init
+MDSCacheConfigService.init
+CSStatService.init
+
+LogService.instance.log_event_bus
+
+ProcessBus.run
+
+net = ProcessBus.serve_rpc(CSRPCService.instance)
+net.listen(listen_host, listen_port)
+
+$log.info "start on #{listen_host}:#{listen_port}"
+
+net.run
+
+ProcessBus.shutdown
+