kurosawa 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fff5741b6786fceddee8b3934e4f370848797c46
-  data.tar.gz: d925216514517c42099bdf1570b030ec88b05554
+  metadata.gz: 14e1940ade23b020c48dd0ea923424bf56fafae2
+  data.tar.gz: 325c412026e1c1efde785a888b8e70254f7ba59b
 SHA512:
-  metadata.gz: 292c41fe1c4c7d513e440b2c48b95f7750e0d3036d618c6d6d7496bad8d3ada4d3b8cecef48f9bdfde71afeebb724f964c6ac9524aa90f7c3e51ec79c8eacae2
-  data.tar.gz: c6394aa4aaac02b6421bd5099b5b9effb523882ec1924beafd33375e5ffa44fd55efbf1755512dff2d9c5671795417347169be7736fa99f9da8834f546bb29ce
+  metadata.gz: 0d4dd6523104e8d90f51c5cdae0998ca417de04e7b81eb6328be73dc4309145f50a17bdff2cedc291afa2dd4e60ae8e5aecef1fb3ff09ffe9af73e797969f527
+  data.tar.gz: ac744719d7b169a2f8811712c3d7f72c3c56583fa1590ef9c30e2adb56b55703866c58f99fe4b422b1a5c0ac62cef31f6973ebb19d7cdd097f00ac97e82a1d37

data/README.md

@@ -1,8 +1,10 @@
-# Kurosawa
+# kurosawa.rb
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/kurosawa`. To experiment with that code, run `bin/console` for an interactive prompt.
+![Kurosawa Ruby](https://github.com/astrobunny/kurosawa.rb/raw/master/docs/images/kurosawa-ruby.jpg)
 
-TODO: Delete this and the text above, and describe your gem
+A RESTful JSON-based database for eventually-consistent filesystem backends. Uses the REST path to determine object hierarchy.
+
+# THIS DATABASE IS STILL IN ALPHA
 
 ## Installation
 
@@ -22,7 +24,54 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+### On your local environment
+
+Run the server
+
+```
+KUROSAWA_FILESYSTEM=file://fs/db bundle exec kurosawa
+```
+
+### With a docker container
+
+```
+docker run -ti -p4567:4567 astrobunny/kurosawa.rb
+```
+
+### Try it out!
+
+Send it REST commands! (Here I am using resty)
+
+```
+$ GET /
+null
+$ PUT / '{"a": 7, "b": {"e":[100,200,300]} }'
+{"a":"7","b":{"e":["300","200","100"]}}
+$ GET /
+{"a":"7","b":{"e":["300","200","100"]}}
+$ GET /a/b
+null
+$ GET /a
+"7"
+$ GET /b
+{"e":["300","200","100"]}
+$ GET /b/e
+["300","200","100"]
+$ PATCH / '{"c": "Idols"}'
+{"a":"7","b":{"e":["300","200","100"]},"c":"Idols"}
+$ GET /c
+"Idols"
+$ DELETE /b/e
+null
+$ GET /
+{"a":"7","b":{},"c":"Idols"}
+$ PUT /c '{}'
+{}
+$ PUT /c '{"5": "a"}'
+{"5":"a"}
+
+```
+
 
 ## Development
 
@@ -32,5 +81,5 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kurosawa.
+Bug reports and pull requests are welcome on GitHub at https://github.com/astrobunny/kurosawa.rb.
 

data/exe/kurosawa

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "kurosawa/database"
+
+Kurosawa::Database.run!

data/lib/kurosawa.rb

@@ -1,5 +1,3 @@
 require "kurosawa/version"
 
-module Kurosawa
-  # Your code goes here...
-end
+require "kurosawa/connection"

data/lib/kurosawa/connection.rb

@@ -0,0 +1,4 @@
+module Kurosawa
+	class Connection
+	end
+end

data/lib/kurosawa/database.rb

@@ -0,0 +1,205 @@
+require "sinatra/base"
+require "sinatra/reloader"
+require "json"
+
+# filesystems
+require "kurosawa/filesystems"
+
+module Kurosawa
+	class Database < ::Sinatra::Base
+
+		set :bind, '0.0.0.0'
+		configure :development do
+			register Sinatra::Reloader
+		end
+
+		filesystem = ENV["KUROSAWA_FILESYSTEM"] || "file://~/.kurosawa"
+
+		set :filesystem, Filesystem.instantiate(filesystem)
+
+		def sanitize_path(path)
+			match = /^\/[a-z0-9\-_\/]*$/i.match(URI.unescape(path))
+			if match
+				match.to_s.sub(/\/+$/, "")
+			else
+				p URI.unescape(path)
+				halt 400
+			end
+		end
+
+		def sanitize_body(body)
+			begin
+				JSON.parse(body)
+			rescue => e
+				puts e
+				halt 400
+			end
+		end
+
+		def random_key
+			SecureRandom.hex(4)
+		end
+
+		def get_property(fs, path)
+			property_entries = fs.ls("#{path}/$")
+			if property_entries.length == 0
+				nil
+			elsif property_entries.length == 1
+				JSON.parse(fs.get(property_entries[0]), symbolize_names: true)
+			else
+				# conflict
+				raise "TODO get_property handle conflict"
+			end
+		end
+ 
+		def set_property(fs, path, type:)
+			puts "set property: #{path}"
+			property_entries = fs.ls("#{path}/$")
+			p property_entries
+			if property_entries.length > 0
+				property_entries.each do |x|
+					fs.del(x)
+				end
+			end
+			fs.put("#{path}/$#{random_key}", {type: type}.to_json)
+		end
+
+
+		def read(fs, path, params)
+
+			puts "read: path=#{path.inspect}"
+			prop = get_property(fs, path)
+			if prop == nil
+				nil
+			elsif prop[:type] == "object"
+				res = {}
+
+				inner_entries = fs.ls("#{path}")
+					.select{ |x| /^\/\$/.match(x) == nil }
+					.map{ |x| x.sub(/\/[^\/]+$/, "") }
+					.map{ |x| match = /(?<first>^\/[^\/]+)/.match(x.sub(/^#{path}/, "")); match[:first] if match }
+					.select{ |x| x != nil}
+					.uniq
+
+				puts "read: object: inner: #{fs.ls("#{path}").inspect}"
+				puts "read: object: inner: #{inner_entries.inspect}"
+
+				inner_entries.each do |x|
+					res[x[1..-1]] = read(fs, path + x, params)
+				end
+
+				res
+
+			elsif prop[:type] == "array"
+				inner_entries = fs.ls("#{path}")
+					.select{ |x| /\/\$/.match(x) == nil }
+					.map{ |x| match = /(?<first>^\/[^\/]+)/.match(x.sub(/^#{path}/, "")); match[:first] if match }
+					.select{ |x| x != nil}
+					.uniq
+
+				puts "read: array: #{inner_entries.inspect}"
+
+				inner_entries.map do |x|
+					read(fs, path + x, params)
+				end
+
+			else
+				inner_entries = fs.ls("#{path}")
+					.select{ |x| /\/\$/.match(x) == nil }
+
+				puts "read: string: #{inner_entries.inspect}"
+
+				if inner_entries.length == 1
+					JSON.parse(fs.get(inner_entries[0]), symbolize_names: true)[:value] 
+				elsif inner_entries.length == 0
+					nil
+				else
+					{"$conflict": inner_entries.map{|x| JSON.parse(fs.get(x), symbolize_names: true) }}
+					#raise "read conflict at #{path} #{inner_entries.inspect}"
+				end
+			end
+		end
+
+		def write(fs, path, body)
+
+			puts "write: path=#{path.inspect}"
+
+			if body.is_a? Hash
+				set_property(fs, path, type: "object")
+				body.each do |k,v|
+					write(fs, "#{path}/#{k}", v)
+				end
+
+			elsif body.is_a? Array
+				set_property(fs, path, type: "array")
+				body.each do |value|
+					write(fs, "#{path}/#{random_key}", value)
+				end
+
+			else
+				existing_entries = fs.ls("#{path}")
+
+				if existing_entries.length > 0
+					existing_entries.each do |x|
+						fs.del(x)
+					end
+				end
+				set_property(fs, path, type: "string")
+				fs.put("#{path}/#{random_key}", {value: body.to_s}.to_json)
+			end
+		end
+
+		def delete(fs, path)
+			inner_entries = fs.ls("#{path}")
+
+			puts "delete: #{inner_entries.inspect}"
+
+			inner_entries.each do |x|
+				fs.del(x)
+			end
+		end
+
+		get "*" do
+			path = sanitize_path(request.path)
+			x = read(settings.filesystem, path, params)
+			status 404 if x == nil
+			x.to_json
+		end
+
+		post "*" do
+			path = sanitize_path(request.path)
+
+		end
+
+		put "*" do
+			path = sanitize_path(request.path)
+			delete(settings.filesystem, path)
+			write(settings.filesystem, path, sanitize_body(request.body.read))
+			read(settings.filesystem, path, params).to_json
+		end
+
+		patch "*" do
+			path = sanitize_path(request.path)
+			write(settings.filesystem, path, sanitize_body(request.body.read))
+			read(settings.filesystem, path, params).to_json
+		end
+
+		delete "*" do
+			path = sanitize_path(request.path)
+			delete(settings.filesystem, path)
+			read(settings.filesystem, path, params).to_json
+		end
+
+		head "*" do
+			path = sanitize_path(request.path)
+
+		end
+
+		options "*" do
+			path = sanitize_path(request.path)
+
+		end
+
+
+	end
+end

data/lib/kurosawa/filesystems.rb

@@ -0,0 +1,29 @@
+
+require "kurosawa/filesystems/local"
+require "kurosawa/filesystems/s3"
+
+require "uri"
+
+module Kurosawa
+	class Filesystem
+		def self.instantiate(fs)
+			if fs.start_with?("file://")
+				Kurosawa::Filesystems::Local.new(fs.sub("file://", ""))
+			elsif fs.start_with?("s3://") or fs.start_with?("s3http://") or fs.start_with?("s3https://")
+				
+				url = URI.parse(fs);
+				tokens = url.userinfo.split(":")
+
+				Kurosawa::Filesystems::S3.new(
+					access_key_id:"#{tokens[0]}",
+					secret_access_key:"#{tokens[1]}",
+					region:"#{tokens[2]}",
+					bucket:url.scheme == "s3" ? url.host : "",
+					endpoint:url.scheme == "s3" ? nil : "#{url.scheme.sub("s3","")}://#{url.host}:#{url.port}",
+					force_path_style: url.scheme != "s3")
+			else
+				raise "Unknown filesystem"
+			end
+		end
+	end
+end

data/lib/kurosawa/filesystems/base.rb

@@ -0,0 +1,25 @@
+
+module Kurosawa::Filesystems
+	class Base
+		def ls(prefix)
+			[]
+		end
+
+		def get(key)
+			""
+		end
+
+		def put(key, value)
+		end
+
+		def del(key)
+		end
+
+		def exists(key)
+			false
+		end
+
+		def cleanup!
+		end
+	end
+end

data/lib/kurosawa/filesystems/local.rb

@@ -0,0 +1,59 @@
+require 'kurosawa/filesystems/base'
+require 'fileutils'
+
+module Kurosawa::Filesystems
+	class Local < Base
+
+		def initialize(root_path)
+			@root_path = File.expand_path(root_path)
+			FileUtils.mkdir_p(@root_path)
+		end
+
+		def root_path
+			@root_path
+		end
+
+		def path(key)
+			File.join(@root_path, key)
+		end
+
+		def ls(prefix)
+			puts "fs: ls #{prefix}"
+			(Dir["#{@root_path}/#{prefix}*"] + Dir["#{@root_path}/#{prefix}/**/*"]).
+				select{|x| File.file?(x)}.
+				map{|x| x.sub(/^#{root_path}/, "").
+				gsub(/\/+/, "/")}.
+				uniq
+			
+		end
+
+		def get(key)
+			puts "fs: get #{path(key)}"
+			File.read(path(key))
+		end
+
+		def put(key, value)
+			puts "fs: put #{path(key)}"
+			FileUtils.mkdir_p(File.dirname(path(key)))
+			File.write(path(key), value)
+		end
+
+		def del(key)
+			puts "fs: delete #{path(key)}"
+			File.delete(path(key)) if File.file?(path(key))
+		end
+
+		def exists(key)
+			puts "fs: exists #{path(key)}"
+			File.exists?(path(key)) and File.file?(path(key))
+		end
+
+		def cleanup!
+			Dir['#{@root_path}**/*'].
+				select { |d| File.directory? d }.
+				select { |d| (Dir.entries(d) - %w[ . .. ]).empty? }.
+				each   { |d| Dir.rmdir d }
+		end
+
+	end
+end

data/lib/kurosawa/filesystems/s3.rb

@@ -0,0 +1,72 @@
+require 'kurosawa/filesystems/base'
+require 'uri'
+require 'aws-sdk'
+
+module Kurosawa::Filesystems
+	class S3 < Base
+
+		def initialize(access_key_id:,secret_access_key:,region:,endpoint:,bucket:,force_path_style:false)
+
+			options = {
+				:access_key_id => URI.encode(access_key_id),
+				:secret_access_key => URI.decode(secret_access_key),
+				:region => region,
+				:force_path_style => force_path_style
+			}
+			options[:endpoint] = endpoint if endpoint
+
+			@s3 = Aws::S3::Client.new(options)
+			@bucket_name = bucket
+			@bucket = Aws::S3::Bucket.new(bucket, client: @s3)
+		end
+
+		def ls(prefix)
+			prefix.gsub!(/^\/+/, "")
+			puts "s3:ls #{prefix}"
+			result = []
+			next_marker = nil
+			loop do
+				options = {
+					bucket: @bucket_name,
+					marker: next_marker
+				}
+
+				options[:prefix] = prefix if prefix.length != 0
+
+				resp = @s3.list_objects(options);
+				result += resp.contents.map { |x| "/#{x.key}" }
+				break if resp.next_marker == nil
+				next_marker = resp.next_marker
+			end
+			result
+		end
+
+		def get(key)
+			key.gsub!(/^\/+/, "")
+			puts "s3:get #{key}"
+			resp = @s3.get_object(bucket: @bucket_name, key: key)
+			resp.body.read
+		end
+
+		def put(key, value)
+			key.gsub!(/^\/+/, "")
+			puts "s3:put #{key}"
+			@s3.put_object(bucket: @bucket_name, key: key, body: value)
+		end
+
+		def del(key)
+			key.gsub!(/^\/+/, "")
+			puts "s3:del #{key}"
+			@s3.delete_object(bucket: @bucket_name, key: key)
+		end
+
+		def exists(key)
+			key.gsub!(/^\/+/, "")
+			puts "s3:exists #{key}"
+			@bucket.objects[key].exists?
+		end
+
+		def cleanup!
+		end
+	end
+end

data/lib/kurosawa/version.rb

@@ -1,3 +1,3 @@
 module Kurosawa
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kurosawa
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - astrobunny
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-17 00:00:00.000000000 Z
+date: 2016-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,25 +52,84 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fakes3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: sinatra-contrib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.4
 description: Tribute to Kurosawa Ruby
 email:
 - admin@astrobunny.com
-executables: []
+executables:
+- kurosawa
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".rspec"
-- ".travis.yml"
 - Gemfile
 - README.md
-- Rakefile
 - bin/console
 - bin/setup
-- kurosawa.gemspec
+- exe/kurosawa
 - lib/kurosawa.rb
+- lib/kurosawa/connection.rb
+- lib/kurosawa/database.rb
+- lib/kurosawa/filesystems.rb
+- lib/kurosawa/filesystems/base.rb
+- lib/kurosawa/filesystems/local.rb
+- lib/kurosawa/filesystems/s3.rb
 - lib/kurosawa/version.rb
-homepage: https://github.com/astrobunny/kurosawa
+homepage: https://github.com/astrobunny/kurosawa.rb
 licenses: []
 metadata:
   allowed_push_host: https://rubygems.org
@@ -90,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: kurosawa.rb

data/.gitignore

@@ -1,9 +0,0 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/

data/.rspec

@@ -1,2 +0,0 @@
---format documentation
---color

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.3.0
-before_install: gem install bundler -v 1.12.4

data/Rakefile

@@ -1,6 +0,0 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
-
-RSpec::Core::RakeTask.new(:spec)
-
-task :default => :spec

data/kurosawa.gemspec

@@ -1,32 +0,0 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'kurosawa/version'
-
-Gem::Specification.new do |spec|
-  spec.name          = "kurosawa"
-  spec.version       = Kurosawa::VERSION
-  spec.authors       = ["astrobunny"]
-  spec.email         = ["admin@astrobunny.com"]
-
-  spec.summary       = %q{kurosawa.rb}
-  spec.description   = %q{Tribute to Kurosawa Ruby}
-  spec.homepage      = "https://github.com/astrobunny/kurosawa"
-
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
-  # to allow pushing to a single host or delete this section to allow pushing to any host.
-  if spec.respond_to?(:metadata)
-    spec.metadata['allowed_push_host'] = "https://rubygems.org"
-  else
-    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
-  end
-
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.12"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-end