thingfish 0.5.0.pre20161103181816 → 0.5.0

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org/"
+gemspec

data/History.rdoc

@@ -1,5 +1,7 @@
-== v0.5.0 [2011-01-18] Michael Granger <ged@FaerieMUD.org>
+== v0.5.0 [2016-11-14] Michael Granger <ged@FaerieMUD.org>
+
+First public release.
+
 
-Rewritten to use Strelka.
 
 

data/Manifest.txt

@@ -22,7 +22,6 @@ lib/thingfish/metastore.rb
 lib/thingfish/metastore/memory.rb
 lib/thingfish/mixins.rb
 lib/thingfish/processor.rb
-lib/thingfish/processor/mp3.rb
 lib/thingfish/processor/sha256.rb
 lib/thingfish/processordaemon.rb
 lib/thingfish/spechelpers.rb
@@ -38,7 +37,6 @@ spec/thingfish/handler_spec.rb
 spec/thingfish/metastore/memory_spec.rb
 spec/thingfish/metastore_spec.rb
 spec/thingfish/mixins_spec.rb
-spec/thingfish/processor/mp3_spec.rb
 spec/thingfish/processor/sha256_spec.rb
 spec/thingfish/processor_spec.rb
 spec/thingfish_spec.rb

data/README.rdoc

@@ -26,19 +26,23 @@ fetch it, all through a REST API.
 
 === Requirements
 
-Thingfish is written in ruby, and is tested using version 2.0.0.
-
-  * Ruby (>= 2.0.0): http://www.ruby-lang.org/en/downloads/
-
-Other versions may work, but are not tested.
+Thingfish is written in ruby, and is tested using {version 2.3}[http://www.ruby-lang.org/en/downloads/]. Other versions may work,
+but are not tested.
 
 === Ruby Modules
 
 You can install Thingfish via the Rubygems package manager:
 
-  $ sudo gem install thingfish
+  $ gem install thingfish
+
+This will install the basic server and its dependencies. Additional functionality is available via separate gems in the following namespaces:
 
-This will install the basic server and its dependencies. You can also install a collection of useful plugins via the 'thingfish-plugins' gem.
+[thingfish-metastore-*]
+  Storage backends for resource metadata
+[thingfish-filestore-*]
+  Storage backends for resources themselves
+[thingfish-processor-*]
+  Filters and extractors for resources
 
 
 == Contributing
@@ -61,7 +65,7 @@ You can submit bug reports, suggestions, and read more about future plans at
 
 == License
 
-Copyright (c) 2007-2014, Michael Granger and Mahlon E. Smith
+Copyright (c) 2007-2016, Michael Granger and Mahlon E. Smith
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

data/Rakefile

@@ -26,12 +26,11 @@ hoespec = Hoe.spec 'thingfish' do
 	self.developer 'Mahlon E. Smith', 'mahlon@martini.nu'
 	self.license "BSD"
 
-	self.dependency 'strelka',         '~> 0.9'
-	self.dependency 'mongrel2',        '~> 0.43'
+	self.dependency 'strelka',      '~> 0.9'
+	self.dependency 'mongrel2',     '~> 0.46'
 
-	self.dependency 'hoe-deveiate',    '~> 0.3',  :development
-	self.dependency 'simplecov',       '~> 0.7',  :development
-	self.dependency 'ruby-mp3info',    '~> 0.8',  :development
+	self.dependency 'hoe-deveiate', '~> 0.3',  :development
+	self.dependency 'simplecov',    '~> 0.7',  :development
 
 	self.require_ruby_version( '>=2.0.0' )
 

data/etc/mongrel2-config.rb

@@ -0,0 +1,57 @@
+# -*- ruby -*-
+# vim: set nosta noet ts=4 sw=4:
+#encoding: utf-8
+
+#
+# This script generates a Mongrel2 configuration database suitable for
+# getting a Thingfish handler running.
+#
+# Load it with:
+#
+#     $ m2sh.rb -c mongrel2.sqlite load examples/mongrel2-config.rb
+#
+# Afterwards, ensure the path to the mongrel2.sqlite file is in your
+# thingfish.conf:
+#
+#     mongrel2:
+#       configdb: /path/to/mongrel2.sqlite
+#
+# ... and start the mongrel2 daemon:
+#
+#     $ mongrel2 /path/to/mongrel2.sqlite thingfish
+#
+# In production use, you'll likely want to mount the Thingfish handler
+# within the URI space of an existing Mongrel2 environment.
+#
+
+require 'mongrel2'
+require 'mongrel2/config/dsl'
+
+server 'thingfish' do
+	name         'Thingfish'
+	default_host 'localhost'
+
+	access_log   'logs/access.log'
+	error_log    'logs/error.log'
+	chroot       ''
+	pid_file     'run/mongrel2.pid'
+
+	bind_addr    '0.0.0.0'
+	port         3474
+
+	xrequest     '/usr/local/lib/mongrel2/filters/sendfile.so'
+
+	host 'localhost' do
+		route '/', handler( 'tcp://127.0.0.1:9900', 'thingfish' )
+	end
+end
+
+setting 'zeromq.threads', 1
+setting 'limits.content_length', 250_000
+setting 'server.daemonize', false
+setting 'upload.temp_store', 'var/uploads/mongrel2.upload.XXXXXX'
+
+mkdir_p 'var/uploads'
+mkdir_p 'run'
+mkdir_p 'logs'
+

data/lib/thingfish.rb

@@ -2,18 +2,7 @@
 
 require 'loggability'
 
-#
 # Network-accessable datastore service
-#
-# == Version
-#
-#  $Id: thingfish.rb,v ce172208b523 2013/11/20 02:21:12 ged $
-#
-# == Authors
-#
-# * Michael Granger <ged@FaerieMUD.org>
-# * Mahlon E. Smith <mahlon@martini.nu>
-#
 module Thingfish
 	extend Loggability
 
@@ -26,7 +15,7 @@ module Thingfish
 	VERSION = '0.5.0'
 
 	# Version control revision
-	REVISION = %q$Revision: ce172208b523 $
+	REVISION = %q$Revision: 3607a807e2bf $
 
 
 	### Get the library version. If +include_buildnum+ is true, the version string will

data/lib/thingfish/handler.rb

@@ -193,7 +193,7 @@ class Thingfish::Handler < Strelka::App
 		"The name(s) of the fields to order results by."
 	param :direction, /^(asc|desc)$/i, "The order direction (ascending or descending)"
 	param :casefold, :boolean, "Whether or not to convert to lowercase before matching"
-	param :relationship, :word, "The name of the relationship between two resources"
+	param :relationship, /^[\w\-]+$/, "The name of the relationship between two resources"
 
 
 	#
@@ -316,6 +316,9 @@ class Thingfish::Handler < Strelka::App
 
 		finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid )
 
+		primary_metadata = self.metastore.fetch( uuid )
+		self.check_resource_permissions( req, uuid, primary_metadata )
+
 		criteria = {
 			'relation' => uuid,
 			'relationship' => rel,
@@ -328,6 +331,9 @@ class Thingfish::Handler < Strelka::App
 		metadata = self.metastore.fetch( uuid )
 
 		res = req.response
+		self.add_etag_headers( req, metadata )
+		self.add_content_disposition( res, metadata )
+
 		res.body = object
 		res.content_type = metadata['format']
 
@@ -349,6 +355,7 @@ class Thingfish::Handler < Strelka::App
 		res.content_type = metadata['format']
 
 		self.add_etag_headers( req, metadata )
+		self.add_content_disposition( res, metadata )
 
 		if object.respond_to?( :path )
 			path = Pathname( object.path )
@@ -411,9 +418,9 @@ class Thingfish::Handler < Strelka::App
 
 		self.check_resource_permissions( req, uuid )
 
-		self.datastore.remove( uuid ) or finish_with( HTTP::NOT_FOUND, "No such object." )
-		metadata = self.metastore.remove( uuid )
 		self.remove_related_resources( uuid )
+		metadata = self.metastore.remove( uuid )
+		self.datastore.remove( uuid ) or finish_with( HTTP::NOT_FOUND, "No such object." )
 		self.send_event( :deleted, :uuid => uuid )
 
 		res = req.response
@@ -441,8 +448,7 @@ class Thingfish::Handler < Strelka::App
 
 		finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid )
 
-		metadata = self.metastore.fetch( uuid )
-		metadata[ 'oid' ] = uuid
+		metadata = self.normalized_metadata_for( uuid )
 		self.check_resource_permissions( req, uuid, metadata )
 
 		res = req.response
@@ -538,13 +544,17 @@ class Thingfish::Handler < Strelka::App
 		metadata = self.metastore.fetch( uuid )
 		self.check_resource_permissions( req, uuid, metadata )
 
-		op_metadata = self.metastore.fetch( uuid, *OPERATIONAL_METADATA_KEYS )
+		op_metadata  = self.metastore.fetch( uuid, *OPERATIONAL_METADATA_KEYS )
 		new_metadata = self.extract_metadata( req )
-		self.metastore.save( uuid, new_metadata.merge(op_metadata) )
+		self.metastore.transaction do
+			self.metastore.remove_except( uuid, *OPERATIONAL_METADATA_KEYS )
+			self.metastore.merge( uuid, new_metadata.merge(op_metadata) )
+		end
 		self.send_event( :metadata_replaced, :uuid => uuid )
 
 		res = req.response
-		res.status = HTTP::NO_CONTENT
+		res.status = HTTP::OK
+		res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) }
 
 		return res
 	end
@@ -565,7 +575,8 @@ class Thingfish::Handler < Strelka::App
 		self.send_event( :metadata_updated, :uuid => uuid )
 
 		res = req.response
-		res.status = HTTP::NO_CONTENT
+		res.status = HTTP::OK
+		res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) }
 
 		return res
 	end
@@ -585,7 +596,8 @@ class Thingfish::Handler < Strelka::App
 		self.send_event( :metadata_deleted, :uuid => uuid )
 
 		res = req.response
-		res.status = HTTP::NO_CONTENT
+		res.status = HTTP::OK
+		res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) }
 
 		return res
 	end
@@ -626,8 +638,8 @@ class Thingfish::Handler < Strelka::App
 		metadata.merge!( self.extract_header_metadata(request) )
 		metadata.merge!( self.extract_default_metadata(request) )
 
-		self.verify_operational_metadata( metadata )
 		self.check_resource_permissions( request, uuid, metadata )
+		self.verify_operational_metadata( metadata )
 
 		if uuid
 			self.log.info "Replacing resource %s (encoding: %p)" %
@@ -722,15 +734,7 @@ class Thingfish::Handler < Strelka::App
 	### Extract and validate supplied metadata from the +request+.
 	def extract_metadata( req )
 		new_metadata = req.params.fields.dup
-		new_metadata.delete( :uuid )
-
-		protected_keys = OPERATIONAL_METADATA_KEYS & new_metadata.keys
-
-		unless protected_keys.empty?
-			finish_with HTTP::FORBIDDEN,
-				"Unable to alter protected metadata. (%s)" % [ protected_keys.join(', ') ]
-		end
-
+		new_metadata = self.remove_operational_metadata( new_metadata )
 		return new_metadata
 	end
 
@@ -750,6 +754,13 @@ class Thingfish::Handler < Strelka::App
 	end
 
 
+	### Fetch the current metadata for +uuid+, altering it for easier
+	### round trips with REST.
+	def normalized_metadata_for( uuid )
+		return self.metastore.fetch(uuid).merge( 'uuid' => uuid )
+	end
+
+
 	### Check that the metadata provided contains valid values for
 	### the operational keys, before saving a resource to disk.
 	def verify_operational_metadata( metadata )
@@ -759,6 +770,13 @@ class Thingfish::Handler < Strelka::App
 	end
 
 
+	### Prune operational +metadata+ from the provided hash.
+	def remove_operational_metadata( metadata )
+		operationals = OPERATIONAL_METADATA_KEYS + [ 'uuid' ]
+		return metadata.reject{|key, _| operationals.include?(key) }
+	end
+
+
 	### Send an event of +type+ with the given +msg+ over the zmq event socket.
 	def send_event( type, msg )
 		esock = self.event_socket or return
@@ -785,6 +803,15 @@ class Thingfish::Handler < Strelka::App
 	end
 
 
+	### Add a filename "hint" for browsers, if the resource being fetched
+	### has a 'title' attribute.
+	def add_content_disposition( res, metadata )
+		return unless metadata[ 'title' ]
+		title = metadata[ 'title' ].encode( 'us-ascii', :undef => :replace )
+		res.headers[ :content_disposition ] = "filename=%p" % [ title ]
+	end
+
+
 	### Supply a method that child handlers can override.  The regular auth
 	### plugin runs too early (but can also be used), this hook allows
 	### child handlers to make access decisions based on the +request+

data/lib/thingfish/processor/sha256.rb

@@ -0,0 +1,51 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'digest/sha2'
+
+require 'thingfish' unless defined?( Thingfish )
+require 'thingfish/processor' unless defined?( Thingfish::Processor )
+
+
+# Calculate and store a sha256 checksum for a resource.
+class Thingfish::Processor::SHA256 < Thingfish::Processor
+	extend Loggability
+
+	# The chunk size to read
+	CHUNK_SIZE = 32 * 1024
+
+	# Loggability API -- log to the :thingfish logger
+	log_to :thingfish
+
+	# The list of handled types
+	handled_types '*/*'
+
+
+	### Synchronous processor API -- generate a checksum during upload.
+	def on_request( request )
+		request.add_metadata( :checksum => self.checksum(request.body) )
+		request.related_resources.each_pair do |io, metadata|
+			metadata[ :checksum ] = self.checksum( io )
+		end
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Given an +io+, return a sha256 checksum of it's contents.
+	def checksum( io )
+		digest = Digest::SHA256.new
+		buf = ''
+
+		while io.read( CHUNK_SIZE, buf )
+			digest.update( buf )
+		end
+
+		io.rewind
+		return digest.hexdigest
+	end
+
+end # class Thingfish::Processor::SHA256
+

data/spec/thingfish/handler_spec.rb

@@ -277,7 +277,9 @@ describe Thingfish::Handler do
 				'format'       => 'image/png',
 				'extent'       => @png_io.string.bytesize,
 				'relation'     => main_uuid,
-				'relationship' => "twinsies"
+				'relationship' => "twinsies",
+				'title'        => 'Make America Smart Again.png',
+				'checksum'     => '123456'
 			})
 
 			req = factory.get( "/#{main_uuid}/related/twinsies" )
@@ -285,6 +287,8 @@ describe Thingfish::Handler do
 
 			expect( res.status_line ).to match( /200 ok/i )
 			expect( res.headers.content_type ).to eq( 'image/png' )
+			expect( res.headers.etag ).to eq( '123456' )
+			expect( res.headers.content_disposition ).to eq( 'filename="Make America Smart Again.png"' )
 			expect( res.body.read ).to eq( TEST_PNG_DATA )
 		end
 
@@ -367,6 +371,20 @@ describe Thingfish::Handler do
 		end
 
 
+		it "adds content disposition filename, if the resource has a title" do
+			uuid = @handler.datastore.save( @png_io )
+			@handler.metastore.save( uuid, {'format' => 'image/png', 'title' => 'spょler"py.txt'} )
+
+			req = factory.get( "/#{uuid}" )
+			result = @handler.handle( req )
+
+			expect( result.status_line ).to match( /200 ok/i )
+			expect( result.body.read ).to eq( @png_io.string )
+			expect( result.headers.content_type ).to eq( 'image/png' )
+			expect( result.headers.content_disposition ).to eq( 'filename="sp?ler\"py.txt"' )
+		end
+
+
 		it "returns a 304 not modified for unchanged client cache requests" do
 			uuid = @handler.datastore.save( @png_io )
 			@handler.metastore.save( uuid, 'format' => 'image/png', 'checksum' => '123456' )
@@ -433,7 +451,7 @@ describe Thingfish::Handler do
 			expect( result.status ).to eq( 200 )
 			expect( result.headers.content_type ).to eq( 'application/json' )
 			expect( content_hash ).to be_a( Hash )
-			expect( content_hash['oid'] ).to eq( uuid )
+			expect( content_hash['uuid'] ).to eq( uuid )
 			expect( content_hash['extent'] ).to eq( 288 )
 			expect( content_hash['created'] ).to eq( Time.at(1378313840).to_s )
 		end
@@ -496,19 +514,20 @@ describe Thingfish::Handler do
 			uuid = @handler.datastore.save( @png_io )
 			@handler.metastore.save( uuid, {
 				'format' => 'image/png',
-				'extent' => 288,
+				'extent' => 288
 			})
 
-			body_json = Yajl.dump({ 'comment' => 'Ignore me!' })
+			body_json = Yajl.dump({ 'comment' => 'Ignore me!', 'uuid' => 123 })
 			req = factory.post( "/#{uuid}/metadata", body_json, 'Content-type' => 'application/json' )
 			result = @handler.handle( req )
 
-			expect( result.status ).to eq( HTTP::NO_CONTENT )
+			expect( result.status ).to eq( HTTP::OK )
 			expect( @handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'Ignore me!' )
+			expect( @handler.metastore.fetch_value(uuid, 'uuid') ).to be_nil
 		end
 
 
-		it "returns FORBIDDEN when attempting to merge metadata with operational keys" do
+		it "ignores attempts to alter operational metadata when merging" do
 			uuid = @handler.datastore.save( @png_io )
 			@handler.metastore.save( uuid, {
 				'format' => 'image/png',
@@ -519,10 +538,8 @@ describe Thingfish::Handler do
 			req = factory.post( "/#{uuid}/metadata", body_json, 'Content-type' => 'application/json' )
 			result = @handler.handle( req )
 
-			expect( result.status ).to eq( HTTP::FORBIDDEN )
-			expect( result.body.string ).to match( /unable to alter protected metadata/i )
-			expect( result.body.string ).to match( /format/i )
-			expect( @handler.metastore.fetch_value(uuid, 'comment') ).to be_nil
+			expect( result.status ).to eq( HTTP::OK )
+			expect( @handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'Ignore me!' )
 			expect( @handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' )
 		end
 
@@ -630,7 +647,7 @@ describe Thingfish::Handler do
 			                   'Content-type' => 'application/json' )
 			result = @handler.handle( req )
 
-			expect( result.status ).to eq( HTTP::NO_CONTENT )
+			expect( result.status ).to eq( HTTP::OK )
 			expect( @handler.metastore.fetch_value(uuid, 'comment') ).to eq( 'Yeah.' )
 			expect( @handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' )
 			expect( @handler.metastore ).to_not include( 'ephemeral' )
@@ -652,8 +669,8 @@ describe Thingfish::Handler do
 			req = factory.delete( "/#{uuid}/metadata" )
 			result = @handler.handle( req )
 
-			expect( result.status ).to eq( HTTP::NO_CONTENT )
-			expect( result.body.string ).to be_empty
+			expect( result.status ).to eq( HTTP::OK )
+			expect( result.body.string ).to_not be_empty
 			expect( @handler.metastore.fetch_value(uuid, 'format') ).to eq( 'image/png' )
 			expect( @handler.metastore.fetch_value(uuid, 'extent') ).to eq( 288 )
 			expect( @handler.metastore.fetch_value(uuid, 'uploadaddress') ).to eq( '127.0.0.1' )