thingfish 0.5.0.pre20161103181816 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 74a092349d627e3de7578603e6cbba04c29f8045
-  data.tar.gz: eb4868b63ef6b8ad8b95c4ff5954f4482924bb3d
+SHA256:
+  metadata.gz: 767d737d3f8a3ca3cae2d7f5b01fa7e7b1967127585efe3201b55fce50a46500
+  data.tar.gz: 769440ac44975755940720e4cd95a610c072874e3c5a1f709bcf3e7cf6cabf39
 SHA512:
-  metadata.gz: 6e3a206fa9f2ad2dd07543f407f85972a42b4f073ae8cc3eab66ead4ec5947b74c6f4228741a96efdb15a22101c4251cc49cee7a6e4b5720a1bf5acefda41967
-  data.tar.gz: a923141975987b85e96e84bd20c017d06609110abf10e319bd76a61ef51be1f0b6e59e2c66d8bc5d3d4d0275c862d8a3826b2e6c17f844daf456d6b7cfb21d91
+  metadata.gz: a83f80a9b4efa7c7e651f2fd1791e4424fa2bd63d6b304b4cc622cfcfc1fd2840ffd1fee370722cc4399453ac662efffc2591e51e054acb33e8e72e776d7a30c
+  data.tar.gz: d6244bb6e4b2c340f977f12ed0f19631bbdd6a10815973c44880c0ebbc8c841b6296744f7cb331ed71a776e26c2d7f24cac136b510afe1eceebe0eb110ff822c

data/History.md

@@ -0,0 +1,48 @@
+# Release History for thingfish
+
+---
+
+## v0.8.0 [2021-01-02] Michael Granger <ged@FaerieMUD.org>
+
+Improvements:
+
+- Several bugfixes
+- Support for newer Rubies
+
+
+## v0.7.0 [2017-09-13] Michael Granger <ged@FaerieMUD.org>
+
+Improvements:
+
+- Explicitly set the HTTP status when returning resources.
+- Add Last-Modified/If-None-Match support.
+- Move the configure() method to help support deferred configuration.
+
+
+## v0.6.0 [2017-01-16] Mahlon E. Smith <mahlon@martini.nu>
+
+Housekeeping:
+
+- Bump Configurability dependency.
+- Migrate away from .rvmrc.
+
+
+## v0.5.1 [2016-11-14] Michael Granger <ged@FaerieMUD.org>
+
+Enhancements:
+
+- Add support for Strelka app discovery.
+
+Bugfixes:
+
+- Remove the processor daemon for now
+- Documentation fixes
+
+
+## v0.5.0 [2016-11-14] Michael Granger <ged@FaerieMUD.org>
+
+First public release.
+
+
+
+

data/{README.rdoc → README.md}

@@ -1,8 +1,16 @@
-= Thingfish
+# Thingfish
 
-* http://bitbucket.org/ged/thingfish
+home
+: https://thing.fish
 
-== Description
+code
+: https://hg.sr.ht/~ged/thingfish
+
+docs
+: https://thing.fish/docs/
+
+
+## Description
 
 Thingfish is a extensible, web-based digital asset manager. It can be used to
 store chunks of data on the network in an application-independent way, link the
@@ -10,38 +18,31 @@ chunks together with metadata, and then search for the chunk you need later and
 fetch it, all through a REST API.
 
 
-== Authors
-
-* Michael Granger <ged@FaerieMUD.org>
-* Mahlon E. Smith <mahlon@martini.nu>
-
-== Contributors
-
-* Jeremiah Jordan <phaedrus@perlreason.com>
-* Ben Bleything <ben@bleything.net>
-* Jeff Davis <jeff-thingfish@j-davis.com>
-
+## Requirements
 
-== Installation
+Thingfish is written in ruby, and is tested using [version 2.7](http://www.ruby-lang.org/en/downloads/). Other versions may work,
+but are not tested.
 
-=== Requirements
 
-Thingfish is written in ruby, and is tested using version 2.0.0.
+## Installation
 
-  * Ruby (>= 2.0.0): http://www.ruby-lang.org/en/downloads/
+You can install Thingfish via Rubygems:
 
-Other versions may work, but are not tested.
+    $ gem install thingfish
 
-=== Ruby Modules
+This will install the basic server and its dependencies. Additional functionality is available via separate gems in the following namespaces:
 
-You can install Thingfish via the Rubygems package manager:
+`thingfish-metastore-*`
+: Storage backends for resource metadata
 
-  $ sudo gem install thingfish
+`thingfish-filestore-*`
+: Storage backends for resources themselves
 
-This will install the basic server and its dependencies. You can also install a collection of useful plugins via the 'thingfish-plugins' gem.
+`thingfish-processor-*`
+: Filters and extractors for resources
 
 
-== Contributing
+## Contributing
 
 You can check out the current development source
 {with Mercurial}[http://bitbucket.org/ged/thingfish], or
@@ -59,9 +60,22 @@ You can submit bug reports, suggestions, and read more about future plans at
 {the project page}[http://bitbucket.org/ged/thingfish].
 
 
-== License
+## Authors
+
+* Michael Granger <ged@FaerieMUD.org>
+* Mahlon E. Smith <mahlon@martini.nu>
+
+
+## Contributors
+
+* Jeremiah Jordan <phaedrus@perlreason.com>
+* Ben Bleything <ben@bleything.net>
+* Jeff Davis <jeff-thingfish@j-davis.com>
+
+
+## License
 
-Copyright (c) 2007-2014, Michael Granger and Mahlon E. Smith
+Copyright (c) 2007-2020, Michael Granger and Mahlon E. Smith
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

data/Rakefile

@@ -1,92 +1,8 @@
-#!/usr/bin/env rake
-#encoding: utf-8
+#!/usr/bin/env ruby -S rake
 
-begin
-	require 'hoe'
-rescue LoadError
-	abort "This Rakefile requires hoe (gem install hoe)"
-end
-
-GEMSPEC = 'thingfish.gemspec'
-
-
-Hoe.plugin :mercurial
-Hoe.plugin :signing
-Hoe.plugin :deveiate
-
-Hoe.plugins.delete :rubyforge
-
-hoespec = Hoe.spec 'thingfish' do
-	self.readme_file = 'README.rdoc'
-	self.history_file = 'History.rdoc'
-	self.extra_rdoc_files = Rake::FileList[ '*.rdoc' ]
-	self.license 'BSD'
-
-	self.developer 'Michael Granger', 'ged@FaerieMUD.org'
-	self.developer 'Mahlon E. Smith', 'mahlon@martini.nu'
-	self.license "BSD"
-
-	self.dependency 'strelka',         '~> 0.9'
-	self.dependency 'mongrel2',        '~> 0.43'
-
-	self.dependency 'hoe-deveiate',    '~> 0.3',  :development
-	self.dependency 'simplecov',       '~> 0.7',  :development
-	self.dependency 'ruby-mp3info',    '~> 0.8',  :development
-
-	self.require_ruby_version( '>=2.0.0' )
-
-	self.hg_sign_tags = true if self.respond_to?( :hg_sign_tags= )
-	spec.check_history_on_release = true if spec.respond_to?( :check_history_on_release= )
-
-	self.rdoc_locations << "deveiate:/usr/local/www/public/code/#{remote_rdoc_dir}"
-end
-
-ENV['VERSION'] ||= hoespec.spec.version.to_s
-
-# Run the tests before checking in
-task 'hg:precheckin' => [ :check_history, :check_manifest, :gemspec, :spec ]
-
-task :test => :spec
-
-# Rebuild the ChangeLog immediately before release
-task :prerelease => 'ChangeLog'
-CLOBBER.include( 'ChangeLog' )
-
-desc "Build a coverage report"
-task :coverage do
-	ENV["COVERAGE"] = 'yes'
-	Rake::Task[:spec].invoke
-end
-CLOBBER.include( 'coverage' )
-
-
-# Use the fivefish formatter for docs generated from development checkout
-if File.directory?( '.hg' )
-	require 'rdoc/task'
+require 'rake/deveiate'
 
-	Rake::Task[ 'docs' ].clear
-	RDoc::Task.new( 'docs' ) do |rdoc|
-	    rdoc.main = "README.rdoc"
-		rdoc.markup = 'markdown'
-	    rdoc.rdoc_files.include( "*.rdoc", "ChangeLog", "lib/**/*.rb" )
-	    rdoc.generator = :fivefish
-		rdoc.title = 'Thingfish'
-	    rdoc.rdoc_dir = 'doc'
-	end
+Rake::DevEiate.setup( 'thingfish' ) do |project|
+	project.publish_to = 'deveiate:/usr/local/www/public/code'
 end
 
-task :gemspec => GEMSPEC
-file GEMSPEC => __FILE__
-task GEMSPEC do |task|
-	spec = $hoespec.spec
-	spec.files.delete( '.gemtest' )
-	spec.signing_key = nil
-	spec.cert_chain = ['certs/ged.pem']
-	spec.version = "#{spec.version}.pre#{Time.now.strftime("%Y%m%d%H%M%S")}"
-	File.open( task.name, 'w' ) do |fh|
-		fh.write( spec.to_ruby )
-	end
-end
-
-CLOBBER.include( GEMSPEC.to_s )
-

data/bin/thingfish

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-#encoding: utf-8
+# frozen_string_literal: true
 
 require 'strelka'
 require 'thingfish/handler'

data/lib/strelka/app/metadata.rb

@@ -1,5 +1,5 @@
 # -*- ruby -*-
-#encoding: utf-8
+# frozen_string_literal: true
 
 require 'strelka'
 require 'strelka/plugins'

data/lib/strelka/apps.rb

@@ -0,0 +1,9 @@
+# -*- ruby -*-
+# frozen_string_literal: true
+
+require 'strelka/discovery'
+
+Strelka::Discovery.register_apps(
+	'thingfish' => 'thingfish/handler.rb'
+)
+

data/lib/strelka/httprequest/metadata.rb

@@ -1,6 +1,6 @@
 # -*- ruby -*-
 # vim: set nosta noet ts=4 sw=4:
-# encoding: utf-8
+# frozen_string_literal: true
 
 require 'strelka/constants'
 require 'strelka/httprequest' unless defined?( Strelka::HTTPRequest )

data/lib/thingfish.rb

@@ -2,18 +2,7 @@
 
 require 'loggability'
 
-#
 # Network-accessable datastore service
-#
-# == Version
-#
-#  $Id: thingfish.rb,v ce172208b523 2013/11/20 02:21:12 ged $
-#
-# == Authors
-#
-# * Michael Granger <ged@FaerieMUD.org>
-# * Mahlon E. Smith <mahlon@martini.nu>
-#
 module Thingfish
 	extend Loggability
 
@@ -23,10 +12,10 @@ module Thingfish
 
 
 	# Package version
-	VERSION = '0.5.0'
+	VERSION = '0.8.0'
 
 	# Version control revision
-	REVISION = %q$Revision: ce172208b523 $
+	REVISION = %q$Revision$
 
 
 	### Get the library version. If +include_buildnum+ is true, the version string will

data/lib/thingfish/behaviors.rb

@@ -1,5 +1,5 @@
 # -*- ruby -*-
-#encoding: utf-8
+# frozen_string_literal: true
 
 require 'rspec'
 

data/lib/thingfish/datastore.rb

@@ -1,5 +1,5 @@
 # -*- ruby -*-
-#encoding: utf-8
+# frozen_string_literal: true
 
 require 'securerandom'
 require 'pluggability'

data/lib/thingfish/datastore/memory.rb

@@ -1,5 +1,5 @@
 # -*- ruby -*-
-#encoding: utf-8
+# frozen_string_literal: true
 
 require 'thingfish' unless defined?( Thingfish )
 require 'thingfish/datastore' unless defined?( Thingfish::Datastore )

data/lib/thingfish/handler.rb

@@ -1,5 +1,5 @@
 # -*- ruby -*-
-#encoding: utf-8
+# frozen_string_literal: true
 
 require 'strelka'
 require 'strelka/app'
@@ -74,22 +74,6 @@ class Thingfish::Handler < Strelka::App
 	singleton_attr_accessor :processors
 
 
-	### Configurability API -- install the configuration
-	def self::configure( config=nil )
-		config = self.defaults.merge( config || {} )
-
-		self.datastore        = config[:datastore]
-		self.metastore        = config[:metastore]
-		self.event_socket_uri = config[:event_socket_uri]
-
-		self.processors = self.load_processors( config[:processors] )
-		self.processors.each do |processor|
-			self.filter( :request, &processor.method(:process_request) )
-			self.filter( :response, &processor.method(:process_response) )
-		end
-	end
-
-
 	### Load the Thingfish::Processors in the given +processor_list+ and return an instance
 	### of each one.
 	def self::load_processors( processor_list )
@@ -110,6 +94,24 @@ class Thingfish::Handler < Strelka::App
 	end
 
 
+	### Configurability API -- install the configuration
+	def self::configure( config=nil )
+		config = self.defaults.merge( config || {} )
+
+		self.datastore        = config[:datastore]
+		self.metastore        = config[:metastore]
+		self.event_socket_uri = config[:event_socket_uri]
+
+		self.plugin( :filters ) # pre-load the filters plugin for deferred config
+
+		self.processors = self.load_processors( config[:processors] )
+		self.processors.each do |processor|
+			self.filter( :request, &processor.method(:process_request) )
+			self.filter( :response, &processor.method(:process_response) )
+		end
+	end
+
+
 	### Set up the metastore, datastore, and event socket when the handler is
 	### created.
 	def initialize( * ) # :notnew:
@@ -145,8 +147,8 @@ class Thingfish::Handler < Strelka::App
 	### Set up the event socket.
 	def setup_event_socket
 		if self.class.event_socket_uri && ! @event_socket
-			@event_socket = Mongrel2.zmq_context.socket( :PUB )
-			@event_socket.linger = 0
+			@event_socket = CZTop::Socket::PUB.new
+			@event_socket.options.linger = 0
 			@event_socket.bind( self.class.event_socket_uri )
 		end
 	end
@@ -193,7 +195,7 @@ class Thingfish::Handler < Strelka::App
 		"The name(s) of the fields to order results by."
 	param :direction, /^(asc|desc)$/i, "The order direction (ascending or descending)"
 	param :casefold, :boolean, "Whether or not to convert to lowercase before matching"
-	param :relationship, :word, "The name of the relationship between two resources"
+	param :relationship, /^[\w\-]+$/, "The name of the relationship between two resources"
 
 
 	#
@@ -316,6 +318,9 @@ class Thingfish::Handler < Strelka::App
 
 		finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid )
 
+		primary_metadata = self.metastore.fetch( uuid )
+		self.check_resource_permissions( req, uuid, primary_metadata )
+
 		criteria = {
 			'relation' => uuid,
 			'relationship' => rel,
@@ -328,8 +333,12 @@ class Thingfish::Handler < Strelka::App
 		metadata = self.metastore.fetch( uuid )
 
 		res = req.response
+		self.add_cache_headers( req, metadata )
+		self.add_content_disposition( res, metadata )
+
 		res.body = object
 		res.content_type = metadata['format']
+		res.status = HTTP::OK
 
 		return res
 	end
@@ -348,7 +357,8 @@ class Thingfish::Handler < Strelka::App
 		res = req.response
 		res.content_type = metadata['format']
 
-		self.add_etag_headers( req, metadata )
+		self.add_cache_headers( req, metadata )
+		self.add_content_disposition( res, metadata )
 
 		if object.respond_to?( :path )
 			path = Pathname( object.path )
@@ -360,6 +370,7 @@ class Thingfish::Handler < Strelka::App
 			res.body = object
 		end
 
+		res.status = HTTP::OK
 		return res
 	end
 
@@ -411,9 +422,9 @@ class Thingfish::Handler < Strelka::App
 
 		self.check_resource_permissions( req, uuid )
 
-		self.datastore.remove( uuid ) or finish_with( HTTP::NOT_FOUND, "No such object." )
-		metadata = self.metastore.remove( uuid )
 		self.remove_related_resources( uuid )
+		metadata = self.metastore.remove( uuid )
+		self.datastore.remove( uuid ) or finish_with( HTTP::NOT_FOUND, "No such object." )
 		self.send_event( :deleted, :uuid => uuid )
 
 		res = req.response
@@ -441,8 +452,7 @@ class Thingfish::Handler < Strelka::App
 
 		finish_with( HTTP::NOT_FOUND, "No such object." ) unless self.metastore.include?( uuid )
 
-		metadata = self.metastore.fetch( uuid )
-		metadata[ 'oid' ] = uuid
+		metadata = self.normalized_metadata_for( uuid )
 		self.check_resource_permissions( req, uuid, metadata )
 
 		res = req.response
@@ -538,13 +548,17 @@ class Thingfish::Handler < Strelka::App
 		metadata = self.metastore.fetch( uuid )
 		self.check_resource_permissions( req, uuid, metadata )
 
-		op_metadata = self.metastore.fetch( uuid, *OPERATIONAL_METADATA_KEYS )
+		op_metadata  = self.metastore.fetch( uuid, *OPERATIONAL_METADATA_KEYS )
 		new_metadata = self.extract_metadata( req )
-		self.metastore.save( uuid, new_metadata.merge(op_metadata) )
+		self.metastore.transaction do
+			self.metastore.remove_except( uuid, *OPERATIONAL_METADATA_KEYS )
+			self.metastore.merge( uuid, new_metadata.merge(op_metadata) )
+		end
 		self.send_event( :metadata_replaced, :uuid => uuid )
 
 		res = req.response
-		res.status = HTTP::NO_CONTENT
+		res.status = HTTP::OK
+		res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) }
 
 		return res
 	end
@@ -565,7 +579,8 @@ class Thingfish::Handler < Strelka::App
 		self.send_event( :metadata_updated, :uuid => uuid )
 
 		res = req.response
-		res.status = HTTP::NO_CONTENT
+		res.status = HTTP::OK
+		res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) }
 
 		return res
 	end
@@ -585,7 +600,8 @@ class Thingfish::Handler < Strelka::App
 		self.send_event( :metadata_deleted, :uuid => uuid )
 
 		res = req.response
-		res.status = HTTP::NO_CONTENT
+		res.status = HTTP::OK
+		res.for( :json, :yaml ) { self.normalized_metadata_for(uuid) }
 
 		return res
 	end
@@ -626,8 +642,8 @@ class Thingfish::Handler < Strelka::App
 		metadata.merge!( self.extract_header_metadata(request) )
 		metadata.merge!( self.extract_default_metadata(request) )
 
-		self.verify_operational_metadata( metadata )
 		self.check_resource_permissions( request, uuid, metadata )
+		self.verify_operational_metadata( metadata )
 
 		if uuid
 			self.log.info "Replacing resource %s (encoding: %p)" %
@@ -702,9 +718,9 @@ class Thingfish::Handler < Strelka::App
 	### Return a Hash of default metadata extracted from the given +request+.
 	def extract_default_metadata( request )
 		return self.extract_connection_metadata( request ).merge(
-			'extent'        => request.headers.content_length,
-			'format'        => request.content_type,
-			'created'       => Time.now.getgm
+			'extent'  => request.headers.content_length,
+			'format'  => request.content_type,
+			'created' => Time.now.getgm
 		)
 	end
 
@@ -722,15 +738,7 @@ class Thingfish::Handler < Strelka::App
 	### Extract and validate supplied metadata from the +request+.
 	def extract_metadata( req )
 		new_metadata = req.params.fields.dup
-		new_metadata.delete( :uuid )
-
-		protected_keys = OPERATIONAL_METADATA_KEYS & new_metadata.keys
-
-		unless protected_keys.empty?
-			finish_with HTTP::FORBIDDEN,
-				"Unable to alter protected metadata. (%s)" % [ protected_keys.join(', ') ]
-		end
-
+		new_metadata = self.remove_operational_metadata( new_metadata )
 		return new_metadata
 	end
 
@@ -750,6 +758,13 @@ class Thingfish::Handler < Strelka::App
 	end
 
 
+	### Fetch the current metadata for +uuid+, altering it for easier
+	### round trips with REST.
+	def normalized_metadata_for( uuid )
+		return self.metastore.fetch(uuid).merge( 'uuid' => uuid )
+	end
+
+
 	### Check that the metadata provided contains valid values for
 	### the operational keys, before saving a resource to disk.
 	def verify_operational_metadata( metadata )
@@ -759,32 +774,58 @@ class Thingfish::Handler < Strelka::App
 	end
 
 
+	### Prune operational +metadata+ from the provided hash.
+	def remove_operational_metadata( metadata )
+		operationals = OPERATIONAL_METADATA_KEYS + [ 'uuid' ]
+		return metadata.reject{|key, _| operationals.include?(key) }
+	end
+
+
 	### Send an event of +type+ with the given +msg+ over the zmq event socket.
 	def send_event( type, msg )
 		esock = self.event_socket or return
 		self.log.debug "Publishing %p event: %p" % [ type, msg ]
-		esock.sendm( type.to_s )
-		esock.send( Yajl.dump(msg) )
+		esock << CZTop::Message.new([ type.to_s, Yajl.dump(msg) ])
 	end
 
 
-	### Add browser cache headers for resources.  This requires the sha256
+	### Add browser cache headers for resources.
+	### Last-Modified is always added.  ETag support requires the sha256
 	### processor plugin to be enabled for stored resources.
-	def add_etag_headers( request, metadata )
+	###
+	def add_cache_headers( request, metadata )
 		response = request.response
-		checksum = metadata[ 'checksum' ]
-		return unless checksum
 
-		if (( match = request.headers[ :if_none_match ] ))
-			match = match.gsub( '"', '' ).split( /,\s*/ )
-			finish_with( HTTP::NOT_MODIFIED ) if match.include?( checksum )
+		# ETag takes precedence if available.
+		#
+		if (( checksum = metadata['checksum'] ))
+			if (( match = request.headers[ :if_none_match ] ))
+				match = match.gsub( '"', '' ).split( /,\s*/ )
+				finish_with( HTTP::NOT_MODIFIED ) if match.include?( checksum )
+			end
+			response.headers[ :etag ] = checksum
+		end
+
+		return unless metadata[ 'created' ]
+
+		if (( modified = request.headers[ :if_modified_since ] ))
+			finish_with( HTTP::NOT_MODIFIED ) if Time.parse( modified ) <= metadata['created'].round
 		end
+		response.headers[ :last_modified ] = metadata[ 'created' ].httpdate
 
-		response.headers[ :etag ] = checksum
 		return
 	end
 
 
+	### Add a filename "hint" for browsers, if the resource being fetched
+	### has a 'title' attribute.
+	def add_content_disposition( res, metadata )
+		return unless metadata[ 'title' ]
+		title = metadata[ 'title' ].encode( 'us-ascii', :undef => :replace )
+		res.headers[ :content_disposition ] = "filename=%p" % [ title ]
+	end
+
+
 	### Supply a method that child handlers can override.  The regular auth
 	### plugin runs too early (but can also be used), this hook allows
 	### child handlers to make access decisions based on the +request+