thingfish 0.5.0.pre20160707192835

data/lib/thingfish.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require 'loggability'
+
+#
+# Network-accessable datastore service
+#
+# == Version
+#
+#  $Id: thingfish.rb,v ce172208b523 2013/11/20 02:21:12 ged $
+#
+# == Authors
+#
+# * Michael Granger <ged@FaerieMUD.org>
+# * Mahlon E. Smith <mahlon@martini.nu>
+#
+module Thingfish
+	extend Loggability
+
+
+	# Loggability API -- log all Thingfish-related stuff to a separate logger
+	log_as :thingfish
+
+
+	# Package version
+	VERSION = '0.5.0'
+
+	# Version control revision
+	REVISION = %q$Revision: ce172208b523 $
+
+
+	### Get the library version. If +include_buildnum+ is true, the version string will
+	### include the VCS rev ID.
+	def self::version_string( include_buildnum=false )
+		vstring = "%s %s" % [ self.name, VERSION ]
+		vstring << " (build %s)" % [ REVISION[/: ([[:xdigit:]]+)/, 1] || '0' ] if include_buildnum
+		return vstring
+	end
+
+
+end # module Thingfish
+
+# vim: set nosta noet ts=4 sw=4:

data/lib/thingfish/behaviors.rb

@@ -0,0 +1,263 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'rspec'
+
+require 'thingfish/handler'
+
+
+RSpec.shared_examples "a Thingfish metastore" do
+
+	let( :metastore ) do
+		Thingfish::Metastore.create( described_class )
+	end
+
+
+	it "can save and fetch data" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		expect( metastore.fetch(TEST_UUID) ).to eq( TEST_METADATA.first )
+		expect( metastore.fetch(TEST_UUID) ).to_not be( TEST_METADATA.first )
+	end
+
+
+	it "returns nil when fetching metadata for an object that doesn't exist" do
+		expect( metastore.fetch(TEST_UUID) ).to be_nil
+	end
+
+
+	it "doesn't care about the case of the UUID when saving and fetching data" do
+		metastore.save( TEST_UUID.downcase, TEST_METADATA.first.freeze )
+		expect( metastore.fetch(TEST_UUID) ).to eq( TEST_METADATA.first )
+	end
+
+
+	it "can fetch a single metadata value for a given oid" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( TEST_METADATA.first['format'] )
+		expect( metastore.fetch_value(TEST_UUID, :extent) ).to eq( TEST_METADATA.first['extent'] )
+	end
+
+
+	it "can fetch a slice of data for a given oid" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		expect( metastore.fetch(TEST_UUID, :format, :extent) ).to eq({
+			'format' => TEST_METADATA.first['format'],
+			'extent' => TEST_METADATA.first['extent'],
+		})
+	end
+
+
+	it "returns nil when fetching a slice of data for an object that doesn't exist" do
+		expect( metastore.fetch_value(TEST_UUID, :format) ).to be_nil
+	end
+
+
+	it "doesn't care about the case of the UUID when fetching data" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		expect( metastore.fetch_value(TEST_UUID.downcase, :format) ).to eq( TEST_METADATA.first['format'] )
+	end
+
+
+	it "can update data" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		metastore.merge( TEST_UUID, format: 'image/jpeg' )
+
+		expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( 'image/jpeg' )
+	end
+
+
+	it "doesn't care about the case of the UUID when updating data" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		metastore.merge( TEST_UUID.downcase, format: 'image/jpeg' )
+
+		expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( 'image/jpeg' )
+	end
+
+
+	it "can remove metadata for a UUID" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		metastore.remove( TEST_UUID )
+
+		expect( metastore.fetch(TEST_UUID) ).to be_nil
+	end
+
+
+	it "can remove a single key/value pair from the metadata for a UUID" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		metastore.remove( TEST_UUID, :useragent )
+
+		expect( metastore.fetch_value(TEST_UUID, :useragent) ).to be_nil
+	end
+
+
+	it "can truncate metadata not in a list of OIDs for a UUID" do
+		keys = Thingfish::Handler::OPERATIONAL_METADATA_KEYS
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		metastore.remove_except( TEST_UUID, *keys )
+
+		metadata = metastore.fetch( TEST_UUID )
+		expect( metadata.size ).to eq( keys.size )
+		expect( metadata.keys ).to include( *keys.map(&:to_s) )
+	end
+
+
+	it "knows if it has data for a given OID" do
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		expect( metastore ).to include( TEST_UUID )
+	end
+
+
+	it "knows how many objects it contains" do
+		expect( metastore.size ).to eq( 0 )
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		expect( metastore.size ).to eq( 1 )
+	end
+
+
+	it "knows how to fetch UUIDs for related resources" do
+		rel_uuid1  = SecureRandom.uuid
+		rel_uuid2  = SecureRandom.uuid
+		unrel_uuid = SecureRandom.uuid
+
+		metastore.save( TEST_UUID, TEST_METADATA.first )
+		metastore.save( rel_uuid1, TEST_METADATA[1].merge('relation' => TEST_UUID.downcase) )
+		metastore.save( rel_uuid2, TEST_METADATA[2].merge('relation' => TEST_UUID.downcase) )
+		metastore.save( unrel_uuid, TEST_METADATA[3] )
+
+		uuids = metastore.fetch_related_oids( TEST_UUID )
+
+		expect( uuids ).to include( rel_uuid1, rel_uuid2 )
+		expect( uuids ).to_not include( unrel_uuid )
+	end
+
+
+	context "with some uploaded metadata" do
+
+		before( :each ) do
+			@uuids = []
+			TEST_METADATA.each do |file|
+				uuid = SecureRandom.uuid
+				@uuids << uuid
+				metastore.save( uuid, file )
+			end
+		end
+
+
+		it "can fetch an array of all of its OIDs" do
+			expect( metastore.oids ).to eq( @uuids )
+		end
+
+		it "can iterate over each of the store's oids" do
+			uuids = []
+			metastore.each_oid {|u| uuids << u }
+
+			expect( uuids ).to eq( @uuids )
+		end
+
+		it "can provide an enumerator over each of the store's oids" do
+			expect( metastore.each_oid.to_a ).to eq( @uuids )
+		end
+
+		it "can search for uuids" do
+			expect( metastore.search.to_a ).to eq( metastore.oids )
+		end
+
+		it "can apply criteria to searches" do
+			results = metastore.search( criteria: {format: 'audio/mp3'} )
+			expect( results.size ).to eq( 2 )
+			results.each do |uuid|
+				expect( metastore.fetch_value(uuid, 'format') ).to eq( 'audio/mp3' )
+			end
+		end
+
+		it "can limit the number of results returned from a search" do
+			expect( metastore.search( limit: 2 ).to_a ).to eq( metastore.oids[0,2] )
+		end
+
+		it "can order the results returned from a search" do
+			results = metastore.search( order: %w[title created] ).to_a
+			sorted_uuids = metastore.each_oid.
+				map {|oid| metastore.fetch(oid, :title, :created).merge(oid: oid) }.
+				sort_by {|tuple| tuple.values_at('title', 'created') }.
+				map {|tuple| tuple[:oid] }
+
+			expect( results ).to eq( sorted_uuids )
+		end
+
+	end
+
+
+end
+
+
+RSpec.shared_examples "a Thingfish datastore" do
+
+	let( :png_io ) { StringIO.new(TEST_PNG_DATA.dup) }
+	let( :text_io ) { StringIO.new(TEST_TEXT_DATA.dup) }
+
+
+	it "returns a UUID when saving" do
+		expect( store.save(png_io) ).to be_a_uuid()
+	end
+
+
+	it "restores the position of the IO after saving" do
+		png_io.pos = 11
+		store.save( png_io )
+		expect( png_io.pos ).to eq( 11 )
+	end
+
+
+	it "can replace existing data" do
+		new_uuid = store.save( text_io )
+		store.replace( new_uuid, png_io )
+
+		rval = store.fetch( new_uuid )
+		expect( rval ).to respond_to( :read )
+		expect( rval.read ).to eq( TEST_PNG_DATA )
+	end
+
+
+	it "doesn't care about the case of the uuid when replacing" do
+		new_uuid = store.save( text_io )
+		store.replace( new_uuid.upcase, png_io )
+
+		rval = store.fetch( new_uuid )
+		expect( rval ).to respond_to( :read )
+		expect( rval.read ).to eq( TEST_PNG_DATA )
+	end
+
+
+	it "can fetch saved data" do
+		oid = store.save( text_io )
+		rval = store.fetch( oid )
+
+		expect( rval ).to respond_to( :read )
+		expect( rval.external_encoding ).to eq( Encoding::ASCII_8BIT )
+		expect( rval.read ).to eq( TEST_TEXT_DATA )
+	end
+
+
+	it "doesn't care about the case of the uuid when fetching" do
+		oid = store.save( text_io )
+		rval = store.fetch( oid.upcase )
+
+		expect( rval ).to respond_to( :read )
+		expect( rval.read ).to eq( TEST_TEXT_DATA )
+	end
+
+
+	it "can remove data" do
+		oid = store.save( text_io )
+		store.remove( oid )
+
+		expect( store.fetch(oid) ).to be_nil
+	end
+
+
+	it "knows if it has data for a given OID" do
+		oid = store.save( text_io )
+		expect( store ).to include( oid )
+	end
+
+end

data/lib/thingfish/datastore.rb

@@ -0,0 +1,55 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'securerandom'
+require 'pluggability'
+require 'stringio'
+require 'strelka'
+
+require 'thingfish' unless defined?( Thingfish )
+require 'thingfish/mixins'
+
+# The base class for storage mechanisms used by Thingfish to store its data
+# blobs.
+class Thingfish::Datastore
+	extend Pluggability,
+	       Strelka::AbstractClass
+	include Enumerable,
+	        Thingfish::Normalization
+
+
+	# Pluggability API -- set the prefix for implementations of Datastore
+	plugin_prefixes 'thingfish/datastore'
+
+	# AbstractClass API -- register some virtual methods that must be implemented
+	# in subclasses
+	pure_virtual :save,
+	             :replace,
+	             :fetch,
+	             :each,
+	             :include?,
+				 :each_oid,
+	             :remove
+
+
+	# :TODO: Make a utility method that provides normalization for IO handling
+	#  (restore .pos, etc.)
+	# def with_io( io ) ... end
+
+	### Return a representation of the object as a String suitable for debugging.
+	def inspect
+		return "#<%p:%#016x>" % [
+			self.class,
+			self.object_id * 2
+		]
+	end
+
+
+	### Provide transactional consistency to the provided block. Concrete datastores should
+	### override this if they can implement it. By default it's a no-op.
+	def transaction
+		yield
+	end
+
+end # class Thingfish::Datastore
+

data/lib/thingfish/datastore/memory.rb

@@ -0,0 +1,93 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'thingfish' unless defined?( Thingfish )
+require 'thingfish/datastore' unless defined?( Thingfish::Datastore )
+
+
+
+# An in-memory datastore for testing and tryout purposes.
+class Thingfish::Datastore::Memory < Thingfish::Datastore
+	extend Loggability
+
+	# Loggability API -- log to the :thingfish logger
+	log_to :thingfish
+
+
+	### Create a new MemoryDatastore, using the given +storage+ object to store
+	### data in. The +storage+ should quack like a Hash.
+	def initialize( storage={} )
+		@storage = storage
+	end
+
+
+	### Save the +data+ read from the specified +io+ and return an ID that can be
+	### used to fetch it later.
+	def save( io )
+		oid = make_object_id()
+		offset = io.pos
+		data = io.read.dup
+
+		self.log.debug "Saving %d bytes of data under OID %s" % [ data.bytesize, oid ]
+		@storage[ oid ] = data
+
+		io.pos = offset
+		return oid
+	end
+
+
+	### Replace the existing object associated with +oid+ with the data read from the
+	### given +io+.
+	def replace( oid, io )
+		offset = io.pos
+		data = io.read.dup
+		oid = normalize_oid( oid )
+
+		self.log.debug "Replacing data under OID %s with %d bytes" % [ oid, data.bytesize ]
+		@storage[ oid ] = data
+
+		io.pos = offset
+		return true
+	end
+
+
+	### Fetch the data corresponding to the given +oid+ as an IOish object.
+	def fetch( oid )
+		oid = normalize_oid( oid )
+		self.log.debug "Fetching data for OID %s" % [ oid ]
+		data = @storage[ oid ] or return nil
+		return StringIO.new( data )
+	end
+
+
+	### Remove the data associated with +oid+ from the Datastore.
+	def remove( oid )
+		oid = normalize_oid( oid )
+		@storage.delete( oid )
+	end
+
+
+	### Return +true+ if the datastore has data associated with the specified +oid+.
+	def include?( oid )
+		oid = normalize_oid( oid )
+		return @storage.include?( oid )
+	end
+
+
+	### Iterator -- yield the UUID of each object in the datastore to the block, or
+	### return an Enumerator for each UUID if called without a block.
+	def each_oid( &block )
+		return @storage.each_key( &block )
+	end
+
+
+	### Iterator -- yield a pair:
+	###    UUID => datablob
+	### of each object in the datastore to the block, or return an Enumerator
+	### for each UUID if called without a block.
+	def each( &block )
+		return @storage.each( &block )
+	end
+
+end # class Thingfish::Datastore::Memory
+