thingfish 0.5.0.pre20160707192835

data/lib/thingfish/metastore.rb

@@ -0,0 +1,55 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'pluggability'
+require 'strelka'
+require 'strelka/mixins'
+
+require 'thingfish' unless defined?( Thingfish )
+require 'thingfish/mixins'
+
+# The base class for storage mechanisms used by Thingfish to store its data
+# blobs.
+class Thingfish::Metastore
+	extend Pluggability,
+	       Strelka::AbstractClass
+	include Thingfish::Normalization
+
+
+	# Pluggability API -- set the prefix for implementations of Metastore
+	plugin_prefixes 'thingfish/metastore'
+
+	# AbstractClass API -- register some virtual methods that must be implemented
+	# in subclasses
+	pure_virtual :oids,
+	             :each_oid,
+	             :save,
+	             :search,
+	             :fetch,
+	             :fetch_value,
+	             :fetch_related_oids,
+	             :merge,
+	             :include?,
+	             :remove,
+	             :remove_except,
+	             :size
+
+	### Return a representation of the object as a String suitable for debugging.
+	def inspect
+		return "#<%p:%#016x %d objects>" % [
+			self.class,
+			self.object_id * 2,
+			self.size
+		]
+	end
+
+
+	### Provide transactional consistency to the provided block. Concrete metastores should
+	### override this if they can implement it. By default it's a no-op.
+	def transaction
+		yield
+	end
+
+
+end # class Thingfish::Metastore
+

data/lib/thingfish/metastore/memory.rb

@@ -0,0 +1,201 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'thingfish' unless defined?( Thingfish )
+require 'thingfish/metastore' unless defined?( Thingfish::Metastore )
+
+
+
+# An in-memory metastore for testing and tryout purposes.
+class Thingfish::Metastore::Memory < Thingfish::Metastore
+	extend Loggability
+	include Thingfish::Normalization
+
+	# Loggability API -- log to the :thingfish logger
+	log_to :thingfish
+
+
+	### Create a new MemoryMetastore, using the given +storage+ object to store
+	### data in. The +storage+ should quack like a Hash.
+	def initialize( storage={} )
+		@storage = storage
+	end
+
+
+	##
+	# The raw Hash of metadata
+	attr_reader :storage
+
+
+	### Return an Array of all stored OIDs.
+	def oids
+		return @storage.keys
+	end
+
+
+	### Iterate over the OID of each entry in the store, yielding to the block if one is given
+	### or returning an Enumerator if one is not.
+	def each_oid( &block )
+		return @storage.each_key( &block )
+	end
+
+
+	### Save the +metadata+ Hash for the specified +oid+.
+	def save( oid, metadata )
+		oid = normalize_oid( oid )
+		@storage[ oid ] = metadata.dup
+	end
+
+
+	### Fetch the data corresponding to the given +oid+ as a Hash-ish object.
+	def fetch( oid, *keys )
+		oid = normalize_oid( oid )
+		metadata = @storage[ oid ] or return nil
+
+		if keys.empty?
+			self.log.debug "Fetching metadata for OID %s" % [ oid ]
+			return metadata.dup
+		else
+			self.log.debug "Fetching metadata for %p for OID %s" % [ keys, oid ]
+			keys = normalize_keys( keys )
+			values = metadata.values_at( *keys )
+			return Hash[ [keys, values].transpose ]
+		end
+	end
+
+
+	### Fetch the value of the metadata associated with the given +key+ for the
+	### specified +oid+.
+	def fetch_value( oid, key )
+		oid = normalize_oid( oid )
+		key = normalize_key( key )
+		data = @storage[ oid ] or return nil
+
+		return data[ key ]
+	end
+
+
+	### Fetch OIDs related to the given +oid+.
+	def fetch_related_oids( oid )
+		oid = normalize_oid( oid )
+		self.log.debug "Fetching OIDs of resources related to %s" % [ oid ]
+		return self.search( :criteria => {:relation => oid}, :include_related => true )
+	end
+
+
+	### Search the metastore for UUIDs which match the specified +criteria+ and
+	### return them as an iterator.
+	def search( options={} )
+		ds = @storage.each_key
+		self.log.debug "Starting search with %p" % [ ds ]
+
+		ds = self.omit_related_resources( ds, options )
+		ds = self.apply_search_criteria( ds, options )
+		ds = self.apply_search_order( ds, options )
+		ds = self.apply_search_direction( ds, options )
+		ds = self.apply_search_limit( ds, options )
+
+		return ds.to_a
+	end
+
+
+	### Omit related resources from the search dataset +ds+ unless the given
+	### +options+ specify otherwise.
+	def omit_related_resources( ds, options )
+		unless options[:include_related]
+			ds = ds.reject {|uuid| @storage[uuid]['relationship'] }
+		end
+		return ds
+	end
+
+
+	### Apply the search :criteria from the specified +options+ to the collection
+	### in +ds+ and return the modified dataset.
+	def apply_search_criteria( ds, options )
+		if (( criteria = options[:criteria] ))
+			criteria.each do |field, value|
+				self.log.debug "  applying criteria: %p => %p" % [ field.to_s, value ]
+				ds = ds.select {|uuid| @storage[uuid][field.to_s] == value }
+			end
+		end
+
+		return ds
+	end
+
+
+	### Apply the search :order from the specified +options+ to the collection in
+	### +ds+ and return the modified dataset.
+	def apply_search_order( ds, options )
+		if (( fields = options[:order] ))
+			ds = ds.to_a.sort_by do |uuid|
+				@storage[ uuid ].values_at( *fields.compact ).map {|val| val || ''}
+			end
+		end
+
+		return ds
+	end
+
+
+	### Apply the search :direction from the specified +options+ to the collection
+	### in +ds+ and return the modified dataset.
+	def apply_search_direction( ds, options )
+		ds.reverse! if options[:direction] && options[:direction] == 'desc'
+		return ds
+	end
+
+
+	### Apply the search :limit from the specified +options+ to the collection in
+	### +ds+ and return the modified dataset.
+	def apply_search_limit( ds, options )
+		if (( limit = options[:limit] ))
+			self.log.debug "  limiting to %s results" % [ limit ]
+			offset = options[:offset] || 0
+			ds = ds.to_a.slice( offset, limit )
+		end
+
+		return ds
+	end
+
+
+	### Update the metadata for the given +oid+ with the specified +values+ hash.
+	def merge( oid, values )
+		oid = normalize_oid( oid )
+		values = normalize_keys( values )
+		@storage[ oid ].merge!( values )
+	end
+
+
+	### Remove all metadata associated with +oid+ from the Metastore.
+	def remove( oid, *keys )
+		oid = normalize_oid( oid )
+		if keys.empty?
+			@storage.delete( oid )
+		else
+			keys = normalize_keys( keys )
+			@storage[ oid ].delete_if {|key, _| keys.include?(key) }
+		end
+	end
+
+
+	### Remove all metadata associated with +oid+ except for the specified +keys+.
+	def remove_except( oid, *keys )
+		oid = normalize_oid( oid )
+		keys = normalize_keys( keys )
+		@storage[ oid ].keep_if {|key,_| keys.include?(key) }
+	end
+
+
+	### Returns +true+ if the metastore has metadata associated with the specified +oid+.
+	def include?( oid )
+		oid = normalize_oid( oid )
+		return @storage.include?( oid )
+	end
+
+
+	### Returns the number of objects the store contains.
+	def size
+		return @storage.size
+	end
+
+end # class Thingfish::Metastore::Memory
+

data/lib/thingfish/mixins.rb

@@ -0,0 +1,57 @@
+# -*- ruby -*-
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'securerandom'
+
+require 'thingfish' unless defined?( Thingfish )
+
+module Thingfish
+
+	# A collection of functions for dealing with object IDs.
+	module Normalization
+
+		###############
+		module_function
+		###############
+
+		### Generate a new object ID.
+		def make_object_id
+			return normalize_oid( SecureRandom.uuid )
+		end
+
+
+		### Normalize the given +oid+.
+		def normalize_oid( oid )
+			return oid.to_s.downcase
+		end
+
+
+		### Return a copy of the given +collection+ after being normalized.
+		def normalize_keys( collection )
+			if collection.respond_to?( :keys )
+				return collection.each_with_object({}) do |(key,val),new_hash|
+					n_key = normalize_key( key )
+					new_hash[ n_key ] = val
+				end
+
+			elsif collection.respond_to?( :map )
+				return collection.map {|key| normalize_key(key) }
+			end
+
+			return nil
+		end
+
+
+		### Return a normalized copy of +key+.
+		def normalize_key( key )
+			return key.to_s.downcase.gsub( /[^\w:]+/, '_' )
+		end
+
+	end # module Normalization
+
+
+end # module Thingfish
+
+# vim: set nosta noet ts=4 sw=4:
+

data/lib/thingfish/processor.rb

@@ -0,0 +1,79 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'pluggability'
+require 'strelka/httprequest/acceptparams'
+
+require 'thingfish' unless defined?( Thingfish )
+
+
+# Thingfish asset processor base class.
+class Thingfish::Processor
+	extend Pluggability
+
+
+	plugin_prefixes 'thingfish/processor'
+
+
+	### Get/set the list of media types this processor can handle.
+	def self::handled_types( *mediatypes )
+		if mediatypes.empty?
+			@handled_types ||= []
+		else
+			@handled_types = mediatypes.collect do |type|
+				Strelka::HTTPRequest::MediaType.parse(type)
+			end
+		end
+
+		return @handled_types
+	end
+
+
+	### Filter hook for request, pass to processor if it is able to
+	### handle the +request+ content type.
+	def process_request( request )
+		return unless self.handled_path?( request )
+		if self.handled_type?( request.content_type )
+			on_request( request )
+		end
+	end
+
+
+	### Process the data and/or metadata in the +request+.
+	def on_request( request )
+		# No-op by default
+	end
+
+
+	### Filter hook for response, pass to processor if it is able to
+	### handle the +response+ content type.
+	def process_response( response )
+		return unless self.handled_path?( response.request )
+		if self.handled_type?( response.content_type )
+			on_response( response )
+		end
+	end
+
+
+	### Process the data and/or metadata in the +response+.
+	def on_response( response )
+		# No-op by default
+	end
+
+
+	### Returns +true+ if the given media +type+ is one the processor handles.
+	def handled_type?( type )
+		return true if self.class.handled_types.empty?
+		self.class.handled_types.find {|handled_type| type =~ handled_type }
+	end
+	alias_method :is_handled_type?, :handled_type?
+
+
+	### Returns +true+ if the given +request+'s path is one that should
+	### be processed.
+	def handled_path?( request )
+		return ! request.path.match( %r|^/?[\w\-]+/metadata| )
+	end
+
+end # class Thingfish::Processor
+

data/lib/thingfish/processor/mp3.rb

@@ -0,0 +1,167 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'mp3info'
+
+require 'thingfish' unless defined?( Thingfish )
+require 'thingfish/processor' unless defined?( Thingfish::Processor )
+
+
+# Attach ID3 data to an mp3, along with any embedded album art as a related resource.
+class Thingfish::Processor::MP3 < Thingfish::Processor
+	extend Loggability
+
+	# Loggability API -- log to the :thingfish logger
+	log_to :thingfish
+
+	# The list of handled types
+	handled_types 'audio/mpeg', 'audio/mpg', 'audio/mp3'
+
+
+	# Null character
+	NULL = "\x0"
+
+	# Attached picture   "PIC"
+	#    Frame size         $xx xx xx
+	#    ---- mp3info is 'helpfully' cropping out frame size.
+	#    Text encoding      $xx
+	#    Image format       $xx xx xx
+	#    Picture type       $xx
+	#    Description        <textstring> $00 (00)
+	#    Picture data       <binary data>
+	PIC_FORMAT = 'h a3 h Z* a*'
+
+	# Attached picture "APIC"
+	#    Text encoding   $xx
+	#    MIME type       <text string> $00
+	#    Picture type    $xx
+	#    Description     <text string according to encoding> $00 (00)
+	#    Picture data    <binary data>
+	APIC_FORMAT = 'h Z* h Z* a*'
+
+
+	### Synchronous processor API -- extract metadata from uploaded MP3s
+	def on_request( request )
+		mp3info = Mp3Info.new( request.body )
+
+		mp3_metadata = self.extract_id3_metadata( mp3info )
+		request.add_metadata( mp3_metadata )
+
+		self.extract_images( mp3info ) do |imageio, metadata|
+			metadata[:title] = "Album art for %s - %s" % mp3_metadata.values_at( 'mp3:artist', 'mp3:title' )
+			request.add_related_resource( imageio, metadata )
+		end
+	end
+
+
+	### Normalize metadata from the MP3Info object and return it as a hash.
+	def extract_id3_metadata( mp3info )
+		self.log.debug "Extracting MP3 metadata"
+
+		mp3_metadata = {
+			'mp3:frequency' => mp3info.samplerate,
+			'mp3:bitrate'   => mp3info.bitrate,
+			'mp3:vbr'       => mp3info.vbr,
+			'mp3:title'     => mp3info.tag.title,
+			'mp3:artist'    => mp3info.tag.artist,
+			'mp3:album'     => mp3info.tag.album,
+			'mp3:year'      => mp3info.tag.year,
+			'mp3:genre'     => mp3info.tag.genre,
+			'mp3:tracknum'  => mp3info.tag.tracknum,
+			'mp3:comments'  => mp3info.tag.comments,
+		}
+
+		# ID3V2 2.2.0 has three-letter tags, so map those if the artist info isn't set
+		if mp3info.hastag2?
+			if mp3_metadata['mp3:artist'].nil?
+				self.log.debug "   extracting old-style ID3v2 info" % [mp3info.tag2.version]
+
+				mp3_metadata.merge!({
+					'mp3:title'     => mp3info.tag2.TT2,
+					'mp3:artist'    => mp3info.tag2.TP1,
+					'mp3:album'     => mp3info.tag2.TAL,
+					'mp3:year'      => mp3info.tag2.TYE,
+					'mp3:tracknum'  => mp3info.tag2.TRK,
+					'mp3:comments'  => mp3info.tag2.COM,
+					'mp3:genre'     => mp3info.tag2.TCO,
+				})
+			end
+		end
+
+		self.log.debug "  raw metadata: %p" % [ mp3_metadata ]
+		return sanitize_values( mp3_metadata )
+	end
+
+
+	### Extract image data from ID3 information, supports both APIC (2.3) and the older style
+	### PIC (2.2).  Return value is a hash with IO keys and mimetype values.
+	### {
+	###    io  => { format => 'image/jpeg' }
+	###    io2 => { format => 'image/jpeg' }
+	### }
+	def extract_images( mp3info )
+		self.log.debug "Extracting embedded images"
+		raise LocalJumpError, "no block given" unless block_given?
+
+		unless mp3info.hastag2?
+			self.log.debug "...no id3v2 tag, so no embedded images possible."
+			return
+		end
+
+		self.log.debug "...id3v2 tag present..."
+
+		if mp3info.tag2.APIC
+			self.log.debug "...extracting APIC (id3v2.3+) image data."
+
+			images = [ mp3info.tag2.APIC ].flatten
+			images.each do |img|
+				blob, mime = img.unpack( APIC_FORMAT ).values_at( 4, 1 )
+				yield( StringIO.new(blob),
+					:format       => mime,
+					:extent       => blob.length,
+					:relationship => 'album-art' )
+			end
+
+		elsif mp3info.tag2.PIC
+			self.log.debug "...extracting PIC (id3v2.2) image data."
+
+			images = [ mp3info.tag2.PIC ].flatten
+			images.each do |img|
+				blob, type = img.unpack( PIC_FORMAT ).values_at( 4, 1 )
+				mime = Mongrel2::Config.mimetypes[ ".#{type.downcase}" ] or next
+				yield( StringIO.new(blob),
+					:format       => mime,
+					:extent       => blob.length,
+					:relationship => 'album-art' )
+			end
+
+		else
+			self.log.debug "...no known image tag types in tags: %p" % [ mp3info.tag2.keys.sort ]
+		end
+	end
+
+
+
+	#######
+	private
+	#######
+
+	### Strip NULLs from the values of the given +metadata_hash+ and return it.
+	def sanitize_values( metadata_hash )
+		metadata_hash.each do |k,v|
+			case v
+			when String
+				metadata_hash[k] = v.chomp(NULL).strip
+			when Array
+				metadata_hash[k] = v.collect {|vv| vv.chomp(NULL).strip }
+			when Numeric, TrueClass, FalseClass
+				# No-op
+			end
+		end
+
+		return metadata_hash.delete_if {|_,v| v.nil? }
+	end
+
+
+end # class Thingfish::Processor::MP3
+