thingfish-metastore-pggraph 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: aa350287482cc8fd51a482da2133f97e31b01d9d
+  data.tar.gz: ad7d4eeeb9db0849b5fa90b9f76633075fb81134
+SHA512:
+  metadata.gz: b2b3bad22b9291d94bd5c9ce659fd2ce3cedb9956aad6b35d1dee21421d97bbca90ceb96450d91ff58e2f8f1bc1a1c1e15ad2f99444606807b4833e0fb037746
+  data.tar.gz: 30b2e55811bd4ade8794f3407a47476e57175245a9914fc45b0cf529dfaecb139ca2017c224612dc7b8c9bd71381aa4de7537cdb71b8ac3fe12ab7d6d32d275c

data/data/thingfish-metastore-pggraph/migrations/20151102_initial.rb

@@ -0,0 +1,45 @@
+# vim: set nosta noet ts=4 sw=4:
+
+### The initial Thingfish::Metastore::PgGraph DDL.
+###
+Sequel.migration do
+	up do
+		create_table( :nodes ) do
+			uuid        :id,           primary_key: true
+			text        :format,        null: false
+			int         :extent,        null: false
+			timestamptz :created,       null: false, default: Sequel.function(:now)
+			inet        :uploadaddress, null: false
+			jsonb       :user_metadata, null: false, default: '{}'
+		end
+
+		create_table( :edges ) do
+			uuid  :id_p, null: false
+			uuid  :id_c, null: false
+			jsonb :prop,  null: false, default: '{}'
+
+			index :id_p
+			index :id_c
+
+			# Remove relationships when a node is deleted.
+			foreign_key [:id_p], :nodes, name: 'edges_p_fkey', key: :id, on_delete: :cascade, on_update: :cascade
+			foreign_key [:id_c], :nodes, name: 'edges_c_fkey', key: :id, on_delete: :cascade, on_update: :cascade
+		end
+
+		# Add functional index from JSON edge props
+		run "CREATE INDEX relation_idx ON edges ( (prop->>'relationship') )"
+
+		# Disallow a node linking to itself -- no self loops.
+		run 'ALTER TABLE edges ADD CONSTRAINT no_self_edge_chk CHECK ( id_p <> id_c )'
+
+		# Allow only a single link between any two nodes, enforcing relations
+		# to be directional parent->child.
+		run 'CREATE UNIQUE INDEX pair_unique_idx ON edges USING btree ((LEAST(id_p, id_c)), (GREATEST(id_p, id_c)))'
+	end
+
+	down do
+		drop_table( :nodes, cascade: true )
+		drop_table( :edges, cascade: true )
+	end
+end
+

data/lib/thingfish/metastore/pggraph.rb

@@ -0,0 +1,311 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'loggability'
+require 'configurability'
+require 'sequel'
+require 'strelka'
+require 'strelka/mixins'
+
+require 'thingfish'
+require 'thingfish/mixins'
+require 'thingfish/metastore'
+
+# Toplevel namespace
+class Thingfish::Metastore::PgGraph < Thingfish::Metastore
+	extend Loggability,
+	       Configurability,
+	       Strelka::MethodUtilities
+	include Thingfish::Normalization
+
+
+	# Load Sequel extensions/plugins
+	Sequel.extension :migration
+
+
+	# Package version
+	VERSION = '0.1.3'
+
+	# Version control revision
+	REVISION = %q$Revision: 1ad0d5bc5083 $
+
+	# The data directory that contains migration files.
+	#
+	DATADIR = if ENV['THINGFISH_METASTORE_PGGRAPH_DATADIR']
+			Pathname.new( ENV['THINGFISH_METASTORE_PGGRAPH_DATADIR'] )
+		elsif Gem.datadir( 'thingfish-metastore-pggraph' )
+			Pathname.new( Gem.datadir('thingfish-metastore-pggraph') )
+		else
+			Pathname.new( __FILE__ ).dirname.parent.parent.parent +
+				'data' + 'thingfish-metastore-pggraph'
+		end
+
+	# The default config values
+	DEFAULT_CONFIG = {
+		uri: 'postgres:/thingfish',
+		slow_query_seconds: 0.01,
+	}
+
+
+	# Loggability API -- use a separate logger
+	log_as :thingfish_metastore_pggraph
+
+	# Configurability API -- load the `pg_metastore`
+	config_key :pggraph_metastore
+
+	##
+	# The URI of the database to use for the metastore
+	singleton_attr_accessor :uri
+
+	##
+	# The Sequel::Database that's used to access the metastore tables
+	singleton_attr_accessor :db
+
+	##
+	# The number of seconds to consider a "slow" query
+	singleton_attr_accessor :slow_query_seconds
+
+
+	### Set up the metastore database and migrate to the latest version.
+	def self::setup_database
+		Sequel.extension :pg_json_ops
+
+		self.db = Sequel.connect( self.uri )
+		self.db.logger = Loggability[ Thingfish::Metastore::PgGraph ]
+		self.db.extension :pg_streaming
+		self.db.stream_all_queries = true
+		self.db.optimize_model_load = true
+		self.db.sql_log_level = :debug
+		self.db.extension( :pg_json )
+		self.db.log_warn_duration = self.slow_query_seconds
+
+		# Ensure the database is current.
+		#
+		unless Sequel::Migrator.is_current?( self.db, self.migrations_dir.to_s )
+			self.log.info "Installing database schema..."
+			Sequel::Migrator.apply( self.db, self.migrations_dir.to_s )
+		end
+	end
+
+
+	### Tear down the configured metastore database.
+	def self::teardown_database
+		self.log.info "Tearing down database schema..."
+		Sequel::Migrator.apply( self.db, self.migrations_dir.to_s, 0 )
+	end
+
+
+	### Return the current database migrations directory as a Pathname
+	def self::migrations_dir
+		return DATADIR + 'migrations'
+	end
+
+
+	### Configurability API -- set up the metastore with the `pg_metastore` section of
+	### the config file.
+	def self::configure( config=nil )
+		config = self.defaults.merge( config || {} )
+
+		self.uri                = config[:uri]
+		self.slow_query_seconds = config[:slow_query_seconds]
+
+		self.setup_database
+	end
+
+
+	### Set up the metastore.
+	def initialize( * ) # :notnew:
+		require 'thingfish/metastore/pggraph/node'
+		require 'thingfish/metastore/pggraph/edge'
+		Thingfish::Metastore::PgGraph::Node.db = self.class.db
+		Thingfish::Metastore::PgGraph::Edge.db = self.class.db
+		@model = Thingfish::Metastore::PgGraph::Node
+	end
+
+
+	######
+	public
+	######
+
+	##
+	# The Sequel model representing the metadata rows.
+	attr_reader :model
+
+
+	#
+	# :section: Thingfish::Metastore API
+	#
+
+	### Return an Array of all stored oids.
+	def oids
+		return self.each_oid.to_a
+	end
+
+
+	### Iterate over each of the store's oids, yielding to the block if one is given
+	### or returning an Enumerator if one is not.
+	def each_oid( &block )
+		return self.model.select_map( :id ).each( &block )
+	end
+
+
+	### Save the +metadata+ Hash for the specified +oid+.
+	def save( oid, metadata )
+		md = self.model.from_hash( metadata )
+		md.id = oid
+		md.save
+	end
+
+
+	### Fetch the data corresponding to the given +oid+ as a Hash-ish object.
+	def fetch( oid, *keys )
+		metadata = self.model[ oid ] or return nil
+
+		if keys.empty?
+			return metadata.to_hash
+		else
+			keys = normalize_keys( keys )
+			values = metadata.to_hash.values_at( *keys )
+			return Hash[ [keys, values].transpose ]
+		end
+	end
+
+
+	### Fetch the value of the metadata associated with the given +key+ for the
+	### specified +oid+.
+	def fetch_value( oid, key )
+		metadata = self.model[ oid ] or return nil
+		key = key.to_sym
+		return metadata[ key ] || metadata.user_metadata[ key ]
+	end
+
+
+	### Fetch UUIDs related to the given +oid+.
+	def fetch_related_oids( oid )
+		oid = normalize_oid( oid )
+		oid = self.model[ oid ]
+		return [] unless oid
+		return oid.related_nodes.map( &:id_c )
+	end
+
+
+	### Search the metastore for UUIDs which match the specified +criteria+ and
+	### return them as an iterator.
+	def search( options={} )
+		ds = self.model.naked.select( :id )
+		self.log.debug "Starting search with %p" % [ ds ]
+
+		ds = self.omit_related_resources( ds, options )
+		ds = self.apply_search_criteria( ds, options )
+		ds = self.apply_search_order( ds, options )
+		ds = self.apply_search_direction( ds, options )
+		ds = self.apply_search_limit( ds, options )
+
+		self.log.debug "Dataset for search is: %s" % [ ds.sql ]
+
+		return ds.map {|row| row[:id] }
+	end
+
+
+	### Update the metadata for the given +oid+ with the specified +values+ hash.
+	def merge( oid, values )
+		values = normalize_keys( values )
+
+		md = self.model[ oid ] or return nil
+		md.merge!( values )
+		md.save
+	end
+
+
+	### Remove all metadata associated with +oid+ from the Metastore.
+	def remove( oid, *keys )
+		self.model[ id: oid ].destroy
+	end
+
+
+	### Remove all metadata associated with +oid+ except for the specified +keys+.
+	def remove_except( oid, *keys )
+		keys = normalize_keys( keys )
+
+		md = self.model[ oid ] or return nil
+		md.user_metadata.keep_if {|key,_| keys.include?(key) }
+		md.save
+	end
+
+
+	### Returns +true+ if the metastore has metadata associated with the specified +oid+.
+	def include?( oid )
+		return self.model.count( id: oid ).nonzero?
+	end
+
+
+	### Returns the number of objects the store contains.
+	def size
+		return self.model.count
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Omit related resources from the search dataset +ds+ unless the given
+	### +options+ specify otherwise.
+	def omit_related_resources( ds, options )
+		unless options[:include_related]
+			self.log.debug "  omitting entries for related resources"
+			ds = ds.unrelated
+		end
+		return ds
+	end
+
+
+	### Apply the search :criteria from the specified +options+ to the collection
+	### in +ds+ and return the modified dataset.
+	def apply_search_criteria( ds, options )
+		if (( criteria = options[:criteria] ))
+			criteria.each do |field, value|
+				self.log.debug "  applying criteria: %p => %p" % [ field.to_s, value ]
+				ds = ds.where_metadata( field => value )
+			end
+		end
+
+		return ds
+	end
+
+
+	### Apply the search :order from the specified +options+ to the collection in
+	### +ds+ and return the modified dataset.
+	def apply_search_order( ds, options )
+		if options[:order]
+			columns = Array( options[:order] )
+			self.log.debug "  ordering results by columns: %p" % [ columns ]
+			ds = ds.order_metadata( columns )
+		end
+
+		return ds
+	end
+
+
+	### Apply the search :direction from the specified +options+ to the collection
+	### in +ds+ and return the modified dataset.
+	def apply_search_direction( ds, options )
+		ds = ds.reverse if options[:direction] && options[:direction] == 'desc'
+		return ds
+	end
+
+
+	### Apply the search :limit from the specified +options+ to the collection in
+	### +ds+ and return the modified dataset.
+	def apply_search_limit( ds, options )
+		if (( limit = options[:limit] ))
+			self.log.debug "  limiting to %s results" % [ limit ]
+			offset = options[:offset] || 0
+			ds = ds.limit( limit, offset )
+		end
+
+		return ds
+	end
+
+end # class Thingfish::Metastore::PgGraph
+

data/lib/thingfish/metastore/pggraph/edge.rb

@@ -0,0 +1,52 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'sequel/model'
+
+require 'thingfish/mixins'
+require 'thingfish/metastore/pggraph' unless defined?( Thingfish::Metastore::PgGraph )
+
+
+### A row representing a relationship between two node objects.
+###
+class Thingfish::Metastore::PgGraph::Edge < Sequel::Model( :edges )
+
+	# Related resource associations
+	many_to_one :node, :key => :id_p
+
+	# Dataset methods
+	#
+	dataset_module do
+		#########
+		protected
+		#########
+
+		### Returns a Sequel expression suitable for use as the key of a query against
+		### the specified property field.
+		###
+		def prop_expr( field )
+			return Sequel.pg_jsonb( :prop ).get_text( field.to_s )
+		end
+	end
+
+
+	### Do some initial attribute setup for new objects.
+	###
+	def initialize( * )
+		super
+		self[ :prop ] ||= Sequel.pg_jsonb({})
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Proxy method -- fetch a value from the edge property hash if it exists.
+	###
+	def method_missing( sym, *args, &block )
+		return self.prop[ sym.to_s ] || super
+	end
+
+end # Thingfish::Metastore::PgGraph::Edge
+

data/lib/thingfish/metastore/pggraph/node.rb

@@ -0,0 +1,202 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'sequel/model'
+
+require 'thingfish/mixins'
+require 'thingfish/metastore/pggraph' unless defined?( Thingfish::Metastore::PgGraph )
+
+
+# A row of metadata describing an asset in a Thingfish store.
+class Thingfish::Metastore::PgGraph::Node < Sequel::Model( :nodes )
+	include Thingfish::Normalization
+
+	# Related resources for this node
+	one_to_many :related_nodes, :key => :id_p, :class => 'Thingfish::Metastore::PgGraph::Edge'
+
+	# Edge relation if this node is a related resource
+	one_to_one :related_to, :key => :id_c, :class => 'Thingfish::Metastore::PgGraph::Edge'
+
+	# Allow instances to be created with a primary key
+	unrestrict_primary_key
+
+
+	# Dataset methods
+	dataset_module do
+
+		### Dataset method: Limit results to metadata which is for a related resource.
+		###
+		def related
+			return self.join_edges( :rel ).exclude( :rel__id_c => nil )
+		end
+
+
+		### Dataset method: Limit results to metadata which is not for a related resource.
+		###
+		def unrelated
+			return self.join_edges( :notrel ).filter( :notrel__id_c => nil )
+		end
+
+
+		### Dataset method: Limit results to records whose operational or user
+		### metadata matches the values from the specified +hash+.
+		###
+		def where_metadata( hash )
+			ds = self
+			hash.each do |field, value|
+
+				# Direct DB column
+				#
+				if self.model.metadata_columns.include?( field.to_sym )
+					ds = ds.where( field.to_sym => value )
+
+				# User metadata or edge relationship
+				#
+				else
+					if field.to_sym == :relationship
+						ds = ds.join_edges unless ds.joined_dataset?
+						ds = ds.filter( Sequel.pg_jsonb( :edges__prop ).get_text( field.to_s ) => value )
+
+					elsif field.to_sym == :relation
+						ds = ds.join_edges unless ds.joined_dataset?
+						ds = self.join_edges.filter( :edges__id_p => value )
+
+					else
+						ds = ds.where( self.user_metadata_expr(field) => value )
+					end
+				end
+			end
+
+			return ds
+		end
+
+
+		### Dataset method: Order results by the specified +columns+.
+		###
+		def order_metadata( *columns )
+			columns.flatten!
+			ds = self
+			columns.each do |column|
+				if Thingfish::Metastore::PgGraph::Node.metadata_columns.include?( column.to_sym )
+					ds = ds.order_append( column.to_sym )
+				else
+					ds = ds.order_append( self.user_metadata_expr(column) )
+				end
+			end
+
+			return ds
+		end
+
+
+
+		#########
+		protected
+		#########
+
+		### Return a dataset linking related nodes to edges.
+		###
+		def join_edges( aka=nil )
+			return self.join_table( :left, :edges, { :id_c => :nodes__id }, { :table_alias => aka } )
+		end
+
+
+		### Returns a Sequel expression suitable for use as the key of a query against
+		### the specified user metadata field.
+		def user_metadata_expr( field )
+			return Sequel.pg_jsonb( :user_metadata ).get_text( field.to_s )
+		end
+
+	end # dataset_module
+
+
+	### Return a new Metadata object from the given +oid+ and one-dimensional +hash+
+	### used by Thingfish.
+	def self::from_hash( hash )
+		metadata = Thingfish::Normalization.normalize_keys( hash )
+
+		md = new
+
+		md.format        = metadata.delete( 'format' )
+		md.extent        = metadata.delete( 'extent' )
+		md.created       = metadata.delete( 'created' )
+		md.uploadaddress = metadata.delete( 'uploadaddress' ).to_s
+
+		md.user_metadata = Sequel.pg_jsonb( metadata )
+
+		return md
+	end
+
+
+	### Return the columns of the table that are used for resource metadata.
+	def self::metadata_columns
+		return self.columns - [self.primary_key, :user_metadata]
+	end
+
+
+	### Do some initial attribute setup for new objects.
+	def initialize( * )
+		super
+		self[ :user_metadata ] ||= Sequel.pg_jsonb({})
+	end
+
+
+	### Return the metadata as a Hash; overridden from Sequel::Model to
+	### merge the user and system pairs together.
+	def to_hash
+		hash = self.values.dup
+
+		hash.delete( :id )
+		hash.merge!( hash.delete(:user_metadata) )
+
+		if related_to = self.related_to
+			hash.merge!( related_to.prop )
+			hash[ :relation ] = related_to.id_p
+		end
+
+		return normalize_keys( hash )
+	end
+
+
+	### Merge new metadata +values+ into the metadata for the resource
+	def merge!( values )
+
+		# Extract and set the column-metadata values first
+		self.class.metadata_columns.each do |col|
+			next unless values.key?( col.to_s )
+			self[ col ] = values.delete( col.to_s )
+		end
+
+		self.user_metadata.merge!( values )
+	end
+
+
+	### Hook creation for new related resources, divert relation data to
+	### a new edge row.
+	###
+	def around_save
+		relationship = self.user_metadata.delete( 'relationship' )
+		relation     = self.user_metadata.delete( 'relation' )
+
+		super
+
+		if relation
+			edge = Thingfish::Metastore::PgGraph::Edge.new
+			edge.prop[ 'relationship' ] = relationship
+			edge.id_p = relation
+			edge.id_c = self.id
+			edge.save
+		end
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Proxy method -- fetch a value from the metadata hash if it exists.
+	def method_missing( sym, *args, &block )
+		return self.user_metadata[ sym.to_s ] || super
+	end
+
+end # Thingfish::Metastore::PgGraph::Node
+

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: thingfish-metastore-pggraph
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- Mahlon E. Smith <mahlon@martini.nu>
+- Michael Granger <ged@faeriemud.org>
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-11-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thingfish
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: loggability
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: configurability
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.35'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.35'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+description: |
+  This is a metadata storage plugin for the Thingfish digital asset
+  manager.  It provides persistent storage for uploaded data to PostgreSQL
+  tables.
+
+  It is heavily based on the regular PG metastore, however it differs by
+  storing objects as nodes, and their relations as edges.
+email: mahlon@martini.nu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- data/thingfish-metastore-pggraph/migrations/20151102_initial.rb
+- lib/thingfish/metastore/pggraph.rb
+- lib/thingfish/metastore/pggraph/edge.rb
+- lib/thingfish/metastore/pggraph/node.rb
+homepage: https://bitbucket.org/mahlon/thingfish-metastore-pggraph
+licenses:
+- BSD-3-Clause
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Graph DDL storage for Thingfish metadata.
+test_files: []