talia_core 0.4.0

data/lib/talia_core/active_source_parts/xml/generic_reader.rb

@@ -0,0 +1,363 @@
+require 'hpricot'
+
+module TaliaCore
+  module ActiveSourceParts
+    module Xml
+
+      # Superclass for importers/readers of generic xml files. This is as close as possible
+      # to the SourceReader class, and will (obviously) only work if a subclass fleshes out
+      # the mappings.
+      #
+      # See the SourceReader class for a simple example.
+      #
+      # When adding new sources, the reader will always check if the element is already
+      # present. If attributes for one source are imported in more than one place, all
+      # subsequent calls will merge the newly imported attributes with the existing ones.
+      class GenericReader
+        
+        extend TaliaUtil::IoHelper
+        include TaliaUtil::Progressable
+
+        # Helper class for state
+        class State
+          attr_accessor :attributes, :element
+        end
+
+        class << self
+
+          # See the IoHelper class for help on the options. A progressor may
+          # be supplied on which the importer will report it's progress.
+          def sources_from_url(url, options = nil, progressor = nil)
+            open_generic(url, options) { |io| sources_from(io, progressor) }
+          end
+
+          def sources_from(source, progressor = nil)
+            reader = self.new(source)
+            reader.progressor = progressor
+            reader.sources
+          end
+
+          # Create a handler for an element from which a source will be created
+          def element(element_name, &handler_block)
+            element_handler(element_name, true, &handler_block)
+          end
+
+          # Create a handler for an element which will be processed but from which
+          # no source will be created
+          def plain_element(element_name, &handler_block)
+            element_handler(element_name, false, &handler_block)
+          end
+
+          # Set the reader to allow the use of root elements for import
+          def can_use_root
+            @use_root = true
+          end
+
+          # True if the reader should also check the root element, instead of
+          # only checking the children
+          def use_root
+            @use_root || false
+          end
+
+          # Returns the registered handlers
+          attr_reader :create_handlers
+
+          private
+
+          # Adds an handler for the the given element. The second parameter will
+          # indicate if the handler will create a new source or not
+          def element_handler(element_name, creating, &handler_block)
+            element_name = "#{element_name}_handler".to_sym
+            raise(ArgumentError, "Duplicate handler for #{element_name}") if(self.respond_to?(element_name))
+            raise(ArgumentError, "Must pass block to handler for #{element_name}") unless(handler_block)
+            @create_handlers ||= {}
+            @create_handlers[element_name] = creating # Indicates whether a soure is created
+            # Define the handler block method
+            define_method(element_name, handler_block)
+          end
+        end # End class methods
+
+        def initialize(source)
+          @doc = Hpricot.XML(source)
+        end
+
+        def sources
+          return @sources if(@sources)
+          @sources = {}
+          if(use_root && self.respond_to?("#{@doc.root.name}_handler".to_sym))
+            run_with_progress('XmlRead', 1) { read_source(@doc.root) }
+          else
+            read_children_of(@doc.root)
+          end
+          @sources.values
+        end
+
+        def add_source_with_check(source_attribs)
+          assit_kind_of(Hash, source_attribs)
+          if((uri = source_attribs['uri']).blank?)
+            raise(RuntimeError, "Problem reading from XML: Source without URI (#{source_attribs.inspect})")
+          else
+            uri = irify(uri)
+            source_attribs['uri'] = uri
+            @sources[uri] ||= {} 
+            @sources[uri].each do |key, value|
+              next unless(new_value = source_attribs.delete(key))
+
+              assit(!((key.to_sym == :type) && (value != 'TaliaCore::DummySource') && (value != new_value)), "Type should not change during import, may be a format problem. (From #{value} to #{new_value})")
+              if(new_value.is_a?(Array) && value.is_a?(Array))
+                # If both are Array-types, the new elements will be appended
+                # and duplicates nwill be removed
+                @sources[uri][key] = (value + new_value).uniq
+              else
+                # Otherwise just replace
+                @sources[uri][key] = new_value
+              end
+            end
+            # Now merge in everything else
+            @sources[uri].merge!(source_attribs)
+          end
+        end
+
+        def create_handlers
+          @handlers ||= (self.class.create_handlers || {})
+        end
+
+        def read_source(element, &block)
+          attribs = call_handler(element, &block)
+          add_source_with_check(attribs) if(attribs)
+        end
+        
+        def read_children_of(element, &block)
+          run_with_progress('Xml Read', element.children.size) do |prog|
+            element.children.each do |element|
+              prog.inc
+              next unless(element.is_a?(Hpricot::Elem))
+              read_source(element, &block)
+            end
+          end
+        end
+
+        def use_root
+          self.class.use_root
+        end
+
+        private
+        
+
+        # Removes all characters that are illegal in IRIs, so that the
+        # URIs can be imported
+        def irify(uri)
+          N::URI.new(uri.to_s.gsub( /[{}|\\^`\s]/, '+')).to_s
+        end
+
+        # Call the handler method for the given element. If a block is given, that
+        # will be called instead
+        def call_handler(element)
+          handler_name = "#{element.name}_handler".to_sym
+          if(self.respond_to?(handler_name) || block_given?)
+            parent_state = @current # Save the state for recursive calls
+            attributes = nil
+            begin
+              creating = (create_handlers[handler_name] || block_given?)
+              @current = State.new
+              @current.attributes = creating ? {} : nil
+              @current.element = element
+              block_given? ? yield : self.send(handler_name)
+              attributes = @current.attributes
+            ensure
+              @current = parent_state # Reset the state to previous value
+            end
+            attributes
+          else
+            TaliaCore.logger.warn("Unknown element in import: #{element.name}")
+            false
+          end
+        end
+
+        def chk_create
+          raise(RuntimeError, "Illegal operation when not creating a source") unless(@current.attributes)
+        end
+
+        # Adds a value for the given predicate (may also be a database field)
+        def add(predicate, object, required = false)
+          if(object.kind_of?(Array))
+            object.each { |obj| set_element(predicate, obj.to_s, required) }
+          else
+            set_element(predicate, object.to_s, required)
+          end
+        end
+        
+        # Adds a date field. This will attempt to parse the original string
+        # and write the result as an ISO 8061 compliant date string. Note
+        # that this won't be able to parse everything you throw at it, though.
+        def add_date(predicate, date, required = false, fmt = nil)
+          add(predicate, parse_date(date, fmt), required)
+        end
+        
+        # Adds a date interval as an ISO 8061 compliant date string. See
+        # add_date for more info. If only one of the dates is given this
+        # will add a normal date string instead of an interval.
+        def add_date_interval(predicate, start_date, end_date, fmt = nil)
+          return if(start_date.blank? && end_date.blank?)
+          if(start_date.blank?)
+            add_date(predicate, start_date, true, fmt)
+          elsif(end_date.blank?)
+            add_date(predicate, end_date, true, fmt)
+          else
+            add(predicate, "#{parse_date(start_date, fmt)}/#{parse_date(end_date, fmt)}", required)
+          end
+        end
+
+        # Adds a relation for the given predicate
+        def add_rel(predicate, object, required = false)
+          object = check_objects(object)
+          if(!object)
+            raise(ArgumentError, "Relation with empty object on #{predicate} (#{@current.attributes['uri']}).") if(required)
+            return
+          end
+          if(object.kind_of?(Array))
+            object.each do |obj| 
+              raise(ArgumentError, "Cannot add relation on database field") if(ActiveSource.db_attr?(predicate))
+              set_element(predicate, "<#{irify(obj)}>", required) 
+            end
+          else
+            raise(ArgumentError, "Cannot add relation on database field") if(ActiveSource.db_attr?(predicate))
+            set_element(predicate, "<#{irify(object)}>", required)
+          end
+        end
+
+        # Add a file to the source being imported
+        def add_file(urls, options = {})
+          return if(urls.blank?)
+          urls = [ urls ] unless(urls.is_a?(Array))
+          files = urls.collect { |url| { :url => url.to_s, :options => options } }
+          @current.attributes[:files] = files if(files.size > 0)
+        end
+
+        # Returns true if the given source was already imported. This can return false
+        # if you call this for the currently importing source. 
+        def source_exists?(uri)
+          !@sources[uri].blank?
+        end
+
+        # Adds a source from the given sub-element. You may either pass a block with
+        # the code to import or the name of an already registered element. If the
+        # special value :from_all_sources is given, it will read from all sub-elements for which
+        # there are registered handlers
+        def add_source(sub_element = nil, &block)
+          if(sub_element)
+            if(sub_element == :from_all_sources)
+              read_children_of(@current.element)
+            else
+              @current.element.search("/#{sub_element}").each { |sub_elem| read_source(sub_elem, &block) }
+            end
+          else
+            raise(ArgumentError, "When adding elements on the fly, you must use a block") unless(block)
+            attribs = call_handler(@current.element, &block)
+            add_source_with_check(attribs) if(attribs)
+          end
+        end
+
+        # Returns true if the currently imported element already contains type information
+        # AND is of the given type.
+        def current_is_a?(type)
+          assit_kind_of(Class, type)
+          @current.attributes['type'] && ("TaliaCore::#{@current.attributes['type']}".constantize <= type)
+        end
+
+        # Adds a nested element. This will not change the currently importing source, but
+        # it will set the currently active element to the nested element. 
+        # If a block is given, it will execute for each of the nested elements that
+        # are found. Otherwise, a method name must be given, and that method will
+        # be executed instead of the block
+        def nested(sub_element, handler_method = nil)
+          original_element = @current.element
+          begin
+            @current.element.search("#{sub_element}").each do |sub_elem|
+              @current.element = sub_elem
+              assit(block_given? ^ (handler_method.is_a?(Symbol)), 'Must have either a handler (x)or a block.')
+              block_given? ? yield : self.send(handler_method)
+            end
+          ensure
+            @current.element = original_element
+          end
+        end
+
+        # Imports another source like add_source and also assigns the new source as
+        # a part of the current one
+        def add_part(sub_element = nil, &block)
+          raise(RuntimeError, "Cannot add child before having an uri to refer to.") unless(@current.attributes['uri'])
+          @current.element.search("/#{sub_element}").each do |sub_elem|
+            attribs = call_handler(sub_elem, &block)
+            if(attribs)
+              attribs[N::TALIA.part_of.to_s] ||= []
+              attribs[N::TALIA.part_of.to_s] << "<#{@current.attributes['uri']}>"
+              add_source_with_check(attribs)
+            end
+          end
+        end
+
+        # Add a property to the source currently being imported
+        def set_element(predicate, object, required)
+          chk_create
+          object = check_objects(object)
+          if(!object)
+            raise(ArgumentError, "No object given, but is required for #{predicate}.") if(required)
+            return
+          end
+          predicate = predicate.respond_to?(:uri) ? predicate.uri.to_s : predicate.to_s
+          if(ActiveSource.db_attr?(predicate))
+            assit(!object.is_a?(Array))
+            @current.attributes[predicate] = object
+          else
+            @current.attributes[predicate] ||= []
+            @current.attributes[predicate] << object
+          end
+        end
+
+        # Check the objects and sort out the blank ones (which should not be used).
+        # If no usable object 
+        def check_objects(objects)
+          if(objects.kind_of?(Array))
+            objects.reject! { |obj| obj.blank? }
+            (objects.size == 0) ? nil : objects
+          else
+            objects.blank? ? nil : objects
+          end
+        end
+
+        # Get an attribute from the current xml element
+        def from_attribute(attrib)
+          @current.element[attrib]
+        end
+
+        # Get the content of exactly one child element of type "elem" of the
+        # currently importing element.
+        def from_element(elem)
+          elements = all_elements(elem)
+          elements = elements.uniq if(elements.size > 1) # Try to ignore dupes
+          raise(ArgumentError, "More than one element of #{elem} in #{@current.element.inspect}") if(elements.size > 1)
+          elements.first
+        end
+
+        # Get the content of all child elements of type "elem" of the currently
+        # importing element
+        def all_elements(elem)
+          result = []
+          @current.element.search("/#{elem}").each { |el| result << el.inner_text.strip }
+          result
+        end
+        
+        # Parses the given string and returns it as a date object
+        def parse_date(date, fmt = nil)
+          return nil if(date.blank?)
+          return DateTime.strptime(date, fmt) if(fmt) # format given
+          return DateTime.new(date.to_i) if(date.size < 5) # this short should be a year
+          DateTime.parse(date)
+        end
+        
+      end
+
+    end
+  end
+end

data/lib/talia_core/active_source_parts/xml/rdf_builder.rb

@@ -0,0 +1,88 @@
+module TaliaCore
+  module ActiveSourceParts
+    module Xml
+
+      # Class for creating xml-rdf data
+      class RdfBuilder < BaseBuilder
+
+        # Writes a simple "flat" triple. If the object is a string, it will be
+        # treated as a "value" while an object (ActiveSource or N::URI) will be treated
+        # as a "link"
+        def write_triple(subject, predicate, object)
+          subject = subject.respond_to?(:uri) ? subject.uri.to_s : subject
+          predicate = predicate.respond_to?(:uri) ? predicate : N::URI.new(predicate) 
+          @builder.rdf :Description, "rdf:about" => subject do
+            write_predicate(predicate, [ object ])
+          end
+        end
+
+        # Writes a complete source to the rdf
+        def write_source(source)
+          @builder.rdf :Description, 'rdf:about' => source.uri.to_s do # Element describing this resource
+            # loop through the predicates
+            source.direct_predicates.each do |predicate|
+              write_predicate(predicate, source[predicate])
+            end
+          end
+        end
+
+        private 
+
+        # Build the structure for the XML file and pass on to
+        # the given block
+        def build_structure
+          @builder.rdf :RDF, self.class.namespaces do 
+            yield
+          end
+        end
+
+
+        def self.namespaces
+          @namespaces ||= begin
+            namespaces = {}
+            N::Namespace.shortcuts.each { |key, value| namespaces["xmlns:#{key.to_s}"] = value.to_s }
+            namespaces
+          end
+        end
+
+        # Build an rdf/xml string for one predicate, with the given values
+        def write_predicate(predicate, values)
+          values.each { |val| write_single_predicate(predicate, val) }
+        end # end method
+
+        def write_single_predicate(predicate, value)
+          is_property = value.respond_to?(:uri)
+          value_properties = is_property ? { 'value' => value } : extract_values(value.to_s)
+          value = value_properties.delete('value')
+          @builder.tag!(predicate.to_name_s, value_properties) do
+            if(is_property)
+              @builder.rdf :Description, 'rdf:about' => value.uri.to_s
+            else
+              @builder.text!(value)
+            end
+          end
+        end
+
+        # Splits up the value, extracting encoded language codes and RDF data types. The 
+        # result will be returned as a hash, with the "true" value being "value"
+        def extract_values(value)
+          result = {}
+          # First split for the type
+          type_split = value.split('^^')
+          # Check if any of the elements contains a language string
+          type_split = type_split.collect { |element| extract_lang(element, result) }
+          result['rdf:datatype'] = type_split.last if(type_split.size > 1)
+          result['value'] = (type_split.first || '')
+          result
+        end
+
+        # Helper to extract a language string. The lang value, if any, will be added to the hash
+        def extract_lang(value, hash)
+          lang_split = value.split('@')
+          hash['xml:lang'] = lang_split.last if(lang_split.size > 1)
+          lang_split.first || ''
+        end
+      end
+    end 
+  end
+end

data/lib/talia_core/active_source_parts/xml/source_builder.rb

@@ -0,0 +1,73 @@
+module TaliaCore
+  module ActiveSourceParts
+    module Xml
+      
+      # Class to build source representations of ActiveSource objects. Talia 
+      # uses a simple XML format to encode the Source Object. The format
+      # maps easily to a Hash as it is used for the new or write_attributes 
+      # methods:
+      #
+      #   <sources>
+      #     <source>
+      #       <attribute>
+      #         <predicate>http://foobar/</predicate>
+      #         <object>http://barbar/</object>
+      #       </attribute>
+      #       ...
+      #     </source>
+      #     <source>
+      #       <attribute>
+      #         <predicate>http://foobar/bar/</pedicate>
+      #         <value>val</value>
+      #         <object>http://some_url</object>
+      #         <value>another</value>
+      #         ...
+      #       </attribute>
+      #       ...
+      #     </source>
+      #     ...
+      #   </sources>
+      class SourceBuilder < BaseBuilder
+        
+        # Builds the RDF for a single source
+        def write_source(source)
+          @builder.source do 
+            source.attributes.each { |attrib, value| write_attribute(attrib, value) }
+            source.direct_predicates.each { |pred| write_attribute(pred, source[pred]) } 
+          end
+        end
+        
+        private
+        
+        # build an attribute entry in a source
+        def write_attribute(predicate, values)
+          predicate = predicate.respond_to?(:uri) ? predicate.uri.to_s : predicate.to_s
+          values = [ values ] unless(values.respond_to?(:each))
+          @builder.attribute do 
+            @builder.predicate { @builder.text!(predicate) }
+            values.each { |val| write_target(val) }
+          end
+        end
+        
+        # Writes a value or object tag, depeding on the target
+        def write_target(target)
+          if(target.respond_to?(:uri))
+            @builder.object { @builder.text!(target.uri.to_s) }
+          else
+            @builder.value { @builder.text!(target.to_s) }
+          end
+        end
+        
+        # Build the structure for the XML file and pass on to
+        # the given block
+        def build_structure
+          @builder.sources do 
+            yield
+          end
+        end
+        
+      end
+    
+    end
+  end
+end

data/lib/talia_core/active_source_parts/xml/source_reader.rb

@@ -0,0 +1,20 @@
+module TaliaCore
+  module ActiveSourceParts
+    module Xml
+
+      # Helper class to read an attribute hash from a Source XML
+      class SourceReader < GenericReader
+
+        element :source do
+          nested :attribute do 
+            add from_element(:predicate), all_elements(:value)
+            add_rel from_element(:predicate), all_elements(:object)
+          end
+          add_file all_elements(:file)
+        end
+
+      end
+      
+    end
+  end
+end

data/lib/talia_core/agent.rb

@@ -0,0 +1,14 @@
+module TaliaCore
+
+  # Some item that "has the power to act". This can either be a person or another
+  # entity, like an institution or a corporation
+  class Agent < Source
+
+    has_rdf_type N::DCT.Agent
+
+    singular_property :name, N::DCNS.title
+    singular_property :description, N::DCNS.description
+
+  end
+
+end

data/lib/talia_core/background_jobs/job.rb

@@ -0,0 +1,82 @@
+module TaliaCore
+  module BackgroundJobs
+
+    # Exception class for blocked jobs
+    class JobBlockedError < RuntimeError
+    end
+
+    # A Background job run from Talia. The Job class is designed to be able to run long-running
+    # tasks both from the command line and Talia's background job runner.
+    class Job
+
+      # Creates a background job with progress metering. If a tag is given, it will attempt to block
+      # the creation of further jobs with the same tag. This will also use the runner script, called
+      # with the current ruby binary
+      def self.submit_with_progress(jobs, options = {})
+        # add the runner script and ruby call to the jobs
+        jobs = make_jobs(jobs)
+        Bj.submit(jobs, options) do |job|
+          if(tag = job.tag)
+            tagged = Bj.table.job.find(:all, :conditions => ["(state != 'finished' and state != 'dead' and tag = ?)", tag])
+            # The error will break the transation and leave the db in a clean state
+            raise(JobBlockedError, "Tried to create another job with tag #{tag}.") unless(tagged.size == 1)
+          end
+          # Update the environment so the runner can find the job id
+          job.env['JOB_ID'] ||= job.id.to_s
+          job.save! 
+          job
+        end
+      end
+
+      # Runs the block with an active progress meter, creating the progress object before starting, and
+      # deleting it from the db after completion. This way the progress_jobs table should remain mostly
+      # clean.
+      def self.run_progress_job
+        job_id = ENV['JOB_ID']
+        raise(RuntimeError, "Cannot run job: Job id not given or non-existent (#{job_id})") unless(job_id && Bj.table.job.exists?(job_id))
+        ProgressJob.create_progress!(job_id) unless(ProgressJob.exists?(:job_id => job_id))
+        yield
+      ensure
+        job_id = ENV['JOB_ID']
+        ProgressJob.delete(:job_id => job_id)
+      end
+
+      # Runs the block with the progress meter for the current job. This may be used multiple times.
+      def self.run_with_progress(message, item_count)
+        # Wrap this in the progress job call. This way it will work fine standalone
+        run_progress_job do
+          job_id = ENV['JOB_ID']
+          # Create the progress meter
+          progress = ProgressJob.find(:first, :conditions => {:job_id => job_id})
+          raise(RuntimeError, 'Progress meter not found for job.') unless progress
+          progress.update_attributes(:item_count => item_count, :progress_message => message, :processed_count => 0, :started_at => Time.now)
+
+          yield(progress)
+
+          progress.finish
+        end
+      end
+
+      private
+
+      def self.ruby
+        return @ruby if(@ruby)
+        c = ::Config::CONFIG
+        ruby = File.join(c['bindir'], c['ruby_install_name']) << c['EXEEXT']
+        @ruby = if system('%s -e 42' % ruby)
+          ruby
+        else
+          system('%s -e 42' % 'ruby') ? 'ruby' : warn('no ruby in PATH/CONFIG')
+        end
+      end
+
+      # Make the caller line for each job
+      def self.make_jobs(jobs)
+        jobs = [ jobs ] unless(jobs.kind_of?(Array))
+        jobs.collect { |current_job| "#{ruby} #{File.join('.', 'script', 'runner')} #{File.join('jobs', current_job)}" }
+      end
+
+    end
+
+  end
+end