active-encode 0.0.1

data/lib/active_encode/callbacks.rb

@@ -0,0 +1,69 @@
+require 'active_support/callbacks'
+
+module ActiveEncode
+  # = Active Encode Callbacks
+  #
+  # Active Encode provides hooks during the life cycle of an encode. Callbacks allow you
+  # to trigger logic during the life cycle of an encode. Available callbacks are:
+  #
+  # * <tt>before_create</tt>
+  # * <tt>around_create</tt>
+  # * <tt>after_create</tt>
+  # * <tt>before_cancel</tt>
+  # * <tt>around_cancel</tt>
+  # * <tt>after_cancel</tt>
+  # * <tt>before_purge</tt>
+  # * <tt>around_purge</tt>
+  # * <tt>after_purge</tt>
+  #
+  module Callbacks
+    extend  ActiveSupport::Concern
+    include ActiveSupport::Callbacks
+
+    included do
+      define_callbacks :create
+      define_callbacks :cancel
+      define_callbacks :purge
+    end
+
+    # These methods will be included into any Active Encode object, adding
+    # callbacks for +create+, +cancel+, and +purge+ methods.
+    module ClassMethods
+      def before_create(*filters, &blk)
+        set_callback(:create, :before, *filters, &blk)
+      end
+      
+      def after_create(*filters, &blk)
+        set_callback(:create, :after, *filters, &blk)
+      end
+
+      def around_create(*filters, &blk)
+        set_callback(:create, :around, *filters, &blk)
+      end
+
+      def before_cancel(*filters, &blk)
+        set_callback(:cancel, :before, *filters, &blk)
+      end
+      
+      def after_cancel(*filters, &blk)
+        set_callback(:cancel, :after, *filters, &blk)
+      end
+
+      def around_cancel(*filters, &blk)
+        set_callback(:cancel, :around, *filters, &blk)
+      end
+
+      def before_purge(*filters, &blk)
+        set_callback(:purge, :before, *filters, &blk)
+      end
+      
+      def after_purge(*filters, &blk)
+        set_callback(:purge, :after, *filters, &blk)
+      end
+
+      def around_purge(*filters, &blk)
+        set_callback(:purge, :around, *filters, &blk)
+      end
+    end
+  end
+end

data/lib/active_encode/core.rb

@@ -0,0 +1,81 @@
+require 'active_support'
+require 'active_encode/callbacks'
+
+module ActiveEncode
+  module Core
+    extend ActiveSupport::Concern
+
+    included do
+      # Encode Identifier
+      attr_accessor :id
+
+      # Encode input
+      attr_accessor :input
+
+      # Encode output(s)
+      attr_accessor :output
+
+      # Encode options
+      attr_accessor :options
+    end
+
+    module ClassMethods
+      def default_options(input)
+        {}
+      end
+
+      def create(input, options = nil)
+        object = new(input, options)
+        object.create!
+      end 
+
+      def find(id)
+        engine_adapter.find(id, cast: self)
+      end
+
+      def list(*filters)
+        engine_adapter.list(filters)
+      end
+    end
+
+    def initialize(input, options = nil)
+      @input = input
+      @options = options || self.class.default_options(input)
+    end
+
+    def create!
+      run_callbacks :create do
+        self.class.engine_adapter.create self
+      end
+    end
+
+    def cancel!
+      run_callbacks :cancel do
+        self.class.engine_adapter.cancel self
+      end
+    end
+
+    def purge!
+      run_callbacks :purge do
+        self.class.engine_adapter.purge self
+      end
+    end
+
+    def remove_output! output_id
+      self.class.engine_adapter.remove_output self, output_id
+    end
+
+    def reload
+      fresh_encode = self.class.engine_adapter.find(id, cast: self.class)
+      @id = fresh_encode.id
+      @input = fresh_encode.input
+      @output = fresh_encode.output
+      @state = fresh_encode.state
+      @current_operations = fresh_encode.current_operations
+      @errors = fresh_encode.errors
+      @tech_metadata = fresh_encode.tech_metadata
+
+      self
+    end
+  end
+end

data/lib/active_encode/engine_adapter.rb

@@ -0,0 +1,52 @@
+require 'active_encode/engine_adapters'
+require 'active_support/core_ext/class/attribute'
+require 'active_support/core_ext/string/inflections'
+
+module ActiveEncode
+  # The <tt>ActiveEncode::EngineAdapter</tt> module is used to load the
+  # correct adapter. The default engine adapter is the :active_job engine.
+  module EngineAdapter #:nodoc:
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :_engine_adapter, instance_accessor: false, instance_predicate: false
+      self.engine_adapter = :inline
+    end
+
+    # Includes the setter method for changing the active engine adapter.
+    module ClassMethods
+      def engine_adapter
+        _engine_adapter
+      end
+
+      # Specify the backend engine provider. The default engine adapter
+      # is the :inline engine. See QueueAdapters for more
+      # information.
+      def engine_adapter=(name_or_adapter_or_class)
+        self._engine_adapter = interpret_adapter(name_or_adapter_or_class)
+      end
+
+      private
+
+      def interpret_adapter(name_or_adapter_or_class)
+        case name_or_adapter_or_class
+        when Symbol, String
+          ActiveEncode::EngineAdapters.lookup(name_or_adapter_or_class).new
+        else
+          if engine_adapter?(name_or_adapter_or_class)
+            name_or_adapter_or_class
+          else
+            raise ArgumentError
+          end
+        end
+      end
+
+      ENGINE_ADAPTER_METHODS = [:create, :find, :list, :cancel, :purge, :remove_output].freeze
+
+      def engine_adapter?(object)
+        ENGINE_ADAPTER_METHODS.all? { |meth| object.respond_to?(meth) }
+      end
+
+    end
+  end
+end

data/lib/active_encode/engine_adapters.rb

@@ -0,0 +1,26 @@
+module ActiveEncode
+  # == Active Encode adapters
+  #
+  # Active Encode has adapters for the following engines:
+  #
+  #
+  #
+  module EngineAdapters
+    extend ActiveSupport::Autoload
+
+    autoload :ActiveJobAdapter
+    autoload :MatterhornAdapter
+    autoload :InlineAdapter
+    autoload :ZencoderAdapter
+    autoload :TestAdapter
+
+    ADAPTER = 'Adapter'.freeze
+    private_constant :ADAPTER
+
+    class << self
+      def lookup(name)
+        const_get(name.to_s.camelize << ADAPTER)
+      end
+    end
+  end
+end

data/lib/active_encode/engine_adapters/active_job_adapter.rb

@@ -0,0 +1,23 @@
+module ActiveEncode
+  module EngineAdapters
+    class ActiveJobAdapter
+      def create(encode)
+      end
+
+      def find(id, opts = {})
+      end
+
+      def list(*filters)
+      end
+
+      def cancel(encode)
+      end
+
+      def purge(encode)
+      end
+
+      def remove_output(encode, output_id)
+      end
+    end
+  end
+end

data/lib/active_encode/engine_adapters/inline_adapter.rb

@@ -0,0 +1,42 @@
+module ActiveEncode
+  module EngineAdapters
+    class InlineAdapter
+      class_attribute :encodes, instance_accessor: false, instance_predicate: false
+      InlineAdapter.encodes ||= {}
+
+      def create(encode)
+        encode.id = SecureRandom.uuid
+        self.class.encodes[encode.id] = encode
+        #start encode
+        encode.state = :running
+        encode
+      end
+
+      def find(id, opts = {})
+        self.class.encodes[id]
+      end
+
+      def list(*filters)
+        raise NotImplementedError
+      end
+
+      def cancel(encode)
+        inline_encode = self.class.encodes[encode.id]
+        return if inline_encode.nil?
+        inline_encode.state = :cancelled
+        #cancel encode
+        inline_encode
+      end
+
+      def purge(encode)
+        self.class.encodes.delete encode.id
+      end
+
+      def remove_output(encode, output_id)
+        inline_encode = self.class.encodes[encode.id]
+        return if inline_encode.nil?
+        inline_encode.output.delete(inline_encode.output.find {|o| o[:id] == output_id})
+      end
+    end
+  end
+end

data/lib/active_encode/engine_adapters/matterhorn_adapter.rb

@@ -0,0 +1,290 @@
+require 'rubyhorn'
+
+module ActiveEncode
+  module EngineAdapters
+    class MatterhornAdapter
+      DEFAULT_ARGS = {'flavor' => 'presenter/source'}
+
+      def create(encode)
+        workflow_id = encode.options[:preset] || "full"
+        workflow_om = if encode.input.is_a? Hash
+          createMultipleFiles(encode.input, workflow_id)
+        else
+          Rubyhorn.client.addMediaPackageWithUrl(DEFAULT_ARGS.merge({'workflow' => workflow_id, 'url' => encode.input, 'filename' => File.basename(encode.input), 'title' => File.basename(encode.input)}))
+        end
+        build_encode(get_workflow(workflow_om), encode.class)
+      end
+
+      def find(id, opts = {})
+        build_encode(fetch_workflow(id), opts[:cast])
+      end
+
+      def list(*filters)
+        raise NotImplementedError #TODO implement this
+      end
+
+      def cancel(encode)
+        workflow_om = Rubyhorn.client.stop(encode.id)
+        build_encode(get_workflow(workflow_om), encode.class)
+      end
+
+      def purge(encode)
+        workflow_om = Rubyhorn.client.stop(encode.id) rescue nil
+        workflow_om ||= Rubyhorn.client.get_stopped_workflow(encode.id) rescue nil
+        purged_workflow = purge_outputs(get_workflow(workflow_om))
+       #Rubyhorn.client.delete_instance(encode.id) #Delete is not working so workflow instances can always be retrieved later!
+        build_encode(purged_workflow, encode.class)
+      end
+
+      def remove_output(encode, output_id)
+        workflow = fetch_workflow(encode.id)
+        output = encode.output.find {|o| o[:id] == output_id}
+        return if output.nil?
+        purge_output(workflow, output_id)
+        output
+      end
+
+      private
+      def fetch_workflow(id)
+        workflow_om = begin
+          Rubyhorn.client.instance_xml(id)
+        rescue Rubyhorn::RestClient::Exceptions::HTTPNotFound
+          nil
+        end
+
+        workflow_om ||= begin
+          Rubyhorn.client.get_stopped_workflow(id)
+        rescue
+          nil
+        end
+
+        get_workflow(workflow_om)
+      end
+
+      def get_workflow(workflow_om)
+        return nil if workflow_om.nil?
+        if workflow_om.ng_xml.is_a? Nokogiri::XML::Document
+          workflow_om.ng_xml.remove_namespaces!.root
+        else
+          workflow_om.ng_xml
+        end
+      end
+
+      def build_encode(workflow, cast)
+        return nil if workflow.nil?
+        encode = cast.new(convert_input(workflow), convert_options(workflow))
+        encode.id = convert_id(workflow)
+        encode.state = convert_state(workflow)
+        encode.current_operations = convert_current_operations(workflow)
+        encode.percent_complete = calculate_percent_complete(workflow)
+        encode.output = convert_output(workflow, encode.options)
+        encode.errors = convert_errors(workflow)
+        encode.tech_metadata = convert_tech_metadata(workflow)
+        encode
+      end
+
+      def convert_id(workflow)
+        workflow.attribute('id').to_s
+      end
+
+      def convert_state(workflow)
+        case workflow.attribute('state').to_s
+        when "INSTANTIATED", "RUNNING" #Should there be a queued state?
+          :running
+        when "STOPPED"
+          :cancelled
+        when "FAILED"
+          workflow.xpath('//operation[@state="FAILED"]').empty? ? :cancelled : :failed
+        when "SUCCEEDED", "SKIPPED" #Should there be a errored state?
+          :completed
+        end
+      end
+
+      def convert_input(workflow)
+        #Need to do anything else since this is a MH url? and this disappears when a workflow is cleaned up
+        workflow.xpath('mediapackage/media/track[@type="presenter/source"]/url/text()').to_s
+      end
+
+      def convert_tech_metadata(workflow)
+        convert_track_metadata(workflow.xpath('//track[@type="presenter/source"]').first)
+      end
+
+      def convert_output(workflow, options)
+        output = []
+        workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"]]').each do |track|
+          label = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
+          url = track.at("url/text()").to_s
+          if url.start_with? "rtmp"
+            url = File.join(options[:stream_base], MatterhornRtmpUrl.parse(url).to_path) if options[:stream_base]
+          end
+          track_id = track.at("@id").to_s
+          output << convert_track_metadata(track).merge({id: track_id, url: url, label: label})
+        end
+        output
+      end
+
+      def convert_current_operations(workflow)
+        current_op = workflow.xpath('//operation[@state!="INSTANTIATED"]/@description').last.to_s
+        current_op.present? ? [current_op] : []
+      end
+
+      def convert_errors(workflow)
+        workflow.xpath('//errors/error/text()').map(&:to_s)
+      end
+
+      def convert_options(workflow)
+        options = {}
+        options[:preset] = workflow.xpath('template/text()').to_s
+        options[:stream_base] = workflow.xpath('//properties/property[@key="avalon.stream_base"]/text()').to_s if workflow.xpath('//properties/property[@key="avalon.stream_base"]/text()').present? #this is avalon-felix specific
+        options
+      end
+
+      def convert_track_metadata(track)
+        return {} if track.nil?
+        metadata = {}
+        metadata[:mime_type] = track.at("mimetype/text()").to_s if track.at('mimetype')
+        metadata[:checksum] = track.at("checksum/text()").to_s if track.at('checksum')
+        metadata[:duration] = track.at("duration/text()").to_s if track.at('duration')
+        if track.at('audio')
+          metadata[:audio_codec] = track.at("audio/encoder/@type").to_s 
+          metadata[:audio_channels] = track.at("audio/channels/text()").to_s
+          metadata[:audio_bitrate] = track.at("audio/bitrate/text()").to_s
+        end
+        if track.at('video')
+          metadata[:video_codec] = track.at("video/encoder/@type").to_s
+          metadata[:video_bitrate] = track.at("video/bitrate/text()").to_s
+          metadata[:video_framerate] = track.at("video/framerate/text()").to_s
+          metadata[:width] = track.at("video/resolution/text()").to_s.split('x')[0]
+          metadata[:height] = track.at("video/resolution/text()").to_s.split('x')[1]
+        end
+        metadata
+      end
+
+      def get_media_package(workflow)
+        mp = workflow.xpath('//mediapackage')
+        first_node = mp.first
+        first_node['xmlns'] = 'http://mediapackage.opencastproject.org'
+        mp
+      end
+
+      def purge_outputs(workflow)
+        #Delete hls tracks first since the next, more general xpath matches them as well
+        workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"] and tags/tag[text()="hls"]]/@id').map(&:to_s).each do |hls_track_id|
+          purge_output(workflow, hls_track_id) rescue nil
+        end
+        workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"]]/@id').map(&:to_s).each do |track_id|
+          purge_output(workflow, track_id) rescue nil
+        end
+
+        workflow
+      end 
+
+      def purge_output(workflow, track_id)
+        media_package = get_media_package(workflow)
+        hls = workflow.xpath("//track[@id='#{track_id}']/tags/tag[text()='hls']").present?
+        job_url = if hls
+          Rubyhorn.client.delete_hls_track(media_package, track_id)
+        else
+          Rubyhorn.client.delete_track(media_package, track_id)
+        end
+        sleep(0.1)
+        job_status = Nokogiri::XML(Rubyhorn.client.get(URI(job_url).path)).root.attribute("status").value()
+        #FIXME have this return a boolean based upon result of operation
+        case job_status
+        when "FINISHED"
+          workflow.at_xpath("//track[@id=\"#{track_id}\"]").remove
+        when "FAILED"
+          workflow.at_xpath('//errors').add_child("<error>Output not purged: #{mp.at_xpath("//*[@id=\"#{track_id}\"]/tags/tag[starts-with(text(),\"quality\")]/text()").to_s}</error>")
+        end
+      end
+
+      def calculate_percent_complete workflow
+        totals = {
+          :transcode => 70,
+          :distribution => 20,
+          :other => 10
+        }
+
+        completed_transcode_operations = workflow.xpath('//operation[@id="compose" and (@state="SUCCEEDED" or @state="SKIPPED")]').size
+        total_transcode_operations = workflow.xpath('//operation[@id="compose"]').size
+        total_transcode_operations = 1 if total_transcode_operations == 0
+        completed_distribution_operations = workflow.xpath('//operation[starts-with(@id,"distribute") and (@state="SUCCEEDED" or @state="SKIPPED")]').size
+        total_distribution_operations = workflow.xpath('//operation[starts-with(@id,"distribute")]').size
+        total_distribution_operations = 1 if total_distribution_operations == 0
+        completed_other_operations = workflow.xpath('//operation[@id!="compose" and not(starts-with(@id,"distribute")) and (@state="SUCCEEDED" or @state="SKIPPED")]').size 
+        total_other_operations = workflow.xpath('//operation[@id!="compose" and not(starts-with(@id,"distribute"))]').size
+        total_other_operations = 1 if total_other_operations == 0
+
+        ((totals[:transcode].to_f / total_transcode_operations) * completed_transcode_operations) +
+        ((totals[:distribution].to_f / total_distribution_operations) * completed_distribution_operations) +
+        ((totals[:other].to_f / total_other_operations) * completed_other_operations)
+      end
+
+      def createMultipleFiles(input, workflow_id)
+	#Create empty media package xml document
+	mp = Rubyhorn.client.createMediaPackage
+
+	#Next line associates workflow title to avalon via masterfile pid
+        title = File.basename(input.values.first)
+	dc = Nokogiri::XML('<dublincore xmlns="http://www.opencastproject.org/xsd/1.0/dublincore/" xmlns:dcterms="http://purl.org/dc/terms/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><dcterms:title>' + title + '</dcterms:title></dublincore>')
+	mp = Rubyhorn.client.addDCCatalog({'mediaPackage' => mp.to_xml, 'dublinCore' => dc.to_xml, 'flavor' => 'dublincore/episode'})
+
+	#Add quality levels - repeated for each supplied file url
+	input.each_pair do |quality, url|
+	  mp = Rubyhorn.client.addTrack({'mediaPackage' => mp.to_xml, 'url' => url, 'flavor' => DEFAULT_ARGS['flavor']})
+	  #Rewrite track to include quality tag
+	  #Get the empty tags element under the newly added track
+	  tags = mp.xpath('//xmlns:track/xmlns:tags[not(node())]', 'xmlns' => 'http://mediapackage.opencastproject.org').first
+	  qualityTag = Nokogiri::XML::Node.new 'tag', mp
+	  qualityTag.content = quality
+	  tags.add_child qualityTag
+	end
+	#Finally ingest the media package
+	begin
+	  Rubyhorn.client.start({"definitionId" => workflow_id, "mediapackage" => mp.to_xml})
+        rescue Rubyhorn::RestClient::Exceptions::HTTPBadRequest
+	  #make this two calls...one to get the workflow definition xml and then the second to submit it along with the mediapackage to start...due to unsolved issue with some MH installs
+	  begin
+            workflow_definition_xml = Rubyhorn.client.definition_xml(workflow_id)
+            Rubyhorn.client.start({"definition" => workflow_definition_xml, "mediapackage" => mp.to_xml})
+          rescue Rubyhorn::RestClient::Exceptions::HTTPNotFound
+            raise StandardError.new("Unable to start workflow")
+          end
+        end
+      end
+    end
+
+    class MatterhornRtmpUrl < Struct.new(:application, :prefix, :media_id, :stream_id, :filename, :extension)
+
+      REGEX = %r{^
+	/(?<application>.+)        # application (avalon)
+	/(?:(?<prefix>.+):)?       # prefix      (mp4:)
+	(?<media_id>[^\/]+)        # media_id    (98285a5b-603a-4a14-acc0-20e37a3514bb)
+	/(?<stream_id>[^\/]+)      # stream_id   (b3d5663d-53f1-4f7d-b7be-b52fd5ca50a3)
+	/(?<filename>.+?)          # filename    (MVI_0057)
+	(?:\.(?<extension>.+))?$   # extension   (mp4)
+      }x
+
+      def initialize(hash)
+	super(*members.map {|member| hash[member]})
+      end
+
+      def self.parse(url_string)
+	# Example input: /avalon/mp4:98285a5b-603a-4a14-acc0-20e37a3514bb/b3d5663d-53f1-4f7d-b7be-b52fd5ca50a3/MVI_0057.mp4
+
+	uri = URI.parse(url_string)
+	match_data = REGEX.match(uri.path)
+	MatterhornRtmpUrl.new match_data
+      end
+
+      alias_method :'_binding', :'binding'
+      def binding
+	_binding
+      end
+
+      def to_path
+	File.join(media_id, stream_id, "#{filename}.#{extension||prefix}")
+      end
+    end
+  end
+end