active_encode 0.0.2

data/active_encode.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'active_encode/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "active_encode"
+  spec.version       = ActiveEncode::VERSION
+  spec.authors       = ["Michael Klein, Chris Colvard"]
+  spec.email         = ["mbklein@gmail.com, chris.colvard@gmail.com"]
+  spec.summary       = %q{Declare encode job classes that can be run by a variety of encoding services}
+  spec.description   = %q{This gem serves as the basis for the interface between a Ruby (Rails) application and a provider of transcoding services such as Opencast Matterhorn, Zencoder, and Amazon Elastic Transcoder.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec-its", "~> 1.2"
+end

data/lib/active_encode.rb

@@ -0,0 +1,2 @@
+require 'active_encode/version'
+require 'active_encode/base'

data/lib/active_encode/base.rb

@@ -0,0 +1,16 @@
+require 'active_encode/core'
+require 'active_encode/engine_adapter'
+require 'active_encode/status'
+require 'active_encode/technical_metadata'
+require 'active_encode/callbacks'
+# require 'active_encode/logging'
+
+module ActiveEncode #:nodoc:
+  class Base
+    include Core
+    include Status
+    include TechnicalMetadata
+    include EngineAdapter
+    include Callbacks
+  end
+end

data/lib/active_encode/callbacks.rb

@@ -0,0 +1,69 @@
+require 'active_support/callbacks'
+
+module ActiveEncode
+  # = Active Encode Callbacks
+  #
+  # Active Encode provides hooks during the life cycle of an encode. Callbacks allow you
+  # to trigger logic during the life cycle of an encode. Available callbacks are:
+  #
+  # * <tt>before_create</tt>
+  # * <tt>around_create</tt>
+  # * <tt>after_create</tt>
+  # * <tt>before_cancel</tt>
+  # * <tt>around_cancel</tt>
+  # * <tt>after_cancel</tt>
+  # * <tt>before_purge</tt>
+  # * <tt>around_purge</tt>
+  # * <tt>after_purge</tt>
+  #
+  module Callbacks
+    extend ActiveSupport::Concern
+    include ActiveSupport::Callbacks
+
+    included do
+      define_callbacks :create
+      define_callbacks :cancel
+      define_callbacks :purge
+    end
+
+    # These methods will be included into any Active Encode object, adding
+    # callbacks for +create+, +cancel+, and +purge+ methods.
+    module ClassMethods
+      def before_create(*filters, &blk)
+        set_callback(:create, :before, *filters, &blk)
+      end
+
+      def after_create(*filters, &blk)
+        set_callback(:create, :after, *filters, &blk)
+      end
+
+      def around_create(*filters, &blk)
+        set_callback(:create, :around, *filters, &blk)
+      end
+
+      def before_cancel(*filters, &blk)
+        set_callback(:cancel, :before, *filters, &blk)
+      end
+
+      def after_cancel(*filters, &blk)
+        set_callback(:cancel, :after, *filters, &blk)
+      end
+
+      def around_cancel(*filters, &blk)
+        set_callback(:cancel, :around, *filters, &blk)
+      end
+
+      def before_purge(*filters, &blk)
+        set_callback(:purge, :before, *filters, &blk)
+      end
+
+      def after_purge(*filters, &blk)
+        set_callback(:purge, :after, *filters, &blk)
+      end
+
+      def around_purge(*filters, &blk)
+        set_callback(:purge, :around, *filters, &blk)
+      end
+    end
+  end
+end

data/lib/active_encode/core.rb

@@ -0,0 +1,79 @@
+require 'active_support'
+require 'active_encode/callbacks'
+
+module ActiveEncode
+  module Core
+    extend ActiveSupport::Concern
+
+    included do
+      # Encode Identifier
+      attr_accessor :id
+
+      # Encode input
+      attr_accessor :input
+
+      # Encode output(s)
+      attr_accessor :output
+
+      # Encode options
+      attr_accessor :options
+    end
+
+    module ClassMethods
+      def default_options(_input)
+        {}
+      end
+
+      def create(input, options = nil)
+        object = new(input, options)
+        object.create!
+      end
+
+      def find(id)
+        engine_adapter.find(id, cast: self)
+      end
+
+      delegate :list, to: :engine_adapter
+    end
+
+    def initialize(input, options = nil)
+      @input = input
+      @options = options || self.class.default_options(input)
+    end
+
+    def create!
+      run_callbacks :create do
+        self.class.engine_adapter.create self
+      end
+    end
+
+    def cancel!
+      run_callbacks :cancel do
+        self.class.engine_adapter.cancel self
+      end
+    end
+
+    def purge!
+      run_callbacks :purge do
+        self.class.engine_adapter.purge self
+      end
+    end
+
+    def remove_output!(output_id)
+      self.class.engine_adapter.remove_output self, output_id
+    end
+
+    def reload
+      fresh_encode = self.class.engine_adapter.find(id, cast: self.class)
+      @id = fresh_encode.id
+      @input = fresh_encode.input
+      @output = fresh_encode.output
+      @state = fresh_encode.state
+      @current_operations = fresh_encode.current_operations
+      @errors = fresh_encode.errors
+      @tech_metadata = fresh_encode.tech_metadata
+
+      self
+    end
+  end
+end

data/lib/active_encode/engine_adapter.rb

@@ -0,0 +1,51 @@
+require 'active_encode/engine_adapters'
+require 'active_support/core_ext/class/attribute'
+require 'active_support/core_ext/string/inflections'
+
+module ActiveEncode
+  # The <tt>ActiveEncode::EngineAdapter</tt> module is used to load the
+  # correct adapter. The default engine adapter is the :active_job engine.
+  module EngineAdapter #:nodoc:
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :_engine_adapter, instance_accessor: false, instance_predicate: false
+      self.engine_adapter = :inline
+    end
+
+    # Includes the setter method for changing the active engine adapter.
+    module ClassMethods
+      def engine_adapter
+        _engine_adapter
+      end
+
+      # Specify the backend engine provider. The default engine adapter
+      # is the :inline engine. See QueueAdapters for more
+      # information.
+      def engine_adapter=(name_or_adapter_or_class)
+        self._engine_adapter = interpret_adapter(name_or_adapter_or_class)
+      end
+
+      private
+
+        def interpret_adapter(name_or_adapter_or_class)
+          case name_or_adapter_or_class
+          when Symbol, String
+            ActiveEncode::EngineAdapters.lookup(name_or_adapter_or_class).new
+          else
+            if engine_adapter?(name_or_adapter_or_class)
+              name_or_adapter_or_class
+            else
+              fail ArgumentError
+            end
+          end
+        end
+
+        ENGINE_ADAPTER_METHODS = [:create, :find, :list, :cancel, :purge, :remove_output].freeze
+
+        def engine_adapter?(object)
+          ENGINE_ADAPTER_METHODS.all? { |meth| object.respond_to?(meth) }
+        end
+    end
+  end
+end

data/lib/active_encode/engine_adapters.rb

@@ -0,0 +1,27 @@
+module ActiveEncode
+  # == Active Encode adapters
+  #
+  # Active Encode has adapters for the following engines:
+  #
+  #
+  #
+  module EngineAdapters
+    extend ActiveSupport::Autoload
+
+    autoload :ActiveJobAdapter
+    autoload :MatterhornAdapter
+    autoload :InlineAdapter
+    autoload :ZencoderAdapter
+    autoload :ShingoncoderAdapter
+    autoload :TestAdapter
+
+    ADAPTER = 'Adapter'.freeze
+    private_constant :ADAPTER
+
+    class << self
+      def lookup(name)
+        const_get(name.to_s.camelize << ADAPTER)
+      end
+    end
+  end
+end

data/lib/active_encode/engine_adapters/active_job_adapter.rb

@@ -0,0 +1,23 @@
+module ActiveEncode
+  module EngineAdapters
+    class ActiveJobAdapter
+      def create(_encode)
+      end
+
+      def find(_id, _opts = {})
+      end
+
+      def list(*_filters)
+      end
+
+      def cancel(_encode)
+      end
+
+      def purge(_encode)
+      end
+
+      def remove_output(_encode, _output_id)
+      end
+    end
+  end
+end

data/lib/active_encode/engine_adapters/inline_adapter.rb

@@ -0,0 +1,42 @@
+module ActiveEncode
+  module EngineAdapters
+    class InlineAdapter
+      class_attribute :encodes, instance_accessor: false, instance_predicate: false
+      InlineAdapter.encodes ||= {}
+
+      def create(encode)
+        encode.id = SecureRandom.uuid
+        self.class.encodes[encode.id] = encode
+        # start encode
+        encode.state = :running
+        encode
+      end
+
+      def find(id, _opts = {})
+        self.class.encodes[id]
+      end
+
+      def list(*_filters)
+        fail NotImplementedError
+      end
+
+      def cancel(encode)
+        inline_encode = self.class.encodes[encode.id]
+        return if inline_encode.nil?
+        inline_encode.state = :cancelled
+        # cancel encode
+        inline_encode
+      end
+
+      def purge(encode)
+        self.class.encodes.delete encode.id
+      end
+
+      def remove_output(encode, output_id)
+        inline_encode = self.class.encodes[encode.id]
+        return if inline_encode.nil?
+        inline_encode.output.delete(inline_encode.output.find { |o| o[:id] == output_id })
+      end
+    end
+  end
+end

data/lib/active_encode/engine_adapters/matterhorn_adapter.rb

@@ -0,0 +1,312 @@
+require 'rubyhorn'
+
+module ActiveEncode
+  module EngineAdapters
+    class MatterhornAdapter
+      DEFAULT_ARGS = { 'flavor' => 'presenter/source' }
+
+      def create(encode)
+        workflow_id = encode.options[:preset] || "full"
+        workflow_om = if encode.input.is_a? Hash
+                        create_multiple_files(encode.input, workflow_id)
+                      else
+                        Rubyhorn.client.addMediaPackageWithUrl(DEFAULT_ARGS.merge('workflow' => workflow_id, 'url' => encode.input, 'filename' => File.basename(encode.input), 'title' => File.basename(encode.input)))
+                      end
+        build_encode(get_workflow(workflow_om), encode.class)
+      end
+
+      def find(id, opts = {})
+        build_encode(fetch_workflow(id), opts[:cast])
+      end
+
+      def list(*_filters)
+        fail NotImplementedError # TODO: implement this
+      end
+
+      def cancel(encode)
+        workflow_om = Rubyhorn.client.stop(encode.id)
+        build_encode(get_workflow(workflow_om), encode.class)
+      end
+
+      def purge(encode)
+        workflow_om = begin
+                        Rubyhorn.client.stop(encode.id)
+                      rescue
+                        nil
+                      end
+        workflow_om ||= begin
+                          Rubyhorn.client.get_stopped_workflow(encode.id)
+                        rescue
+                          nil
+                        end
+        purged_workflow = purge_outputs(get_workflow(workflow_om))
+        # Rubyhorn.client.delete_instance(encode.id) #Delete is not working so workflow instances can always be retrieved later!
+        build_encode(purged_workflow, encode.class)
+      end
+
+      def remove_output(encode, output_id)
+        workflow = fetch_workflow(encode.id)
+        output = encode.output.find { |o| o[:id] == output_id }
+        return if output.nil?
+        purge_output(workflow, output_id)
+        output
+      end
+
+      private
+
+        def fetch_workflow(id)
+          workflow_om = begin
+            Rubyhorn.client.instance_xml(id)
+          rescue Rubyhorn::RestClient::Exceptions::HTTPNotFound
+            nil
+          end
+
+          workflow_om ||= begin
+            Rubyhorn.client.get_stopped_workflow(id)
+          rescue
+            nil
+          end
+
+          get_workflow(workflow_om)
+        end
+
+        def get_workflow(workflow_om)
+          return nil if workflow_om.nil?
+          if workflow_om.ng_xml.is_a? Nokogiri::XML::Document
+            workflow_om.ng_xml.remove_namespaces!.root
+          else
+            workflow_om.ng_xml
+          end
+        end
+
+        def build_encode(workflow, cast)
+          return nil if workflow.nil?
+          encode = cast.new(convert_input(workflow), convert_options(workflow))
+          encode.id = convert_id(workflow)
+          encode.state = convert_state(workflow)
+          encode.current_operations = convert_current_operations(workflow)
+          encode.percent_complete = calculate_percent_complete(workflow)
+          encode.output = convert_output(workflow, encode.options)
+          encode.errors = convert_errors(workflow)
+          encode.tech_metadata = convert_tech_metadata(workflow)
+          encode
+        end
+
+        def convert_id(workflow)
+          workflow.attribute('id').to_s
+        end
+
+        def convert_state(workflow)
+          case workflow.attribute('state').to_s
+          when "INSTANTIATED", "RUNNING" # Should there be a queued state?
+            :running
+          when "STOPPED"
+            :cancelled
+          when "FAILED"
+            workflow.xpath('//operation[@state="FAILED"]').empty? ? :cancelled : :failed
+          when "SUCCEEDED", "SKIPPED" # Should there be a errored state?
+            :completed
+          end
+        end
+
+        def convert_input(workflow)
+          # Need to do anything else since this is a MH url? and this disappears when a workflow is cleaned up
+          workflow.xpath('mediapackage/media/track[@type="presenter/source"]/url/text()').to_s
+        end
+
+        def convert_tech_metadata(workflow)
+          convert_track_metadata(workflow.xpath('//track[@type="presenter/source"]').first)
+        end
+
+        def convert_output(workflow, options)
+          output = []
+          workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"]]').each do |track|
+            label = track.xpath('tags/tag[starts-with(text(),"quality")]/text()').to_s
+            url = track.at("url/text()").to_s
+            if url.start_with? "rtmp"
+              url = File.join(options[:stream_base], MatterhornRtmpUrl.parse(url).to_path) if options[:stream_base]
+            end
+            track_id = track.at("@id").to_s
+            output << convert_track_metadata(track).merge(id: track_id, url: url, label: label)
+          end
+          output
+        end
+
+        def convert_current_operations(workflow)
+          current_op = workflow.xpath('//operation[@state!="INSTANTIATED"]/@description').last.to_s
+          current_op.present? ? [current_op] : []
+        end
+
+        def convert_errors(workflow)
+          workflow.xpath('//errors/error/text()').map(&:to_s)
+        end
+
+        def convert_options(workflow)
+          options = {}
+          options[:preset] = workflow.xpath('template/text()').to_s
+          options[:stream_base] = workflow.xpath('//properties/property[@key="avalon.stream_base"]/text()').to_s if workflow.xpath('//properties/property[@key="avalon.stream_base"]/text()').present? # this is avalon-felix specific
+          options
+        end
+
+        def convert_track_metadata(track)
+          return {} if track.nil?
+          metadata = {}
+          metadata[:mime_type] = track.at("mimetype/text()").to_s if track.at('mimetype')
+          metadata[:checksum] = track.at("checksum/text()").to_s if track.at('checksum')
+          metadata[:duration] = track.at("duration/text()").to_s if track.at('duration')
+          if track.at('audio')
+            metadata[:audio_codec] = track.at("audio/encoder/@type").to_s
+            metadata[:audio_channels] = track.at("audio/channels/text()").to_s
+            metadata[:audio_bitrate] = track.at("audio/bitrate/text()").to_s
+          end
+          if track.at('video')
+            metadata[:video_codec] = track.at("video/encoder/@type").to_s
+            metadata[:video_bitrate] = track.at("video/bitrate/text()").to_s
+            metadata[:video_framerate] = track.at("video/framerate/text()").to_s
+            metadata[:width] = track.at("video/resolution/text()").to_s.split('x')[0]
+            metadata[:height] = track.at("video/resolution/text()").to_s.split('x')[1]
+          end
+          metadata
+        end
+
+        def get_media_package(workflow)
+          mp = workflow.xpath('//mediapackage')
+          first_node = mp.first
+          first_node['xmlns'] = 'http://mediapackage.opencastproject.org'
+          mp
+        end
+
+        def purge_outputs(workflow)
+          # Delete hls tracks first since the next, more general xpath matches them as well
+          workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"] and tags/tag[text()="hls"]]/@id').map(&:to_s).each do |hls_track_id|
+            begin
+              purge_output(workflow, hls_track_id)
+            rescue
+              nil
+            end
+          end
+          workflow.xpath('//track[@type="presenter/delivery" and tags/tag[text()="streaming"]]/@id').map(&:to_s).each do |track_id|
+            begin
+              purge_output(workflow, track_id)
+            rescue
+              nil
+            end
+          end
+
+          workflow
+        end
+
+        def purge_output(workflow, track_id)
+          media_package = get_media_package(workflow)
+          hls = workflow.xpath("//track[@id='#{track_id}']/tags/tag[text()='hls']").present?
+          job_url = if hls
+                      Rubyhorn.client.delete_hls_track(media_package, track_id)
+                    else
+                      Rubyhorn.client.delete_track(media_package, track_id)
+                    end
+          sleep(0.1)
+          job_status = Nokogiri::XML(Rubyhorn.client.get(URI(job_url).path)).root.attribute("status").value
+          # FIXME: have this return a boolean based upon result of operation
+          case job_status
+          when "FINISHED"
+            workflow.at_xpath("//track[@id=\"#{track_id}\"]").remove
+          when "FAILED"
+            workflow.at_xpath('//errors').add_child("<error>Output not purged: #{mp.at_xpath("//*[@id=\"#{track_id}\"]/tags/tag[starts-with(text(),\"quality\")]/text()")}</error>")
+          end
+        end
+
+        def calculate_percent_complete(workflow)
+          totals = {
+            transcode: 70,
+            distribution: 20,
+            other: 10
+          }
+
+          completed_transcode_operations = workflow.xpath('//operation[@id="compose" and (@state="SUCCEEDED" or @state="SKIPPED")]').size
+          total_transcode_operations = workflow.xpath('//operation[@id="compose"]').size
+          total_transcode_operations = 1 if total_transcode_operations == 0
+          completed_distribution_operations = workflow.xpath('//operation[starts-with(@id,"distribute") and (@state="SUCCEEDED" or @state="SKIPPED")]').size
+          total_distribution_operations = workflow.xpath('//operation[starts-with(@id,"distribute")]').size
+          total_distribution_operations = 1 if total_distribution_operations == 0
+          completed_other_operations = workflow.xpath('//operation[@id!="compose" and not(starts-with(@id,"distribute")) and (@state="SUCCEEDED" or @state="SKIPPED")]').size
+          total_other_operations = workflow.xpath('//operation[@id!="compose" and not(starts-with(@id,"distribute"))]').size
+          total_other_operations = 1 if total_other_operations == 0
+
+          ((totals[:transcode].to_f / total_transcode_operations) * completed_transcode_operations) +
+            ((totals[:distribution].to_f / total_distribution_operations) * completed_distribution_operations) +
+            ((totals[:other].to_f / total_other_operations) * completed_other_operations)
+        end
+
+        def create_multiple_files(input, workflow_id)
+          # Create empty media package xml document
+          mp = Rubyhorn.client.createMediaPackage
+
+          # Next line associates workflow title to avalon via masterfile pid
+          title = File.basename(input.values.first)
+          dc = Nokogiri::XML('<dublincore xmlns="http://www.opencastproject.org/xsd/1.0/dublincore/" xmlns:dcterms="http://purl.org/dc/terms/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><dcterms:title>' + title + '</dcterms:title></dublincore>')
+          mp = Rubyhorn.client.addDCCatalog('mediaPackage' => mp.to_xml, 'dublinCore' => dc.to_xml, 'flavor' => 'dublincore/episode')
+
+          # Add quality levels - repeated for each supplied file url
+          input.each_pair do |quality, url|
+            mp = Rubyhorn.client.addTrack('mediaPackage' => mp.to_xml, 'url' => url, 'flavor' => DEFAULT_ARGS['flavor'])
+            # Rewrite track to include quality tag
+            # Get the empty tags element under the newly added track
+            tags = mp.xpath('//xmlns:track/xmlns:tags[not(node())]', 'xmlns' => 'http://mediapackage.opencastproject.org').first
+            quality_tag = Nokogiri::XML::Node.new 'tag', mp
+            quality_tag.content = quality
+            tags.add_child quality_tag
+          end
+          # Finally ingest the media package
+          begin
+                  Rubyhorn.client.start("definitionId" => workflow_id, "mediapackage" => mp.to_xml)
+                rescue Rubyhorn::RestClient::Exceptions::HTTPBadRequest
+                  # make this two calls...one to get the workflow definition xml and then the second to submit it along with the mediapackage to start...due to unsolved issue with some MH installs
+                  begin
+                          workflow_definition_xml = Rubyhorn.client.definition_xml(workflow_id)
+                          Rubyhorn.client.start("definition" => workflow_definition_xml, "mediapackage" => mp.to_xml)
+                        rescue Rubyhorn::RestClient::Exceptions::HTTPNotFound
+                          raise StandardError, "Unable to start workflow"
+                        end
+                end
+        end
+    end
+
+    class MatterhornRtmpUrl
+      class_attribute :members
+      self.members = [:application, :prefix, :media_id, :stream_id, :filename, :extension]
+      attr_accessor(*members)
+      REGEX = %r{^
+  /(?<application>.+)        # application (avalon)
+  /(?:(?<prefix>.+):)?       # prefix      (mp4:)
+  (?<media_id>[^\/]+)        # media_id    (98285a5b-603a-4a14-acc0-20e37a3514bb)
+  /(?<stream_id>[^\/]+)      # stream_id   (b3d5663d-53f1-4f7d-b7be-b52fd5ca50a3)
+  /(?<filename>.+?)          # filename    (MVI_0057)
+  (?:\.(?<extension>.+))?$   # extension   (mp4)
+      }x
+
+      # @param [MatchData] match_data
+      def initialize(match_data)
+        self.class.members.each do |key|
+          send("#{key}=", match_data[key])
+        end
+      end
+
+      def self.parse(url_string)
+        # Example input: /avalon/mp4:98285a5b-603a-4a14-acc0-20e37a3514bb/b3d5663d-53f1-4f7d-b7be-b52fd5ca50a3/MVI_0057.mp4
+
+        uri = URI.parse(url_string)
+        match_data = REGEX.match(uri.path)
+        MatterhornRtmpUrl.new match_data
+      end
+
+      alias_method :_binding, :binding
+      def binding
+        _binding
+      end
+
+      def to_path
+        File.join(media_id, stream_id, "#{filename}.#{extension || prefix}")
+      end
+    end
+  end
+end