active_encode 0.2 → 0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3346c9f9ae44a17bf60b9f71dbf6a3bd8b951883
-  data.tar.gz: 8f228d1184a1f61629bbed814176d131bc98de40
+  metadata.gz: 6e507e3110f65efc706aed54986fd98043fb0911
+  data.tar.gz: 0a5a69eabd90c026e23258156f2d4f93a136a4fd
 SHA512:
-  metadata.gz: 47fde2ffc94ad11ef0b8ab3af29672f3c8707128d887d2dd48249f0e13346b66b5450434ab1d79f1b840bf2ac695c05f5cd6f1ef11ce697d16bbd63193a5ad8e
-  data.tar.gz: 74073edc357c19e84fb51c80044684dceae704dcb8bef6061a49b62b269c78aa60fb47563db691d00fe798e76bfd7af88467290a4c135438beff51efbc2989b7
+  metadata.gz: 8fcd0939d74b37b03a61f93fc37baa2960bf2591b1e11430e07e816439b5102d66d71ee422ad87dcbab3da0020e2e7a01400f0730f7ed28b3ce3a46a546b0acc
+  data.tar.gz: b0e11f988144fce3220a7a551f0a1cb9c66f832023e89a1e989caf942c666932daddc60ca9b778b444b5c27de39827e11f0e8eb7a2fecf9304b65c1cf6d12c6e

data/.travis.yml

@@ -1,5 +1,14 @@
 language: ruby
 cache: bundler
+before_install:
+#  - sudo apt-get install mediainfo
+   - sudo apt-get install libmms0
+   - wget https://mediaarea.net/download/binary/libzen0/0.4.37/libzen0_0.4.37-1_amd64.xUbuntu_14.04.deb
+   - sudo dpkg -i libzen0_0.4.37-1_amd64.xUbuntu_14.04.deb
+   - wget https://mediaarea.net/download/binary/libmediainfo0/18.08.1/libmediainfo0_18.08.1-1_amd64.xUbuntu_14.04.deb
+   - sudo dpkg -i libmediainfo0_18.08.1-1_amd64.xUbuntu_14.04.deb
+   - wget https://mediaarea.net/download/binary/mediainfo/18.08.1/mediainfo_18.08.1-1_amd64.xUbuntu_14.04.deb
+   - sudo dpkg -i mediainfo_18.08.1-1_amd64.xUbuntu_14.04.deb
 sudo: false
 rvm:
   - 2.3.7

data/README.md

@@ -52,20 +52,7 @@ encode.cancel!
 encode.cancelled?  # true
 ```
 
-> `#purge!` and `#remove_output!` and the following documentation have been deprecated and will be removed in ActiveEncode 0.3.
-
-If the encoding job should be deleted, call purge:
-```ruby
-encode.purge!
-```
-
-Purge will attempt to remove all outputs that have been generated.  It is also possible to remove only a single output using its id:
-
-```ruby
-encode.remove_output! 'track-9'
-```
-
-An encoding job is meant to be the record of the work of the encoding engine and not the current state of the outputs.  Therefore removing outputs will not be reflected in the encoding job.
+An encoding job is meant to be the record of the work of the encoding engine and not the current state of the outputs.  Therefore removed outputs will not be reflected in the encoding job.
 
 ### Custom jobs
 
@@ -94,8 +81,6 @@ Engine adapters are shims between ActiveEncode and the back end encoding service
 | Zencoder                 |    X   |  X   |   X    |        |                  |
 | Matterhorn               |    X   |  X   |   X    |   X    |         X        |
 
-> The Inline and Shingoncoder adapters are deprecated and will be removed in ActiveEncode 0.3.
-
 ## Contributing
 
 1. Fork it ( https://github.com/projecthydra-labs/active_encode/fork )

data/active_encode.gemspec

@@ -7,12 +7,12 @@ require 'active_encode/version'
 Gem::Specification.new do |spec|
   spec.name          = "active_encode"
   spec.version       = ActiveEncode::VERSION
-  spec.authors       = ["Michael Klein, Chris Colvard"]
-  spec.email         = ["mbklein@gmail.com, chris.colvard@gmail.com"]
+  spec.authors       = ["Michael Klein, Chris Colvard, Phuong Dinh"]
+  spec.email         = ["mbklein@gmail.com, chris.colvard@gmail.com, phuongdh@gmail.com"]
   spec.summary       = 'Declare encode job classes that can be run by a variety of encoding services'
   spec.description   = 'This gem serves as the basis for the interface between a Ruby (Rails) application and a provider of transcoding services such as Opencast Matterhorn, Zencoder, and Amazon Elastic Transcoder.'
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.homepage      = "https://github.com/samvera-labs/active_encode"
+  spec.license       = "Apache-2.0"
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }

data/app/jobs/active_encode/polling_job.rb

@@ -4,12 +4,12 @@ module ActiveEncode
     def perform(encode)
       encode.run_callbacks(:status_update) { encode }
       case encode.state
-      when :error
-        encode.run_callbacks(:error) { encode }
+      when :failed
+        encode.run_callbacks(:failed) { encode }
       when :cancelled
         encode.run_callbacks(:cancelled) { encode }
-      when :complete
-        encode.run_callbacks(:complete) { encode }
+      when :completed
+        encode.run_callbacks(:completed) { encode }
       when :running
         ActiveEncode::PollingJob.set(wait: ActiveEncode::Polling::POLLING_WAIT_TIME).perform_later(encode)
       else # other states are illegal and ignored

data/lib/active_encode/callbacks.rb

@@ -14,39 +14,20 @@ module ActiveEncode
   # * <tt>before_cancel</tt>
   # * <tt>around_cancel</tt>
   # * <tt>after_cancel</tt>
-  # * <tt>before_purge</tt>
-  # * <tt>around_purge</tt>
-  # * <tt>after_purge</tt>
   #
   module Callbacks
     extend ActiveSupport::Concern
 
     CALLBACKS = [
       :after_find, :after_reload, :before_create, :around_create,
-      :after_create, :before_cancel, :around_cancel, :after_cancel,
-      :before_purge, :around_purge, :after_purge
+      :after_create, :before_cancel, :around_cancel, :after_cancel
     ].freeze
 
     included do
       extend ActiveModel::Callbacks
 
       define_model_callbacks :find, :reload, only: :after
-      define_model_callbacks :create, :cancel, :purge
-
-      def self.before_purge(*filters, &blk)
-        ActiveSupport::Deprecation.warn("before_purge will be removed without replacement in ActiveEncode 0.3")
-        set_callback(:purge, :before, *filters, &blk)
-      end
-
-      def self.after_purge(*filters, &blk)
-        ActiveSupport::Deprecation.warn("after_purge will be removed without replacement in ActiveEncode 0.3")
-        set_callback(:purge, :after, *filters, &blk)
-      end
-
-      def self.around_purge(*filters, &blk)
-        ActiveSupport::Deprecation.warn("around_purge will be removed without replacement in ActiveEncode 0.3")
-        set_callback(:purge, :around, *filters, &blk)
-      end
+      define_model_callbacks :create, :cancel
     end
   end
 end

data/lib/active_encode/core.rb

@@ -22,9 +22,6 @@ module ActiveEncode
 
       attr_accessor :current_operations
       attr_accessor :percent_complete
-
-      # @deprecated
-      attr_accessor :tech_metadata
     end
 
     module ClassMethods
@@ -44,20 +41,14 @@ module ActiveEncode
           encode.send(:merge!, engine_adapter.find(id))
         end
       end
-
-      def list(*args)
-        ActiveSupport::Deprecation.warn("#list will be removed without replacement in ActiveEncode 0.3")
-        engine_adapter.list(args)
-      end
     end
 
     def initialize(input_url, options = nil)
       @input = Input.new.tap{ |input| input.url = input_url }
-      @options = options || self.class.default_options(input_url)
+      @options = self.class.default_options(input_url).merge(Hash(options))
     end
 
     def create!
-      # TODO: Raise ArgumentError if self has an id?
       run_callbacks :create do
         merge!(self.class.engine_adapter.create(self.input.url, self.options))
       end
@@ -69,18 +60,6 @@ module ActiveEncode
       end
     end
 
-    def purge!
-      ActiveSupport::Deprecation.warn("#purge! will be removed without replacement in ActiveEncode 0.3")
-      run_callbacks :purge do
-        self.class.engine_adapter.purge self
-      end
-    end
-
-    def remove_output!(output_id)
-      ActiveSupport::Deprecation.warn("#remove_output will be removed without replacement in ActiveEncode 0.3")
-      self.class.engine_adapter.remove_output self, output_id
-    end
-
     def reload
       run_callbacks :reload do
         merge!(self.class.engine_adapter.find(id))
@@ -91,15 +70,6 @@ module ActiveEncode
       !id.nil?
     end
 
-    # @deprecated
-    def tech_metadata
-      metadata = {}
-      [:width, :height, :frame_rate, :duration, :file_size,
-       :audio_codec, :video_codec, :audio_bitrate, :video_bitrate, :checksum].each do |key|
-        metadata[key] = input.send(key)
-      end
-    end
-
     protected
 
       def merge!(encode)
@@ -114,10 +84,6 @@ module ActiveEncode
         @current_operations = encode.current_operations
         @percent_complete = encode.percent_complete
 
-        # deprecated
-        @tech_metadata = encode.tech_metadata
-        @finished_at = encode.finished_at
-
         self
       end
   end

data/lib/active_encode/engine_adapters.rb

@@ -8,13 +8,11 @@ module ActiveEncode
   module EngineAdapters
     extend ActiveSupport::Autoload
 
-    autoload :ActiveJobAdapter
     autoload :MatterhornAdapter
-    autoload :InlineAdapter
     autoload :ZencoderAdapter
-    autoload :ShingoncoderAdapter
     autoload :ElasticTranscoderAdapter
     autoload :TestAdapter
+    autoload :FfmpegAdapter
 
     ADAPTER = 'Adapter'.freeze
     private_constant :ADAPTER

data/lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb

@@ -18,25 +18,12 @@ module ActiveEncode
         build_encode(get_job_details(id))
       end
 
-      # TODO: implement list_jobs_by_pipeline and list_jobs_by_status
-      def list(*_filters)
-        raise NotImplementedError
-      end
-
       # Can only cancel jobs with status = "Submitted"
       def cancel(id)
         response = client.cancel_job(id: id)
         build_encode(get_job_details(id)) if response.successful?
       end
 
-      def purge(_encode)
-        raise NotImplementedError
-      end
-
-      def remove_output(_encode, _output_id)
-        raise NotImplementedError
-      end
-
       private
 
         # Needs region and credentials setup per http://docs.aws.amazon.com/sdkforruby/api/Aws/ElasticTranscoder/Client.html
@@ -56,11 +43,20 @@ module ActiveEncode
           encode.current_operations = convert_current_operations(job)
           encode.percent_complete = convert_percent_complete(job)
           encode.created_at = convert_time(job.timing["submit_time_millis"])
-          encode.updated_at = convert_time(job.timing["start_time_millis"])
-          encode.finished_at = convert_time(job.timing["finish_time_millis"])
+          encode.updated_at = convert_time(job.timing["finish_time_millis"] || job.timing["start_time_millis"]) || encode.created_at
           encode.output = convert_output(job)
           encode.errors = convert_errors(job)
-          encode.tech_metadata = convert_tech_metadata(job.input.detected_properties)
+
+          encode.input.id = job.input.key
+          tech_md = convert_tech_metadata(job.input.detected_properties)
+          [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
+           :audio_bitrate, :video_bitrate, :file_size].each do |field|
+            encode.input.send("#{field}=", tech_md[field])
+          end
+          encode.input.state = encode.state
+          encode.input.created_at = encode.created_at
+          encode.input.updated_at = encode.updated_at
+
           encode
         end
 
@@ -109,17 +105,27 @@ module ActiveEncode
         end
 
         def convert_output(job)
-          output = []
-          job.outputs.each do |o|
+          job.outputs.collect do |o|
             # It is assumed that the first part of the output key can be used to label the  output
             # e.g. "quality-medium/somepath/filename.flv"
-            label = o.key.split("/", 2).first
-            url = job.output_key_prefix + o.key
-            extras = { id: o.id, url: url, label: label }
-            extras[:hls_url] = url + ".m3u8" if url.include?("/hls/") # TODO: find a better way to signal hls
-            output << convert_tech_metadata(o).merge(extras)
+            output = ActiveEncode::Output.new
+            output.id = o.id
+            output.label = o.key.split("/", 2).first
+            output.url = job.output_key_prefix + o.key
+            # TODO: If HLS is considered distinct from this output then it should be a different output
+            # TODO: If HLS is not considered distinct from this output then this should be handled by a method on a ActiveEncode::Base subclass or consuming client
+            # extras[:hls_url] = url + ".m3u8" if url.include?("/hls/") # TODO: find a better way to signal hls
+            tech_md = convert_tech_metadata(o)
+            [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
+             :audio_bitrate, :video_bitrate, :file_size].each do |field|
+              output.send("#{field}=", tech_md[field])
+            end
+            output.state = convert_state(o)
+            output.created_at = convert_time(job.timing["submit_time_millis"])
+            output.updated_at = convert_time(job.timing["finish_time_millis"] || job.timing["start_time_millis"]) || output.created_at
+
+            output
           end
-          output
         end
 
         def convert_errors(job)
@@ -130,9 +136,8 @@ module ActiveEncode
           return {} if props.blank?
           metadata_fields = {
             file_size: { key: :file_size, method: :itself },
-            duration_millis: { key: :duration, method: :to_s },
-            frame_rate: { key: :video_framerate, method: :itself },
-            segment_duration: { key: :segment_duration, method: :itself },
+            duration_millis: { key: :duration, method: :to_i },
+            frame_rate: { key: :frame_rate, method: :to_i },
             width: { key: :width, method: :itself },
             height: { key: :height, method: :itself }
           }
@@ -144,7 +149,6 @@ module ActiveEncode
             next if conversion.nil?
             metadata[conversion[:key]] = value.send(conversion[:method])
           end
-
           metadata
         end
     end

data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb

@@ -0,0 +1,243 @@
+require 'fileutils'
+require 'nokogiri'
+
+module ActiveEncode
+  module EngineAdapters
+    class FfmpegAdapter
+      WORK_DIR = ENV["ENCODE_WORK_DIR"] || "encodes" # Should read from config
+
+      def create(input_url, options = {})
+        new_encode = ActiveEncode::Base.new(input_url, options)
+        new_encode.id = SecureRandom.uuid
+        new_encode.created_at = Time.new
+        new_encode.updated_at = Time.new
+        new_encode.current_operations = []
+        new_encode.output = []
+
+        # Create a working directory that holds all output files related to the encode
+        FileUtils.mkdir_p working_path("", new_encode.id)
+        FileUtils.mkdir_p working_path("outputs", new_encode.id)
+
+        # Extract technical metadata from input file
+        `mediainfo --Output=XML --LogFile=#{working_path("input_metadata", new_encode.id)} #{input_url}`
+        new_encode.input = build_input new_encode
+
+        if new_encode.input.duration.blank?
+          new_encode.state = :failed
+          new_encode.percent_complete = 1
+
+          if new_encode.input.file_size.blank?
+            new_encode.errors = ["#{input_url} does not exist or is not accessible"]
+          else
+            new_encode.errors = ["Error inspecting input: #{input_url}"]
+          end
+
+          write_errors new_encode
+          return new_encode
+        end
+
+        new_encode.state = :running
+        new_encode.percent_complete = 1
+        new_encode.errors = []
+
+        # Run the ffmpeg command and save its pid
+        command = ffmpeg_command(input_url, new_encode.id, options)
+        pid = Process.spawn(command)
+        File.open(working_path("pid", new_encode.id), 'w') { |file| file.write pid }
+        new_encode.input.id = pid
+
+        # Prevent zombie process
+        Process.detach(pid)
+
+        new_encode
+      end
+
+      # Return encode object from file system
+      def find(id, opts={})
+        encode_class = opts[:cast]
+        encode = ActiveEncode::Base.new(nil, opts)
+        encode.id = id
+        encode.output = []
+        encode.created_at, encode.updated_at = get_times encode.id
+        encode.input = build_input encode
+        encode.percent_complete = calculate_percent_complete encode
+
+        pid = get_pid(id)
+        encode.input.id = pid if pid.present?
+
+        if File.file? working_path("error.log", id)
+          error = File.read working_path("error.log", id)
+          if error.present?
+            encode.state = :failed
+            encode.errors = [error]
+
+            return encode
+          end
+        end
+
+        encode.errors = []
+
+        encode.current_operations = []
+        encode.created_at, encode.updated_at = get_times encode.id
+
+        if running? pid
+          encode.state = :running
+          encode.current_operations = ["transcoding"]
+        elsif progress_ended?(encode.id) && encode.percent_complete == 100
+            encode.state = :completed
+        elsif encode.percent_complete < 100
+            encode.state = :cancelled
+        end
+
+        encode.output = build_outputs encode if encode.completed?
+
+        encode
+      end
+
+      # Cancel ongoing encode using pid file
+      def cancel(id)
+        pid = get_pid(id)
+        Process.kill 'SIGTERM', pid.to_i
+
+        find id
+      end
+
+private
+
+      def get_times id
+        updated_at = if File.file? working_path("progress", id)
+            File.mtime(working_path("progress", id))
+          elsif File.file? working_path("error.log", id)
+            File.mtime(working_path("error.log", id))
+          else
+            File.mtime(working_path("input_metadata", id))
+          end
+
+        return File.mtime(working_path("input_metadata", id)), updated_at
+      end
+
+      def write_errors encode
+        File.write(working_path("error.log", encode.id), encode.errors.join("\n"))
+      end
+
+      def build_input encode
+        input = ActiveEncode::Input.new
+        metadata = get_tech_metadata(working_path("input_metadata", encode.id))
+        input.url = metadata[:url]
+        input.assign_tech_metadata(metadata)
+        input.created_at = encode.created_at
+        input.updated_at = encode.created_at
+        input.id = "N/A"
+
+        input
+      end
+
+      def build_outputs encode
+        id = encode.id
+        outputs = []
+        Dir["#{File.absolute_path(working_path('outputs', id))}/*"].each do |file_path|
+          output = ActiveEncode::Output.new
+          output.url = "file://#{file_path}"
+          original_extension = File.extname(encode.input.url)
+          original_filename = File.basename(encode.input.url).chomp(original_extension)
+          output.label = file_path[/#{Regexp.quote(original_filename)}\-(.*?)#{Regexp.quote(File.extname(file_path))}$/, 1]
+          output.id = "#{encode.input.id}-#{output.label}"
+          output.created_at = encode.created_at
+          output.updated_at = File.mtime file_path
+
+          # Extract technical metadata from output file
+          metadata_path = working_path("output_metadata-#{output.label}", id)
+          `mediainfo --Output=XML --LogFile=#{metadata_path} #{output.url}` unless File.file? metadata_path
+          output.assign_tech_metadata(get_tech_metadata(metadata_path))
+
+          outputs << output
+        end
+
+        outputs
+      end
+
+      def ffmpeg_command(input_url, id, opts)
+        output_opt = opts[:outputs].collect do |output|
+          file_name = "outputs/#{File.basename(input_url, File.extname(input_url))}-#{output[:label]}.#{output[:extension]}"
+          " #{output[:ffmpeg_opt]} #{working_path(file_name, id)}"
+        end.join(" ")
+
+        "ffmpeg -y -loglevel error -progress #{working_path("progress", id)} -i #{input_url} #{output_opt} > #{working_path("error.log", id)} 2>&1"
+      end
+
+      def get_pid(id)
+        if File.file? working_path("pid", id)
+          File.read(working_path("pid", id)).remove("\n")
+        else
+          nil
+        end
+      end
+
+      def working_path(path, id)
+        File.join(WORK_DIR, id, path)
+      end
+
+      def running?(pid)
+        begin
+          Process.getpgid pid.to_i
+          true
+        rescue Errno::ESRCH
+          false
+        end
+      end
+
+      def calculate_percent_complete encode
+        data = read_progress encode.id
+        if data.blank?
+          1
+        else
+          (progress_value("out_time_ms=", data).to_i * 0.0001 / encode.input.duration).round
+        end
+      end
+
+      def read_progress id
+        if File.file? working_path("progress", id)
+          File.read working_path("progress", id)
+        else
+          nil
+        end
+      end
+
+      def progress_ended? id
+        "end" == progress_value("progress=", read_progress(id))
+      end
+
+      def progress_value key, data
+        if data.present? && key.present?
+          ri = data.rindex(key) + key.length
+          data[ri..data.index("\n", ri)-1]
+        else
+          nil
+        end
+      end
+
+      def get_tech_metadata file_path
+        doc = Nokogiri::XML File.read(file_path)
+        doc.remove_namespaces!
+        { url: get_xpath_text(doc, '//media/@ref', :to_s),
+          width: get_xpath_text(doc, '//Width/text()', :to_f),
+          height: get_xpath_text(doc, '//Height/text()', :to_f),
+          frame_rate: get_xpath_text(doc, '//FrameRate/text()', :to_f),
+          duration: get_xpath_text(doc, '//Duration/text()', :to_f),
+          file_size: get_xpath_text(doc, '//FileSize/text()', :to_i),
+          audio_codec: get_xpath_text(doc, '//track[@type="Audio"]/CodecID/text()', :to_s),
+          audio_bitrate: get_xpath_text(doc, '//track[@type="Audio"]/BitRate/text()', :to_i),
+          video_codec: get_xpath_text(doc, '//track[@type="Video"]/CodecID/text()', :to_s),
+          video_bitrate: get_xpath_text(doc, '//track[@type="Video"]/BitRate/text()', :to_i) }
+      end
+
+      def get_xpath_text doc, xpath, cast_method
+        if doc.xpath(xpath).first
+          doc.xpath(xpath).first.text.send(cast_method)
+        else
+          nil
+        end
+      end
+    end
+  end
+end