active_encode 0.4.1 → 0.5.0

data/SUPPORT.md

@@ -0,0 +1,5 @@
+If you would like to report an issue, first search [the list of issues](https://github.com/samvera-labs/active_encode/issues/) to see if someone else has already reported it, and then feel free to [create a new issue](https://github.com/samvera-labs/active_encode/issues/new).
+
+If you have questions or need help, please email [the Samvera community tech list](https://groups.google.com/forum/#!forum/samvera-tech) or stop by the #dev channel in [the Samvera community Slack team](https://wiki.duraspace.org/pages/viewpage.action?pageId=87460391#Getintouch!-Slack).
+
+You can learn more about the various Samvera communication channels on the [Get in touch!](https://wiki.duraspace.org/pages/viewpage.action?pageId=87460391) wiki page.

data/active_encode.gemspec

@@ -21,10 +21,11 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "rails"
 
+  spec.add_development_dependency "aws-sdk"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "coveralls"
   spec.add_development_dependency "database_cleaner"
-  spec.add_development_dependency "engine_cart"
+  spec.add_development_dependency "engine_cart", "~> 2.2"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "rspec-its"

data/app/controllers/active_encode/encode_record_controller.rb

@@ -0,0 +1,12 @@
+module ActiveEncode
+  class EncodeRecordController < ActionController::Base
+    rescue_from ActiveRecord::RecordNotFound do |e|
+      render json: { message: e.message }, status: :not_found
+    end
+
+    def show
+      @encode_record = ActiveEncode::EncodeRecord.find(params[:id])
+      render json: @encode_record.raw_object, status: :ok
+    end
+  end
+end

data/config/routes.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+ActiveEncode::Engine.routes.draw do
+  resources :encode_record, only: [:show]
+end

data/lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb

@@ -1,12 +1,27 @@
+require 'addressable/uri'
+require 'aws-sdk'
+require 'file_locator'
+
 module ActiveEncode
   module EngineAdapters
     class ElasticTranscoderAdapter
-      # TODO: add a stub for an input helper (supplied by an initializer) that transforms encode.input into a zencoder accepted url
+
+      JOB_STATES = {
+        "Submitted" => :running, "Progressing" => :running, "Canceled" => :cancelled,
+        "Error" => :failed, "Complete" => :completed
+      }
+
+      # Require options to include :pipeline_id, :masterfile_bucket and :outputs
+      # Example :outputs value:
+      # [{ key: "quality-low/hls/fireworks", preset_id: '1494429796844-aza6zh', segment_duration: '2' },
+      #  { key: "quality-medium/hls/fireworks", preset_id: '1494429797061-kvg9ki', segment_duration: '2' },
+      #  { key: "quality-high/hls/fireworks", preset_id: '1494429797265-9xi831', segment_duration: '2' }]
       def create(input_url, options = {})
+        s3_key = copy_to_input_bucket input_url, options[:masterfile_bucket]
         job = client.create_job(
-          input: { key: input_url },
+          input: { key: s3_key },
           pipeline_id: options[:pipeline_id],
-          output_key_prefix: options[:output_key_prefix],
+          output_key_prefix: options[:output_key_prefix] || "#{SecureRandom.uuid}/",
           outputs: options[:outputs],
           user_metadata: options[:user_metadata]
         ).job
@@ -31,28 +46,33 @@ module ActiveEncode
           @client ||= Aws::ElasticTranscoder::Client.new
         end
 
+        def s3client
+          Aws::S3::Client.new
+        end
+
         def get_job_details(job_id)
-          client.read_job(id: job_id).job
+          client.read_job(id: job_id)&.job
         end
 
         def build_encode(job)
           return nil if job.nil?
-          encode = ActiveEncode::Base.new(convert_input(job), convert_options(job))
+          encode = ActiveEncode::Base.new(convert_input(job), {})
           encode.id = job.id
-          encode.state = convert_state(job)
-          encode.current_operations = convert_current_operations(job)
-          encode.percent_complete = convert_percent_complete(job)
-          encode.created_at = convert_time(job.timing["submit_time_millis"])
-          encode.updated_at = convert_time(job.timing["finish_time_millis"] || job.timing["start_time_millis"]) || encode.created_at
-          encode.output = convert_output(job)
-          encode.errors = convert_errors(job)
-
-          encode.input.id = job.input.key
+        	encode.state = JOB_STATES[job.status]
+        	encode.current_operations = []
+        	encode.percent_complete = convert_percent_complete(job)
+        	encode.created_at = convert_time(job.timing["submit_time_millis"])
+        	encode.updated_at = convert_time(job.timing["finish_time_millis"]) || convert_time(job.timing["start_time_millis"]) || encode.created_at
+
+        	encode.output = convert_output(job)
+        	encode.errors = job.outputs.select { |o| o.status == "Error" }.collect(&:status_detail).compact
+
           tech_md = convert_tech_metadata(job.input.detected_properties)
-          [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
-           :audio_bitrate, :video_bitrate, :file_size].each do |field|
+          [:width, :height, :frame_rate, :duration, :file_size].each do |field|
             encode.input.send("#{field}=", tech_md[field])
           end
+
+          encode.input.id = job.id
           encode.input.state = encode.state
           encode.input.created_at = encode.created_at
           encode.input.updated_at = encode.updated_at
@@ -65,6 +85,11 @@ module ActiveEncode
           Time.at(time_millis / 1000)
         end
 
+        def convert_bitrate(rate)
+          return nil if rate.nil?
+          (rate.to_f * 1024).to_s
+        end
+
         def convert_state(job)
           case job.status
           when "Submitted", "Progressing" # Should there be a queued state?
@@ -78,13 +103,12 @@ module ActiveEncode
           end
         end
 
-        def convert_current_operations(_job)
-          current_ops = []
-          current_ops
+        def convert_percent_complete(job)
+          job.outputs.inject(0) { |sum, output| sum + output_percentage(output) } / job.outputs.length
         end
 
-        def convert_percent_complete(job)
-          case job.status
+        def output_percentage(output)
+          case output.status
           when "Submitted"
             10
           when "Progressing", "Canceled", "Error"
@@ -100,29 +124,67 @@ module ActiveEncode
           job.input
         end
 
-        def convert_options(_job_details)
-          {}
+        def copy_to_input_bucket input_url, bucket
+          case Addressable::URI.parse(input_url).scheme
+          when nil,'file'
+            upload_to_s3 input_url, bucket
+          when 's3'
+            check_s3_bucket input_url, bucket
+          end
+        end
+
+        def check_s3_bucket input_url, source_bucket
+          # logger.info("Checking `#{input_url}'")
+          s3_object = FileLocator::S3File.new(input_url).object
+          if s3_object.bucket_name == source_bucket
+            # logger.info("Already in bucket `#{source_bucket}'")
+            s3_object.key
+          else
+            s3_key = File.join(SecureRandom.uuid,s3_object.key)
+            # logger.info("Copying to `#{source_bucket}/#{input_url}'")
+            target = Aws::S3::Object.new(bucket_name: source_bucket, key: input_url)
+            target.copy_from(s3_object, multipart_copy: s3_object.size > 15728640) # 15.megabytes
+            s3_key
+          end
+        end
+
+        def upload_to_s3 input_url, source_bucket
+          original_input = input_url
+          bucket = Aws::S3::Resource.new(client: s3client).bucket(source_bucket)
+          filename = FileLocator.new(input_url).location
+          s3_key = File.join(SecureRandom.uuid,File.basename(filename))
+          # logger.info("Copying `#{original_input}' to `#{source_bucket}/#{input_url}'")
+          obj = bucket.object(s3_key)
+          obj.upload_file filename
+
+          s3_key
+        end
+
+        def read_preset(id)
+          client.read_preset(id: id).preset
         end
 
         def convert_output(job)
-          job.outputs.collect do |o|
-            # It is assumed that the first part of the output key can be used to label the  output
-            # e.g. "quality-medium/somepath/filename.flv"
+          pipeline = client.read_pipeline(id: job.pipeline_id).pipeline
+          job.outputs.collect do |joutput|
+            preset = read_preset(joutput.preset_id)
+            extension = preset.container == 'ts' ? '.m3u8' : ''
+            tech_md = convert_tech_metadata(joutput, preset).merge({
+              managed: false,
+              id: joutput.id,
+              label: joutput.key.split("/", 2).first,
+              url: "s3://#{pipeline.output_bucket}/#{job.output_key_prefix}#{joutput.key}#{extension}"
+            })
+
             output = ActiveEncode::Output.new
-            output.id = o.id
-            output.label = o.key.split("/", 2).first
-            output.url = job.output_key_prefix + o.key
-            # TODO: If HLS is considered distinct from this output then it should be a different output
-            # TODO: If HLS is not considered distinct from this output then this should be handled by a method on a ActiveEncode::Base subclass or consuming client
-            # extras[:hls_url] = url + ".m3u8" if url.include?("/hls/") # TODO: find a better way to signal hls
-            tech_md = convert_tech_metadata(o)
+            output.state = convert_state(joutput)
+            output.created_at = convert_time(job.timing["submit_time_millis"])
+            output.updated_at = convert_time(job.timing["finish_time_millis"] || job.timing["start_time_millis"]) || output.created_at
+
             [:width, :height, :frame_rate, :duration, :checksum, :audio_codec, :video_codec,
-             :audio_bitrate, :video_bitrate, :file_size].each do |field|
+             :audio_bitrate, :video_bitrate, :file_size, :label, :url, :id].each do |field|
               output.send("#{field}=", tech_md[field])
             end
-            output.state = convert_state(o)
-            output.created_at = convert_time(job.timing["submit_time_millis"])
-            output.updated_at = convert_time(job.timing["finish_time_millis"] || job.timing["start_time_millis"]) || output.created_at
 
             output
           end
@@ -132,12 +194,13 @@ module ActiveEncode
           job.outputs.select { |o| o.status == "Error" }.collect(&:status_detail).compact
         end
 
-        def convert_tech_metadata(props)
-          return {} if props.blank?
+        def convert_tech_metadata(props, preset=nil)
+          return {} if props.nil? || props.empty?
           metadata_fields = {
             file_size: { key: :file_size, method: :itself },
             duration_millis: { key: :duration, method: :to_i },
             frame_rate: { key: :frame_rate, method: :to_i },
+            segment_duration: { key: :segment_duration, method: :itself },
             width: { key: :width, method: :itself },
             height: { key: :height, method: :itself }
           }
@@ -149,6 +212,19 @@ module ActiveEncode
             next if conversion.nil?
             metadata[conversion[:key]] = value.send(conversion[:method])
           end
+
+          unless preset.nil?
+            audio = preset.audio
+            video = preset.video
+            metadata.merge!({
+              audio_codec: audio&.codec,
+              audio_channels: audio&.channels,
+              audio_bitrate: convert_bitrate(audio&.bit_rate),
+              video_codec: video&.codec,
+              video_bitrate: convert_bitrate(video&.bit_rate)
+            })
+          end
+
           metadata
         end
     end

data/lib/active_encode/engine_adapters/ffmpeg_adapter.rb

@@ -191,7 +191,8 @@ private
         if data.blank?
           1
         else
-          (progress_value("out_time_ms=", data).to_i * 0.0001 / encode.input.duration).round
+          progress_in_milliseconds = progress_value("out_time_ms=", data).to_i / 1000.0
+          (progress_in_milliseconds / encode.input.duration * 100).round
         end
       end
 
@@ -219,11 +220,13 @@ private
       def get_tech_metadata file_path
         doc = Nokogiri::XML File.read(file_path)
         doc.remove_namespaces!
+        duration = get_xpath_text(doc, '//Duration/text()', :to_f)
+        duration = duration * 1000 unless duration.nil?      # Convert to milliseconds
         { url: get_xpath_text(doc, '//media/@ref', :to_s),
           width: get_xpath_text(doc, '//Width/text()', :to_f),
           height: get_xpath_text(doc, '//Height/text()', :to_f),
           frame_rate: get_xpath_text(doc, '//FrameRate/text()', :to_f),
-          duration: get_xpath_text(doc, '//Duration/text()', :to_f),
+          duration: duration,
           file_size: get_xpath_text(doc, '//FileSize/text()', :to_i),
           audio_codec: get_xpath_text(doc, '//track[@type="Audio"]/CodecID/text()', :to_s),
           audio_bitrate: get_xpath_text(doc, '//track[@type="Audio"]/BitRate/text()', :to_i),

data/lib/active_encode/version.rb

@@ -1,3 +1,3 @@
 module ActiveEncode
-  VERSION = '0.4.1'.freeze
+  VERSION = '0.5.0'.freeze
 end

data/lib/file_locator.rb

@@ -0,0 +1,94 @@
+require 'addressable/uri'
+require 'aws-sdk'
+
+class FileLocator
+  attr_reader :source
+
+  class S3File
+    attr_reader :bucket, :key
+
+    def initialize(uri)
+      uri = Addressable::URI.parse(uri)
+      @bucket = URI.decode(uri.host)
+      @key = URI.decode(uri.path).sub(%r(^/*(.+)/*$),'\1')
+    end
+
+    def object
+      @object ||= Aws::S3::Object.new(bucket_name: bucket, key: key)
+    end
+  end
+
+  def initialize(source)
+    @source = source
+  end
+
+  def uri
+    if @uri.nil?
+      if source.is_a? File
+        @uri = Addressable::URI.parse("file://#{URI.encode(File.expand_path(source))}")
+      else
+        encoded_source = source
+        begin
+          @uri = Addressable::URI.parse(encoded_source)
+        rescue URI::InvalidURIError
+          if encoded_source == source
+            encoded_source = URI.encode(encoded_source)
+            retry
+          else
+            raise
+          end
+        end
+
+        if @uri.scheme.nil?
+          @uri = Addressable::URI.parse("file://#{URI.encode(File.expand_path(source))}")
+        end
+      end
+    end
+    @uri
+  end
+
+  def location
+    case uri.scheme
+    when 's3'
+      S3File.new(uri).object.presigned_url(:get)
+    when 'file'
+      URI.decode(uri.path)
+    else
+      @uri.to_s
+    end
+  end
+
+  def exist?
+    case uri.scheme
+    when 's3'
+      S3File.new(uri).object.exists?
+    when 'file'
+      File.exist?(location)
+    else
+      false
+    end
+  end
+  alias_method :exists?, :exist?
+
+  def reader
+    case uri.scheme
+    when 's3'
+      S3File.new(uri).object.get.body
+    when 'file'
+      File.open(location,'r')
+    else
+      Kernel::open(uri.to_s, 'r')
+    end
+  end
+
+  def attachment
+    case uri.scheme
+    when 's3'
+      uri
+    when 'file'
+      File.open(location,'r')
+    else
+      location
+    end
+  end
+end

data/spec/controllers/encode_record_controller_spec.rb

@@ -0,0 +1,52 @@
+require 'rails_helper'
+
+describe ActiveEncode::EncodeRecordController, type: :controller, db_clean: true do
+  routes { ActiveEncode::Engine.routes }
+
+  let(:encode_record) { ActiveEncode::EncodeRecord.create(id: 1, global_id: "app://ActiveEncode/Encode/1", state: "running", adapter: "ffmpeg", title: "Test", raw_object: raw_object)}
+  let(:raw_object) do
+    "{\"input\":{\"url\":\"file:///Users/cjcolvar/Documents/Code/samvera/active_encode/spec/fixtures/fireworks.mp4\",\"width\":960.0,\"height\":540.0,\"frame_rate\":29.671,\"duration\":6024,\"file_size\":1629578,\"audio_codec\":\"mp4a-40-2\",\"video_codec\":\"avc1\",\"audio_bitrate\":69737,\"video_bitrate\":2092780,\"created_at\":\"2018-12-17T16:54:50.401-05:00\",\"updated_at\":\"2018-12-17T16:54:50.401-05:00\",\"id\":\"8156\"},\"options\":{},\"id\":\"35efa965-ec51-409d-9495-2ae9669adbcc\",\"output\":[{\"url\":\"file:///Users/cjcolvar/Documents/Code/samvera/active_encode/.internal_test_app/encodes/35efa965-ec51-409d-9495-2ae9669adbcc/outputs/fireworks-low.mp4\",\"label\":\"low\",\"id\":\"8156-low\",\"created_at\":\"2018-12-17T16:54:50.401-05:00\",\"updated_at\":\"2018-12-17T16:54:59.169-05:00\",\"width\":640.0,\"height\":480.0,\"frame_rate\":29.671,\"duration\":6038,\"file_size\":905987,\"audio_codec\":\"mp4a-40-2\",\"video_codec\":\"avc1\",\"audio_bitrate\":72000,\"video_bitrate\":1126859},{\"url\":\"file:///Users/cjcolvar/Documents/Code/samvera/active_encode/.internal_test_app/encodes/35efa965-ec51-409d-9495-2ae9669adbcc/outputs/fireworks-high.mp4\",\"label\":\"high\",\"id\":\"8156-high\",\"created_at\":\"2018-12-17T16:54:50.401-05:00\",\"updated_at\":\"2018-12-17T16:54:59.169-05:00\",\"width\":1280.0,\"height\":720.0,\"frame_rate\":29.671,\"duration\":6038,\"file_size\":2102027,\"audio_codec\":\"mp4a-40-2\",\"video_codec\":\"avc1\",\"audio_bitrate\":72000,\"video_bitrate\":2721866}],\"state\":\"completed\",\"errors\":[],\"created_at\":\"2018-12-17T16:54:50.401-05:00\",\"updated_at\":\"2018-12-17T16:54:59.169-05:00\",\"current_operations\":[],\"percent_complete\":100,\"global_id\":{\"uri\":\"gid://ActiveEncode/Encode/35efa965-ec51-409d-9495-2ae9669adbcc\"}}"
+  end
+
+  before do
+    encode_record
+  end
+
+  describe 'GET show' do
+    before do
+      get :show, params: { id: record_id }
+    end
+
+    context 'when record exists' do
+      let(:record_id) { 1 }
+
+      it "responds with a 200 status code" do
+        expect(response.status).to eq 200
+      end
+
+      it "responds with JSON" do
+        expect(response.content_type).to eq "application/json"
+      end
+
+      it "returns the encode record's raw json object" do
+        expect(response.body).to eq raw_object
+      end
+    end
+
+    context 'when record does not exist' do
+      let(:record_id) { "non-existant" }
+
+      it "responds with a 404 status code" do
+        expect(response.status).to eq 404
+      end
+
+      it "responds with JSON" do
+        expect(response.content_type).to eq "application/json"
+      end
+
+      it "returns the encode record's raw json object" do
+        expect(response.body).to eq "{\"message\":\"Couldn't find ActiveEncode::EncodeRecord with 'id'=#{record_id}\"}"
+      end
+    end
+  end
+end

data/spec/integration/elastic_transcoder_adapter_spec.rb

@@ -12,9 +12,11 @@ describe ActiveEncode::EngineAdapters::ElasticTranscoderAdapter do
   end
 
   let(:client) { Aws::ElasticTranscoder::Client.new(stub_responses: true) }
+  let(:s3client) { Aws::S3::Client.new(stub_responses: true) }
 
   before do
     allow(Aws::ElasticTranscoder::Client).to receive(:new).and_return(client)
+    allow(Aws::S3::Client).to receive(:new).and_return(s3client)
   end
 
   let(:created_job) do
@@ -25,14 +27,15 @@ describe ActiveEncode::EngineAdapters::ElasticTranscoderAdapter do
     client.stub_responses(:create_job, Aws::ElasticTranscoder::Types::ReadJobResponse.new(job: j))
 
     ActiveEncode::Base.create(
-     "somefile.mp4",
+     "spec/fixtures/fireworks.mp4",
      pipeline_id: "1471963629141-kmcocm",
-       output_key_prefix: "elastic-transcoder-samples/output/hls/",
-       outputs: [{
-         key: 'hls0400k/' + "e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a",
-         preset_id: "1351620000001-200050",
-         segment_duration: "2"
-       }])
+     masterfile_bucket: "BucketName",
+     output_key_prefix: "elastic-transcoder-samples/output/hls/",
+     outputs: [{
+       key: 'hls0400k/' + "e8fe80f5bsomefilesource_bucket7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a",
+       preset_id: "1351620000001-200050",
+       segment_duration: "2",
+     }])
   end
 
   let(:running_job) do
@@ -90,14 +93,14 @@ describe ActiveEncode::EngineAdapters::ElasticTranscoderAdapter do
     ActiveEncode::Base.find('failed-id')
   end
 
-  let(:completed_output) { [{ id: "2", url: "elastic-transcoder-samples/output/hls/hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a", label: "hls0400k", :width=>400, :height=>224, :frame_rate=>25, :file_size=>6901104, :duration=>117353 }] }
+  let(:completed_output) { [{ id: "2", url: "s3://BucketName/elastic-transcoder-samples/output/hls/hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a", label: "hls0400k", :width=>400, :height=>224, :frame_rate=>25, :file_size=>6901104, :duration=>117353 }] }
   let(:completed_tech_metadata) { { :width=>1280, :height=>720, :frame_rate=>25, :file_size=>21069678, :duration=>117312 } }
   let(:failed_tech_metadata) { {} }
 
   it_behaves_like "an ActiveEncode::EngineAdapter"
 
   describe "#create" do
-    let(:create_output) { [{ id: "2", url: "elastic-transcoder-samples/output/hls/hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a", label: "hls0400k" }] }
+    let(:create_output) { [{ id: "2", url: "s3://BucketName/elastic-transcoder-samples/output/hls/hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a", label: "hls0400k" }] }
 
     subject { created_job }
 
@@ -114,7 +117,7 @@ describe ActiveEncode::EngineAdapters::ElasticTranscoderAdapter do
 
   describe "#find" do
     context "a running encode" do
-      let(:running_output) { [{ id: "2", url: "elastic-transcoder-samples/output/hls/hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a", label: "hls0400k" }] }
+      let(:running_output) { [{ id: "2", url: "s3://BucketName/elastic-transcoder-samples/output/hls/hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a", label: "hls0400k" }] }
       let(:running_tech_metadata) { {:width=>1280, :height=>720, :frame_rate=>25, :file_size=>21069678, :duration=>117312} }
 
       subject { running_job }
@@ -133,4 +136,35 @@ describe ActiveEncode::EngineAdapters::ElasticTranscoderAdapter do
       end
     end
   end
+
+  describe "#check_s3_bucket" do
+    context "when file exists in masterfile_bucket" do
+      let(:input_url) { "s3://bucket1/file.mp4" }
+      let(:source_bucket) { "bucket1" }
+
+      it "just returns the key" do
+        # TODO: move these bucket helpers out to a service class so we don't have to test private methods
+        expect(described_class.new.send(:check_s3_bucket, input_url, source_bucket)).to eq "file.mp4"
+      end
+    end
+
+    context "when file is in another bucket" do
+      let(:input_url) { "s3://bucket1/file.mp4" }
+      let(:source_bucket) { "bucket2" }
+
+      it "copies to masterfile_bucket" do
+        # TODO: move these bucket helpers out to a service class so we don't have to test private methods
+        allow(SecureRandom).to receive(:uuid).and_return("randomstring")
+        expect(described_class.new.send(:check_s3_bucket, input_url, source_bucket)).to eq "randomstring/file.mp4"
+      end
+    end
+  end
+
+  describe "#output_percentage" do
+    let(:output) { double(ActiveEncode::Output, status: "Random status") }
+
+    it "returns 0 for any other status" do
+      expect(described_class.new.send(:output_percentage, output)).to eq 0
+    end
+  end
 end