abrizer 0.1.0

data/ansible/roles/rvm_io.rvm1-ruby/defaults/main.yml

@@ -0,0 +1,44 @@
+---
+
+# Install 1 or more versions of ruby
+# The last ruby listed will be set as the default ruby
+rvm1_rubies:
+  - 'ruby-2.2.2'
+
+# Delete a specific version of ruby (ie. ruby-2.1.0)
+rvm1_delete_ruby:
+
+# Install path for rvm (defaults to system wide)
+rvm1_install_path: '/usr/local/rvm'
+
+# Add or remove any install flags
+# NOTE: If you are doing a USER BASED INSTALL then
+#       make sure you ADD the --user-install flag below
+rvm1_install_flags: '--auto-dotfiles'
+
+# Add additional ruby install flags
+rvm1_ruby_install_flags:
+
+# Set the owner for the rvm directory
+rvm1_user: 'root'
+
+# URL for the latest installer script
+rvm1_rvm_latest_installer: 'https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer'
+
+# rvm version to use
+rvm1_rvm_version: 'stable'
+
+# Check and update rvm, disabling this will force rvm to never update
+rvm1_rvm_check_for_updates: True
+
+# GPG key verification, use an empty string if you want to skip this
+# Note: Unless you know what you're doing, just keep it as is
+#           Identity proof: https://keybase.io/mpapis
+#           PGP message: https://rvm.io/mpapis.asc
+rvm1_gpg_keys: 'D39DC0E3'
+
+# The GPG key server
+rvm1_gpg_key_server: 'hkp://keys.gnupg.net'
+
+# autolib mode, see https://rvm.io/rvm/autolibs
+rvm1_autolib_mode: 3

data/ansible/roles/rvm_io.rvm1-ruby/meta/main.yml

@@ -0,0 +1,21 @@
+---
+galaxy_info:
+  author: Nick Janetakis
+  description: The official rvm role to install and manage your ruby versions.
+  company:
+  license: license (MIT)
+  min_ansible_version: 1.5
+
+  platforms:
+  - name: EL
+    versions:
+    - all
+  - name: Ubuntu
+    versions:
+    - all
+
+  categories:
+  - development
+  - web
+
+dependencies: []

data/ansible/roles/rvm_io.rvm1-ruby/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+
+- include: 'rvm.yml'
+- include: 'rubies.yml'

data/ansible/roles/rvm_io.rvm1-ruby/tasks/rubies.yml

@@ -0,0 +1,78 @@
+---
+
+- name: Detect if rubies are installed
+  command: '{{ rvm1_rvm }} {{ item }} do true'
+  changed_when: False
+  failed_when: False
+  register: detect_rubies
+  with_items: '{{ rvm1_rubies }}'
+  when: rvm1_rubies
+
+- name: Install rubies
+  command: '{{ rvm1_rvm }} install {{ item.item }} {{ rvm1_ruby_install_flags }}'
+  when: rvm1_rubies and item.rc|default(0) != 0
+  with_items: '{{ detect_rubies.results }}'
+  become: yes
+  become_user: '{{ rvm1_user }}'
+
+- name: Detect default ruby version
+  command: '{{ rvm1_rvm }} alias list default'
+  changed_when: False
+  register: detect_default_ruby_version
+  become: yes
+  become_user: '{{ rvm1_user }}'
+
+- name: Select default ruby
+  command: '{{ rvm1_rvm }} alias create default {{ rvm1_default_ruby_version }}'
+  when: detect_default_ruby_version.stdout|default() == '' or
+        rvm1_default_ruby_version not in detect_default_ruby_version.stdout
+  become: yes
+  become_user: '{{ rvm1_user }}'
+
+- name: Detect installed ruby patch number
+  shell: >
+    {{ rvm1_rvm }} list strings | grep {{ item }} | tail -n 1
+  with_items: '{{ rvm1_rubies }}'
+  changed_when: False
+  register: ruby_patch
+  always_run: yes # Run even when in --check mode (http://docs.ansible.com/ansible/playbooks_checkmode.html)
+  become: yes
+  become_user: '{{ rvm1_user }}'
+
+- name: Install bundler if not installed
+  shell: >
+    ls {{ rvm1_install_path }}/wrappers/{{ item.stdout }}
+    | if ! grep "^bundler " ; then {{ rvm1_install_path }}/wrappers/{{ item.stdout }}/gem install bundler ; fi
+  args:
+    creates: '{{ rvm1_install_path }}/wrappers/{{ item.stdout }}/bundler'
+  with_items: '{{ ruby_patch.results }}'
+  register: bundler_install
+  changed_when: '"Successfully installed bundler" in bundler_install.stdout'
+  become: yes
+  become_user: '{{ rvm1_user }}'
+  
+- name: Symlink ruby related binaries on the system path
+  file:
+    state: 'link'
+    src: '{{ rvm1_install_path }}/wrappers/default/{{ item }}'
+    dest: '{{ rvm1_symlink_to }}/{{ item }}'
+    owner: 'root'
+    group: 'root'
+  when: not '--user-install' in rvm1_install_flags
+  with_items: '{{ rvm1_symlink_binaries }}'
+  
+- name: Detect if ruby version can be deleted
+  command: '{{ rvm1_rvm }} {{ rvm1_delete_ruby }} do true'
+  changed_when: False
+  failed_when: False
+  register: detect_delete_ruby
+  when: rvm1_delete_ruby
+  become: yes
+  become_user: '{{ rvm1_user }}'
+
+- name: Delete ruby version
+  command: '{{ rvm1_rvm }} remove {{ rvm1_delete_ruby }}'
+  changed_when: False
+  when: rvm1_delete_ruby and detect_delete_ruby.rc == 0
+  become: yes
+  become_user: '{{ rvm1_user }}'

data/ansible/roles/rvm_io.rvm1-ruby/tasks/rvm.yml

@@ -0,0 +1,55 @@
+---
+
+- name: Detect rvm binary
+  stat: path='{{ rvm1_rvm }}'
+  register: rvm_binary
+
+- name: Detect rvm installer
+  stat: path='{{ rvm1_temp_download_path }}/rvm-installer.sh'
+  register: rvm_installer
+
+- name: Detect current rvm version
+  command: '{{ rvm1_rvm}} version'
+  changed_when: False
+  register: rvm_current_version
+  when: rvm_binary.stat.exists
+
+- name: Install rvm installer
+  get_url:
+    url: '{{ rvm1_rvm_latest_installer }}'
+    dest: '{{ rvm1_temp_download_path }}/rvm-installer.sh'
+  when: not rvm_installer.stat.exists
+
+- name: Configure rvm installer
+  file:
+    path: '{{ rvm1_temp_download_path }}/rvm-installer.sh'
+    mode: 0755
+  when: not rvm_binary.stat.exists
+
+- name: Import GPG keys
+  command: 'gpg --keyserver {{ rvm1_gpg_key_server }} --recv-keys {{ rvm1_gpg_keys }}'
+  changed_when: False
+  when: rvm1_gpg_keys != ''
+  become: yes
+  become_user: '{{ rvm1_user }}'
+
+- name: Install rvm
+  command: >
+    {{ rvm1_temp_download_path }}/rvm-installer.sh {{ rvm1_rvm_version }}
+    --path {{ rvm1_install_path }} {{ rvm1_install_flags }}
+  when: not rvm_binary.stat.exists
+  become: yes
+  become_user: '{{ rvm1_user }}'
+
+- name: Update rvm
+  shell: '{{ rvm1_rvm }} get {{ rvm1_rvm_version }} && {{ rvm1_rvm }} reload'
+  changed_when: False
+  when: rvm_binary.stat.exists and rvm1_rvm_check_for_updates
+  become: yes
+  become_user: '{{ rvm1_user }}'
+
+- name: Configure rvm
+  command: '{{ rvm1_rvm }} autolibs {{ rvm1_autolib_mode }}'
+  when: not rvm_binary.stat.exists
+  become: yes
+  become_user: '{{ rvm1_user }}'

data/ansible/roles/rvm_io.rvm1-ruby/tests/inventory

@@ -0,0 +1 @@
+localhost

data/ansible/roles/rvm_io.rvm1-ruby/tests/test.yml

@@ -0,0 +1,14 @@
+---
+
+- hosts: localhost
+  remote_user: travis
+
+  vars:
+    rvm1_rubies:
+      - 'ruby-2.1.0'
+      - 'ruby-2.1.2'
+    rvm1_install_path: '/home/travis/.rvm'
+    rvm1_install_flags: '--auto-dotfiles --user-install'
+
+  roles:
+    - rvm1-ansible

data/ansible/roles/rvm_io.rvm1-ruby/vars/main.yml

@@ -0,0 +1,22 @@
+---
+
+rvm1_temp_download_path: '/tmp'
+
+rvm1_default_ruby_version: '{{ rvm1_rubies | last if rvm1_rubies and rvm1_rubies is iterable else "" }}'
+
+rvm1_rvm: '{{ rvm1_install_path }}/bin/rvm'
+
+rvm1_symlink_binaries:
+  - 'bundle'
+  - 'bundler'
+  - 'erb'
+  - 'executable-hooks-uninstaller'
+  - 'gem'
+  - 'irb'
+  - 'rake'
+  - 'rdoc'
+  - 'ri'
+  - 'ruby'
+  - 'testrb'
+
+rvm1_symlink_to: '/usr/local/bin'

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "abrizer"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/abrizer

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'abrizer'
+require 'abrizer/cli'
+require 'byebug'
+Abrizer::CLI.start(ARGV)

data/lib/abrizer/adaptation.rb

@@ -0,0 +1,53 @@
+module Abrizer
+  class Adaptation
+
+    include FilepathHelpers
+
+    attr_reader :width, :height, :bitrate
+
+    def initialize(width:, height:, bitrate:)
+      @width = width
+      @height = height
+      @bitrate = bitrate
+    end
+
+    def ffmpeg_cmd(input, output_directory, pass)
+      cmd = %Q|ffmpeg -y -i #{input} -vf \
+          scale='#{width}:trunc(#{width}/dar/2)*2',setsar=1 \
+          -an -c:v libx264 -x264opts 'keyint=48:min-keyint=48:no-scenecut' \
+          -b:v #{bitrate}k -preset faster |
+      if pass == 2
+        cmd += %Q| -maxrate #{constrained_bitrate}k -bufsize #{bitrate}k -pass 2 #{filepath(input, output_directory)} |
+      else
+        cmd += " -pass 1 -f mp4 /dev/null "
+      end
+      cmd
+    end
+
+    # TODO: make the constrained bitrate (maxrate) value configurable
+    def constrained_bitrate
+      @bitrate * 1.1
+    end
+
+    def outfile_basename(input)
+      extname = File.extname input
+      basename = File.basename input, extname
+      "#{basename}-#{width}x#{height}-#{bitrate}"
+    end
+
+    def filepath(input, output_directory)
+      name = "#{outfile_basename(input)}.mp4"
+      File.join output_directory, name
+    end
+
+    def filepath_fragmented(input, output_directory)
+      name = "#{outfile_basename(input)}-frag.mp4"
+      File.join output_directory, name
+    end
+
+    def to_s
+      "Width: #{@width}, Height: #{@height}, Bitrate: #{@bitrate}"
+    end
+
+  end
+end

data/lib/abrizer/adaptation_finder.rb

@@ -0,0 +1,59 @@
+module Abrizer
+  # TODO: AdaptationFinder is incomplete. Basically what we want to do is to
+  # find out the height, width, and aspect ratio of the original and then
+  # determine which adaptations to create. So for a 3:2 video we'll have 4
+  # adaptations but ought to only create ones that are the same size and smaller
+  # than the input file. Adaptations that are larger should not be created.
+  # So first we find the aspect ratio to see the value adaptations we could
+  # apply and then we select the ones that are the same width and smaller.
+  # If there is not an exact match for the
+  # All of the aspect ratios here are given based on real files that have come
+  # through our workflow, so there might be some missing.
+  class AdaptationFinder
+    attr_reader :adaptations, :info
+    def initialize(filename)
+      @filename = filename
+      @informer = Abrizer::FfprobeInformer.new(filename)
+      find_adaptations
+    end
+
+    # TODO: analyze the incoming file and determine which preset to use
+    def find_adaptations
+      raw_adaptations = case @informer.display_aspect_ratio
+      when "4:3"
+        ar_4_3_adaptations
+      when "16:9"
+        ar_16_9_adaptations
+      else
+        puts "Unable to find appropriate adaptation set!"
+        exit
+      end
+      adaptations = raw_adaptations.map{|adaptation| Abrizer::Adaptation.new(adaptation)}
+      @adaptations = adaptations.select{|adaptation| adaptation.width <= @informer.width}
+    end
+
+    # The bitrates here are based on H.264 encoding. 
+    def ar_4_3_adaptations
+      [
+        {width: 224, height: 168, bitrate: 200},
+        {width: 448, height: 336, bitrate: 400},
+        {width: 640, height: 480, bitrate: 800},
+        {width: 720, height: 540, bitrate: 1000},
+      ]
+    end
+
+    # The bitrates here are based on H.264 encoding.
+    def ar_16_9_adaptations
+      # Average video bitrate from here: https://bitmovin.com/video-bitrate-streaming-hls-dash/
+      [
+        {width:  256, height: 144, bitrate: 200},
+        {width:  512, height: 288, bitrate: 400},
+        {width:  768, height: 432, bitrate: 800},
+        {width: 1024, height: 576, bitrate: 1200},
+        {width: 1280, height: 720, bitrate: 2400},
+        {width: 1920, height: 1080, bitrate: 4800},
+      ]
+    end
+
+  end
+end

data/lib/abrizer/all.rb

@@ -0,0 +1,20 @@
+module Abrizer
+  class All
+
+    def initialize(filename, output_dir=nil)
+      @filename = filename
+      @output_directory = output_dir
+    end
+
+    def run
+      Abrizer::Processor.process(@filename, @output_directory)
+      Abrizer::ProgressiveMp4.new(@filename, @output_directory).create
+      Abrizer::ProgressiveVp9.new(@filename, @output_directory).create
+      Abrizer::PackageDashBento.new(@filename, @output_directory).package
+      Abrizer::PackageHlsBento.new(@filename, @output_directory).package
+      Abrizer::Sprites.new(@filename, @output_directory).create
+      Abrizer::Cleaner.new(@filename, @output_directory).clean
+    end
+
+  end
+end

data/lib/abrizer/cleaner.rb

@@ -0,0 +1,37 @@
+module Abrizer
+  class Cleaner
+
+    include FilepathHelpers
+
+    def initialize(filename, output_dir=nil)
+      @filename = filename
+      @output_directory = output_dir
+      @adaptations = Abrizer::AdaptationFinder.new(@filename).adaptations
+    end
+
+    def clean
+      delete_adaptations(@adaptations)
+      clean_audio_file
+      remove_pass1_log_files
+    end
+
+    def delete_adaptations(adapts)
+      adapts.map do |adaptation|
+        filepath = adaptation.filepath_fragmented(@filename, output_directory)
+        FileUtils.rm filepath if File.exist? filepath
+      end
+    end
+
+    def clean_audio_file
+      FileUtils.rm audio_filepath_fragmented if File.exist? audio_filepath_fragmented
+    end
+
+    def remove_pass1_log_files
+      glob = File.join output_directory, "ffmpeg2pass*"
+      Dir.glob(glob).each do |log_filepath|
+        FileUtils.rm log_filepath
+      end
+    end
+
+  end
+end

data/lib/abrizer/cli.rb

@@ -0,0 +1,84 @@
+require 'thor'
+module Abrizer
+  class CLI < Thor
+
+    desc 'all <filepath> <output_directory>', 'Runn all processes including creating ABR streams, progressive download versions, and images and video sprites'
+    def all(filepath, output_dir=nil)
+      filepath = File.expand_path filepath
+      output_dir = File.expand_path output_dir
+      Abrizer::All.new(filepath, output_dir).run
+    end
+
+    desc 'abr <filepath> <output_directory>', 'From file create ABR streams, includes processing MP4 adaptations for packaging'
+    def abr(filepath, output_dir=nil)
+      filepath = File.expand_path filepath
+      output_dir = File.expand_path output_dir
+      Abrizer::Processor.process(filepath, output_dir)
+      Abrizer::PackageDashBento.new(filepath, output_dir).package
+      Abrizer::PackageHlsBento.new(filepath, output_dir).package
+    end
+
+    desc 'process <filepath> <output_directory>', 'From mezzanine or preservation file create intermediary adaptations'
+    def process(filepath, output_dir=nil)
+      filepath = File.expand_path filepath
+      output_dir = File.expand_path output_dir
+      Abrizer::Processor.process(filepath, output_dir)
+    end
+
+    desc 'mp4 <filepath> <output_directory>', 'Create a single progressive download version as an MP4 from the next to largest adaptation and audio. The adaptation and audio file must already exist.'
+    def mp4(filepath, output_dir=nil)
+      filepath = File.expand_path filepath
+      output_dir = File.expand_path output_dir
+      Abrizer::ProgressiveMp4.new(filepath, output_dir).create
+    end
+
+    desc 'vp9 <filepath> <output_directory>', 'Create a single VP9 progressive download version from the original video.'
+    def vp9(filepath, output_dir=nil)
+      filepath = File.expand_path filepath
+      output_dir = File.expand_path output_dir
+      Abrizer::ProgressiveVp9.new(filepath, output_dir).create
+    end
+
+    desc 'adaptations <filepath>', 'Display which adaptations will be created from input file'
+    def adaptations(filepath)
+      adaptations = Abrizer::AdaptationFinder.new(filepath).adaptations
+      puts adaptations
+    end
+
+    desc 'inform <filepath>', 'Display information about the video/audio file'
+    def inform(filepath)
+      informer = FfprobeInformer.new(filepath)
+      puts informer.json_result
+      puts informer
+    end
+
+    desc 'package <dash_or_hls> <filepath> <output_directory>', "Package dash or hls from adaptations"
+    def package(dash_or_hls, filepath, output_dir=nil)
+      filepath = File.expand_path filepath
+      output_dir = File.expand_path output_dir
+      case dash_or_hls
+      when "dash"
+        Abrizer::PackageDashBento.new(filepath, output_dir).package
+      when "hls"
+        Abrizer::PackageHlsBento.new(filepath, output_dir).package
+      when "all"
+        Abrizer::PackageDashBento.new(filepath, output_dir).package
+        Abrizer::PackageHlsBento.new(filepath, output_dir).package
+      else
+        puts "Not a valid packaging value. Try dash or hls."
+      end
+    end
+
+    desc 'sprites <filepath> <output_directory>', 'Create image sprites and metadata WebVTT file'
+    def sprites(filepath, output_dir=nil)
+      filepath = File.expand_path filepath
+      output_dir = File.expand_path output_dir
+      Abrizer::Sprites.new(filepath, output_dir).create
+    end
+
+    desc 'clean <filepath> <output_directory>', 'Clean up intermediary files'
+    def clean(filepath, output_dir=nil)
+      Abrizer::Cleaner.new(filepath, output_dir).clean
+    end
+  end
+end

data/lib/abrizer/ffmpeg_processor.rb

@@ -0,0 +1,59 @@
+module Abrizer
+  class FfmpegProcessor
+
+    include FilepathHelpers
+
+    def initialize(filename, output_dir=nil)
+      @filename = filename
+      @output_directory = output_dir
+      @adaptation_finder = Abrizer::AdaptationFinder.new(@filename)
+    end
+
+    def process
+      make_directory
+      Dir.chdir output_directory
+      process_first_pass
+      process_second_passes
+      process_audio
+    end
+
+    def make_directory
+      FileUtils.mkdir output_directory unless File.exist? output_directory
+    end
+
+    def first_pass_adaptation
+      adaptations = Abrizer::AdaptationFinder.new(@filename).adaptations
+      sorted = adaptations.sort_by do |adaptation|
+       adaptation.width
+      end
+      sorted[-2]
+    end
+
+    def first_pass_cmd
+      first_pass_adaptation.ffmpeg_cmd(@filename, output_directory, 1)
+    end
+
+    def process_first_pass
+      puts first_pass_cmd
+      `#{first_pass_cmd}`
+    end
+
+    # Creates a file per adaptation based on aspect ratio and resolution
+    def process_second_passes
+      @adaptation_finder.adaptations.each do |adaptation|
+        cmd = adaptation.ffmpeg_cmd(@filename, output_directory, 2)
+        puts cmd
+        `#{cmd}`
+        `mp4fragment #{adaptation.filepath(@filename, output_directory)} #{adaptation.filepath_fragmented(@filename, output_directory)}`
+        FileUtils.rm adaptation.filepath(@filename, output_directory)
+      end
+    end
+
+    def process_audio
+      `ffmpeg -y -i #{@filename} -c:a libfdk_aac -b:a 128k -vn #{audio_filepath}`
+      `mp4fragment #{audio_filepath} #{audio_filepath_fragmented}`
+      FileUtils.rm audio_filepath
+    end
+
+  end
+end

data/lib/abrizer/ffprobe_informer.rb

@@ -0,0 +1,69 @@
+module Abrizer
+  class FfprobeInformer
+    attr_reader :json_result, :info
+    def initialize(filename)
+      @filename = filename
+      get_info
+    end
+
+    def get_info
+      @json_result = `#{ffmpeg_info_cmd}`
+      @info = JSON.parse @json_result
+    end
+
+    def width
+      video_stream['width'] if video_stream
+    end
+
+    def height
+      video_stream['height'] if video_stream
+    end
+
+    def display_aspect_ratio #dar
+      dar = video_stream['display_aspect_ratio']
+      sar = video_stream['sample_aspect_ratio']
+      if dar == "0:1" #&& sar == "0:1"
+        calculate_aspect_ratio_from_wh
+      else
+        dar
+      end
+    end
+
+    def calculate_aspect_ratio_from_wh
+      new_width = width
+      new_height = height
+      w = width
+      h = height
+      while h != 0
+        rem = w % h
+        w = h
+        h = rem
+      end
+      new_height = new_height / w
+      new_width = new_width / w
+      "#{new_width}:#{new_height}"
+    end
+
+    def video_stream
+      @info['streams'].find do |stream|
+        stream['codec_type'] == 'video'
+      end
+    end
+
+    def audio_stream
+      @info['streams'].find do |stream|
+        stream['codec_type'] == 'audio'
+      end
+    end
+
+    def ffmpeg_info_cmd
+      "ffprobe -v error -print_format json -show_format -show_streams #{@filename}"
+    end
+
+    def to_s
+      ffmpeg_info_cmd + "\n" +
+      "#{width}x#{height} DAR:#{display_aspect_ratio}"
+    end
+
+  end
+end