shingoncoder 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7fb618ddf8d3d2e620724cafc6aa1b70261663ee
+  data.tar.gz: ca0a71f31dd49e557c03f0966de9259370da521a
+SHA512:
+  metadata.gz: bf2ce885ed263227ccf3cac248adf7cc7a90046696e824125add9b9c72ec2d2093cbc6f09f8cf8b678efed835442e774255a0c14ef24720f5c2bae981974b988
+  data.tar.gz: c9238d764fe40159ee4557cd55889d1a423d99419061220df8c35e0b0d55e65cf8911b083a37b25e896fef893990dbf971ca665d51cb6a5a8efe03e0a2240afd

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.3
+before_install: gem install bundler -v 1.10.6

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shingon.gemspec
+gemspec
+
+gem 'sqlite3'
+gem 'byebug'

data/LICENSE.txt

@@ -0,0 +1,15 @@
+##########################################################################
+# Copyright 2013 Notre Dame, Data Curation Experts, WGBH
+# Additional copyright may be held by others, as reflected in the commit log
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

data/README.md

@@ -0,0 +1,54 @@
+# Shingoncoder
+
+A asynchronous system for handling video transcoding backed by FFMpeg.
+
+## Installation
+
+Include this gem in your Gemfile
+
+```ruby
+gem 'shingoncoder'
+```
+
+Create the necessary tables:
+
+```ruby
+rails console
+Shingoncoder::Backend::JobRegistry.create_tables!
+```
+
+The asynchronous behavior is handled by ActiveJob, so you need only start your workers in the usual manner.
+
+
+## Configuration
+
+TODO
+
+```ruby
+Shingoncoder::Backend::TranscodeService.config.ffmpeg_path = '/usr/bin/ffmpeg'
+```
+
+
+## Usage
+
+```ruby
+Shingoncoder::Job.create(input: 'file:///opt/originals/full.mov')
+Shingoncoder::Job.create(input: 'file:///opt/originals/full.mov',
+                         outputs: [{ label: 'vp8 for the web',
+                                     url: 'file:///opt/derivatives/output.webm'}])
+```
+
+This returns a Shingoncoder::Response object which contains a Job ID and one or more Output IDs (one per output file).
+
+### Output options
+```
+url - where to put the file, (Must be a file URI for now)
+size - The resolution of the output video (WxH, in pixels). Defaults to 320x240.
+label - optional label
+```
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/jcoyne/shingoncoder. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "shingon"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/shingoncoder.rb

@@ -0,0 +1,10 @@
+require "shingoncoder/version"
+require 'active_support'
+require 'active_record'
+
+module Shingoncoder
+  extend ActiveSupport::Autoload
+  autoload :Job
+  autoload :Response
+  autoload :Backend
+end

data/lib/shingoncoder/backend.rb

@@ -0,0 +1,10 @@
+module Shingoncoder
+  module Backend
+    extend ActiveSupport::Autoload
+    autoload :Job
+    autoload :JobRegistry
+    autoload :TranscodeJob
+    autoload :TranscodeService
+    autoload :Config
+  end
+end

data/lib/shingoncoder/backend/config.rb

@@ -0,0 +1,72 @@
+module Shingoncoder::Backend
+  class Config
+    attr_writer :ffmpeg_path, :video_bitrate, :video_attributes, :size_attributes, :audio_attributes
+
+    def ffmpeg_path
+      @ffmpeg_path ||= 'ffmpeg'
+    end
+
+    def video_bitrate
+      @video_bitrate ||= default_video_bitrate
+    end
+
+    def video_attributes
+      @video_attributes ||= default_video_attributes
+    end
+
+    def size_attributes
+      @size_attributes ||= default_size_attributes
+    end
+
+    def audio_attributes
+      @audio_attributes ||= default_audio_attributes
+    end
+
+    def mpeg4
+      @mpeg4 ||= CodecConfig.new('-vcodec libx264 -acodec libfdk_aac')
+    end
+
+    def webm
+      @webm ||= CodecConfig.new('-vcodec libvpx -acodec libvorbis')
+    end
+
+    def mkv
+      @mkv ||= CodecConfig.new('-vcodec ffv1')
+    end
+
+    def jpeg
+      @jpeg ||= CodecConfig.new('-vcodec mjpeg')
+    end
+
+    class CodecConfig
+      attr_writer :codec
+
+      def initialize(default)
+        @codec = default
+      end
+
+      def codec
+        @codec
+      end
+    end
+
+
+    protected
+      def default_video_bitrate
+        '345k'
+      end
+
+      def default_video_attributes
+        "-g 30 -b:v #{video_bitrate}"
+      end
+
+      def default_size_attributes
+        "320x240"
+      end
+
+      def default_audio_attributes
+        "-ac 2 -ab 96k -ar 44100"
+      end
+  end
+end
+

data/lib/shingoncoder/backend/job.rb

@@ -0,0 +1,27 @@
+module Shingoncoder::Backend
+  class Job
+    class << self
+      # Register the job and kicks off several derivatives.
+      # @param [Hash] options
+      # @option [String] :input the url to the file
+      # @return [Integer] :id for the job
+      def create(options)
+        uri = URI(options.fetch(:input))
+        filename = case uri.scheme
+        when 'file'
+          uri.path
+        else
+          raise ArgumentError, "#{uri.scheme} is not yet implemented"
+        end
+
+        job = Shingoncoder::Backend::JobRegistry.create(options)
+
+        job.outputs.each do |output|
+          TranscodeJob.perform_later(output.id)
+        end
+
+        job.id
+      end
+    end
+  end
+end

data/lib/shingoncoder/backend/job_registry.rb

@@ -0,0 +1,96 @@
+module Shingoncoder
+  module Backend
+    class JobRegistry
+
+      class << self
+        def create(input)
+          raise ArgumentError, "Required argument ':input' is missing" unless input.key?(:input)
+          outputs = input.delete(:outputs)
+          outputs ||= default_output(input)
+          Job.create(Job.input_column_name => input) do |job|
+            outputs.each do |val|
+              raise ArgumentError, "Output directive must have a url" unless val.key?(:url)
+
+              job.outputs.build(Output.input_column_name => val)
+            end
+          end
+        end
+
+        def create_tables!
+          Job.create_table!
+          Output.create_table!
+        end
+
+        def drop_tables!
+          Job.drop_table!
+          Output.drop_table!
+        end
+
+        private
+
+          def default_output(input)
+            [{ url: "#{input.fetch(:input)}.mp4" }]
+          end
+      end
+
+      class Job < ActiveRecord::Base
+        ##
+        # :singleton-method:
+        # Customizable data column name. Defaults to 'data'.
+        cattr_accessor :input_column_name
+        self.input_column_name = 'input'
+        has_many :outputs
+
+        serialize input_column_name, JSON
+
+        class << self
+          def drop_table!
+            connection.schema_cache.clear_table_cache!(table_name)
+            connection.drop_table table_name
+          end
+
+          def create_table!
+            connection.schema_cache.clear_table_cache!(table_name)
+            connection.create_table(table_name) do |t|
+              t.text input_column_name
+            end
+          end
+        end
+      end
+
+      class Output < ActiveRecord::Base
+        belongs_to :job
+
+        ##
+        # :singleton-method:
+        # Customizable data column name. Defaults to 'data'.
+        cattr_accessor :input_column_name
+        self.input_column_name = 'input'
+        serialize input_column_name, JSON
+
+        def url
+          self[input_column_name]['url']
+        end
+
+        def config
+          self[input_column_name]
+        end
+
+        class << self
+          def drop_table!
+            connection.schema_cache.clear_table_cache!(table_name)
+            connection.drop_table table_name
+          end
+
+          def create_table!
+            connection.schema_cache.clear_table_cache!(table_name)
+            connection.create_table(table_name) do |t|
+              t.references :job, index: true, foreign_key: true
+              t.text input_column_name
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shingoncoder/backend/transcode_job.rb

@@ -0,0 +1,11 @@
+require 'active_job'
+module Shingoncoder::Backend
+  class TranscodeJob < ActiveJob::Base
+    queue_as :shingoncoder
+
+    # @param [Fixnum] id the identifier of the job registry record
+    def perform(id)
+      TranscodeService.new(id).transcode
+    end
+  end
+end

data/lib/shingoncoder/backend/transcode_service.rb

@@ -0,0 +1,133 @@
+require 'open3'
+module Shingoncoder::Backend
+  class TranscodeService
+    include Open3
+    class_attribute :config, :timeout
+    self.config = Config.new
+
+    BLOCK_SIZE = 1024
+
+    # @param [Fixnum] id the Output ID (not to be confused with the job_id)
+    def initialize(id)
+      @id = id
+    end
+
+    def transcode
+      execute "#{config.ffmpeg_path} #{input_options} -i \"#{input_path}\" #{output_options} #{output_path}"
+    end
+
+    private
+
+    def output_path
+      URI(output.url).path
+    end
+
+    def input_path
+      URI(output.job.input.fetch('input')).path
+    end
+
+    def output
+      @output ||= JobRegistry::Output.find(@id)
+    end
+
+    def output_format
+      @format ||= File.extname(output.url).sub(/\./, '')
+    end
+
+    def size_attribute
+      output.config.fetch('size', config.size_attributes)
+    end
+
+    def codecs(format)
+      case format
+      when 'mp4'
+        config.mpeg4.codec
+      when 'webm'
+        config.webm.codec
+      when "mkv"
+        config.mkv.codec
+      when "jpg"
+        config.jpeg.codec
+      else
+        raise ArgumentError, "Unknown format `#{format}'"
+      end
+    end
+
+    def input_options
+      output_format == 'jpg' ? " -itsoffset -2" : ''
+    end
+
+    def output_options
+      output_options = "-s #{size_attribute} #{codecs(output_format)}"
+
+      output_options+= case output_format
+      when "jpg"
+        " -vframes 1 -an -f rawvideo"
+      else
+        " #{config.video_attributes} #{config.audio_attributes}"
+      end
+    end
+
+    def execute(command)
+      context = {}
+      if timeout
+        execute_with_timeout(timeout, command, context)
+      else
+        execute_without_timeout(command, context)
+      end
+    end
+
+    def execute_with_timeout(timeout, command, context)
+      begin
+        status = Timeout::timeout(timeout) do
+          execute_without_timeout(command, context)
+        end
+      rescue Timeout::Error => ex
+        pid = context[:pid]
+        Process.kill("KILL", pid)
+        raise TimeoutError, "Unable to execute command \"#{command}\"\nThe command took longer than #{timeout} seconds to execute"
+      end
+
+    end
+
+    def execute_without_timeout(command, context)
+      exit_status = nil
+      err_str = ''
+      stdin, stdout, stderr, wait_thr = popen3(command)
+      context[:pid] = wait_thr[:pid]
+      stdin.close
+      stdout.close
+      files = [stderr]
+
+      until all_eof?(files) do
+        ready = IO.select(files, nil, nil, 60)
+
+        if ready
+          readable = ready[0]
+          readable.each do |f|
+            fileno = f.fileno
+
+            begin
+              data = f.read_nonblock(BLOCK_SIZE)
+
+              case fileno
+                when stderr.fileno
+                  err_str << data
+              end
+            rescue EOFError
+              Rails.logger "Caught an eof error in ShellBasedProcessor"
+              # No big deal.
+            end
+          end
+        end
+        end
+        exit_status = wait_thr.value
+
+        raise "Unable to execute command \"#{command}\". Exit code: #{exit_status}\nError message: #{err_str}" unless exit_status.success?
+      end
+
+      def all_eof?(files)
+        files.find { |f| !f.eof }.nil?
+      end
+  end
+end

data/lib/shingoncoder/job.rb

@@ -0,0 +1,25 @@
+module Shingoncoder
+  class Job
+    class << self
+      # @param [Hash] options
+      # @option [String] :input the url to the file
+      def create(options = {})
+        id = Backend::Job.create(options)
+        # TODO in the future the backend could be via a HTTP API
+        Response.new('id' => id, 'code' => 201)
+      end
+
+      def cancel
+        fail NotImplementedError
+      end
+
+      def progress
+        fail NotImplementedError
+      end
+
+      def details
+        fail NotImplementedError
+      end
+    end
+  end
+end

data/lib/shingoncoder/response.rb

@@ -0,0 +1,11 @@
+module Shingoncoder
+  class Response
+    attr_reader :code, :body
+
+    def initialize(properties)
+      @id = properties.fetch('id')
+      @code = properties.fetch('code')
+      @body = { 'id' => @id }
+    end
+  end
+end

data/lib/shingoncoder/version.rb

@@ -0,0 +1,3 @@
+module Shingoncoder
+  VERSION = "0.0.1"
+end

data/shingoncoder.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'shingoncoder/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "shingoncoder"
+  spec.version       = Shingoncoder::VERSION
+  spec.authors       = ["Justin Coyne"]
+  spec.email         = ["jcoyne@justincoyne.com"]
+
+  spec.summary       = %q{Create and manage asynchronous transcoding jobs}
+  spec.description   = %q{Backed by ActiveJob and FFMpeg}
+  spec.homepage      = "https://github.com/jcoyne/shingoncoder"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'activerecord'
+  spec.add_dependency 'railties'
+  spec.add_dependency 'activejob'
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: shingoncoder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Justin Coyne
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-09-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Backed by ActiveJob and FFMpeg
+email:
+- jcoyne@justincoyne.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/shingoncoder.rb
+- lib/shingoncoder/backend.rb
+- lib/shingoncoder/backend/config.rb
+- lib/shingoncoder/backend/job.rb
+- lib/shingoncoder/backend/job_registry.rb
+- lib/shingoncoder/backend/transcode_job.rb
+- lib/shingoncoder/backend/transcode_service.rb
+- lib/shingoncoder/job.rb
+- lib/shingoncoder/response.rb
+- lib/shingoncoder/version.rb
+- shingoncoder.gemspec
+homepage: https://github.com/jcoyne/shingoncoder
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Create and manage asynchronous transcoding jobs
+test_files: []
+has_rdoc: