arthropod_hls_video_encoder 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8311820e563fed91a93536b6ef5defa75b560ab8d68d02bbcfc9ee32d851ea6c
+  data.tar.gz: 8fe7ad465fb6f41cba32b3f58d673e47b6703978b2c6f9e979749a4dcd41041d
+SHA512:
+  metadata.gz: cd0f75b5521288300a26fbebcf17b2051381abe7629e601ac3ce3b670f559dcd89b3852ee069137d00fbb38975de1f66136c0074ffb08b6574ae23d678447a08
+  data.tar.gz: 104d79c41ca986d505a2bb911ea8aa3bcc74302b4200e40cf1b2fa028e3c4e1a2dd456ca69bec8dce7ae766b754bd345a318df474711f678401cb7351def542c

data/.gitignore

@@ -0,0 +1,2 @@
+ffmpeg*
+.byebug_history

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    arthropod_hls_video_encoder (0.0.1)
+      arthropod (= 0.0.2)
+      aws-sdk-sqs
+      fog-aws
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    arthropod (0.0.2)
+      aws-sdk-sqs
+    aws-eventstream (1.0.3)
+    aws-partitions (1.251.0)
+    aws-sdk-core (3.84.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-sqs (1.23.1)
+      aws-sdk-core (~> 3, >= 3.71.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.1.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    builder (3.2.3)
+    byebug (11.0.1)
+    excon (0.70.0)
+    fog-aws (3.5.2)
+      fog-core (~> 2.1)
+      fog-json (~> 1.1)
+      fog-xml (~> 0.1)
+      ipaddress (~> 0.8)
+    fog-core (2.1.2)
+      builder
+      excon (~> 0.58)
+      formatador (~> 0.2)
+      mime-types
+    fog-json (1.2.0)
+      fog-core
+      multi_json (~> 1.10)
+    fog-xml (0.1.3)
+      fog-core
+      nokogiri (>= 1.5.11, < 2.0.0)
+    formatador (0.2.5)
+    ipaddress (0.8.3)
+    jmespath (1.4.0)
+    mime-types (3.3)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2019.1009)
+    mini_portile2 (2.4.0)
+    multi_json (1.14.1)
+    nokogiri (1.10.7)
+      mini_portile2 (~> 2.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  arthropod_hls_video_encoder!
+  byebug
+
+BUNDLED WITH
+   2.0.2

data/README.md

@@ -0,0 +1,78 @@
+# Arthropod-hls-video-encoder
+
+## Installation
+
+```
+gem install arthropod_hls_video_encoder
+```
+
+## Usage
+
+Just run it with the required arguments.
+```shell
+$ arthropod_hls_video_encoder -h
+
+Usage: arthropod_hls_video_encoder [options]
+    -q, --queue [string]             SQS queue name
+    -i, --access-key-id [string]     AWS access key ID, default to the AWS_ACCESS_KEY_ID environment variable
+    -k, --secret-access-key [string] AWS secret access key, default to the AWS_SECRET_ACCESS_KEY environment variable
+    -r, --region [string]            AWS region, default to the AWS_REGION environment variable
+```
+
+Example of client side call:
+```ruby
+result = Arthropod::Client.push(queue_name: "hls_video_encoder", body: {
+  video_url: "https://s3-#{ENV['S3_REGION']}.amazonaws.com/#{ENV['S3_BUCKET']}/#{medium.temporary_key}",
+  root_dir: Digest::SHA1.hexdigest("#{ENV["SECURE_UPLOADER_KEY"]}#{medium.uuid}").insert(3, '/'),
+  aws_access_key_id: ENV['S3_ACCESS_KEY_ID'],
+  aws_secret_access_key: ENV['S3_SECRET_ACCESS_KEY'],
+  region: ENV['S3_REGION'],
+  endpoint: ENV['S3_ENDPOINT'],
+  host: ENV['S3_HOST'],
+  bucket: ENV['S3_BUCKET'],
+  profiles: [
+    {
+      codec: "libx264",
+      bandwidth: 1500000,
+      resolution: 720,
+      name: 'high',
+    },
+    {
+      codec: "libx264",
+      bandwidth: 800000,
+      resolution: 720,
+      name: 'low'
+    }
+  ]
+})
+```
+
+* `video_url`: the URL of the video you want to transcode to HLS
+* `root`: the destination directory in your bucket
+* `aws_access_key_id`: an AWS access key to access your bucket
+* `aws_secret_access_key`:  an AWS secret access key to access your bucket
+* `region`: the region of your bucket
+* `endpoint`: the endpoint of your S3 instance if you have one (useful for Minio)
+* `host`: the host of your S3 instance if you have one (useful for Minio)
+* `bucket`: your bucket name
+* `profiles`: the HLS profile you want to generate
+
+The result object is a follow.
+
+```ruby
+{
+  key: "[string]",
+  thumbnail_key: "[string]",
+  small_thumbnail_key: "[string]",
+  preview_key: "[string]",
+  duration: "[string]"
+}
+```
+
+* `key`: the key the root HLS file in your bucket
+* `thumbnail_key`: the key of the auto-generated thumbnail (the thumbnail is taken at half the video time)
+* `small_thumbnail_key`: same thing, but with a smaller thumbnail
+* `preview_key`: a GIF preview of your video
+* `duration`: the duration of the video
+
+*Both the input and output are very opiniated and follow my needs*

data/arthropod_hls_video_encoder.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "arthropod_hls_video_encoder/version"
+
+Gem::Specification.new do |gem|
+  gem.name          = "arthropod_hls_video_encoder"
+  gem.version       = ArthropodHlsVideoEncoder::VERSION
+  gem.authors       = ["Victor Goya"]
+  gem.email         = ["goya.victor@gmail.com"]
+  gem.description   = "HLS video encoder using Arthropod"
+  gem.summary       = "HLS video encoder using Arthropod"
+
+  gem.files         = `git ls-files -z`.split("\x0")
+  gem.executables   = %w(arthropod_hls_video_encoder)
+  gem.require_paths = ["lib"]
+  gem.bindir        = 'bin'
+
+  gem.licenses      = ["MIT"]
+
+  gem.required_ruby_version = "~> 2.0"
+
+  gem.add_dependency 'arthropod', '= 0.0.2'
+  gem.add_dependency 'aws-sdk-sqs'
+  gem.add_dependency 'fog-aws'
+
+  gem.add_development_dependency "byebug"
+end

data/bin/arthropod_hls_video_encoder

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'aws-sdk-sqs'
+require 'arthropod'
+require 'arthropod_hls_video_encoder/encoder'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on("-q", "--queue [string]", "SQS queue name") do |q|
+    options[:queue] = q
+  end
+  opts.on("-i", "--access-key-id [string]", "AWS access key ID, default to the AWS_ACCESS_KEY_ID environment variable") do |i|
+    options[:access_key_id] = i
+  end
+  opts.on("-k", "--secret-access-key [string]", "AWS secret access key, default to the AWS_SECRET_ACCESS_KEY environment variable") do |k|
+    options[:secret_access_key] = k
+  end
+  opts.on("-r", "--region [string]", "AWS region, default to the AWS_REGION environment variable") do |r|
+    options[:region] = r
+  end
+end.parse!
+
+client = Aws::SQS::Client.new({
+  access_key_id: options[:access_key_id] || ENV["AWS_ACCESS_KEY_ID"],
+  secret_access_key: options[:secret_access_key] || ENV["AWS_SECRET_ACCESS_KEY"],
+  region: options[:region] || ENV["AWS_REGION"],
+})
+
+Arthropod::Server.pull(client: client, queue_name: options[:queue]) do |request|
+  ArthropodHlsVideoEncoder::Encoder.new({
+    video_url: request.body["video_url"],
+    root_dir: request.body["root_dir"],
+    aws_access_key_id: request.body["aws_access_key_id"],
+    aws_secret_access_key: request.body["aws_secret_access_key"],
+    region: request.body["region"],
+    endpoint: request.body["endpoint"],
+    host: request.body["host"],
+    bucket: request.body["bucket"],
+    profiles: request.body["profiles"]
+  }).perform!
+end

data/lib/arthropod_hls_video_encoder/encoder.rb

@@ -0,0 +1,158 @@
+require 'shellwords'
+require 'json'
+require 'securerandom'
+require 'fog/aws'
+
+module ArthropodHlsVideoEncoder
+  class Encoder
+    attr_reader :video_url, :aws_access_key_id, :aws_secret_access_key, :region, :endpoint, :host, :bucket, :profiles, :job_id, :root_dir
+
+    def initialize(video_url:, root_dir:, aws_access_key_id:, aws_secret_access_key:, region:, endpoint:, host:, bucket:, profiles:)
+      @video_url = video_url
+      @root_dir = root_dir
+      @aws_access_key_id = aws_access_key_id
+      @aws_secret_access_key = aws_secret_access_key
+      @region = region
+      @endpoint = endpoint
+      @host = host
+      @bucket = bucket
+      @profiles = profiles
+      @job_id = SecureRandom.uuid
+    end
+
+    def perform!
+      Dir.mktmpdir do |wdir|
+        @wdir = wdir
+
+        download_input!
+
+        {
+          key: perform_video_encoding!,
+          thumbnail_key: get_thumbnail!,
+          small_thumbnail_key: get_small_thumbnail!,
+          preview_key: get_preview!,
+          duration: get_duration!
+        }
+      end
+    end
+
+    def download_input!
+      unless File.exists? input_path
+        call_command("curl #{Shellwords.escape(video_url)} -s -o #{input_path}")
+      end
+    end
+
+    def perform_video_encoding!
+      # Reencode
+      ffmpeg_configurations = profiles.map { |profile| ffmpeg_configuration_for(profile) }.join(" ")
+
+      call_command("ffmpeg -i #{input_path} -pass 1 #{ffmpeg_configurations}")
+      call_command("ffmpeg -i #{input_path} -pass 2 #{ffmpeg_configurations}")
+
+      # create index file
+      indices = profiles.map do |profile|
+        [
+          "#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=#{profile["bandwidth"]}",
+          "#{profile["name"]}_#{job_id}.m3u8"
+        ]
+      end
+      File.open("#{@wdir}/index.m3u8", 'w') { |f| f.write("#EXTM3U\n" + indices.flatten.join("\n")) }
+
+      # Upload to storage
+      Dir["#{@wdir}/*.{ts,m3u8}"].each do |path|
+        upload(path, "#{root_dir}/#{File.basename(path)}")
+      end
+
+      "#{root_dir}/index.m3u8"
+    end
+
+    def get_thumbnail!
+      call_command "ffmpeg -i #{input_path} -vcodec mjpeg -vframes 1 -filter:v scale=\"1080:-1\" -q:v 10 -an -f rawvideo -ss #{video_middle} #{thumbnail_path}"
+
+      "#{root_dir}/thumbnail.jpg".tap do |key|
+        upload(thumbnail_path, key)
+      end
+    end
+
+    def get_small_thumbnail!
+      call_command "ffmpeg -i #{input_path} -vcodec mjpeg -vframes 1 -filter:v scale=\"640:-1\" -q:v 10 -an -f rawvideo -ss #{video_middle} #{small_thumbnail_path}"
+
+      "#{root_dir}/small_thumbnail.jpg".tap do |key|
+        upload(small_thumbnail_path, key)
+      end
+    end
+
+    def get_preview!
+      call_command "ffmpeg -y -ss #{video_middle} -t 3 -i #{input_path} -vf fps=10,scale=320:-1:flags=lanczos,palettegen #{palette_path}"
+      call_command "ffmpeg -ss #{video_middle} -t 3 -i #{input_path} -i #{palette_path} -filter_complex \"fps=10,scale=320:-1:flags=lanczos[x];[x][1:v]paletteuse\" #{preview_path}"
+
+      "#{root_dir}/preview.gif".tap do |key|
+        upload(preview_path, key)
+      end
+    end
+
+    def get_duration!
+      output = JSON.parse(`ffprobe -of json -show_format_entry name -show_format #{input_path} -loglevel quiet`)
+      output["format"]["duration"].to_i
+    end
+
+    protected
+
+    def input_path
+      Shellwords.escape("#{@wdir}/input")
+    end
+
+    def thumbnail_path
+      Shellwords.escape("#{@wdir}/thumbnail.jpeg")
+    end
+
+    def small_thumbnail_path
+      Shellwords.escape("#{@wdir}/small_thumbnail.jpeg")
+    end
+
+    def preview_path
+      Shellwords.escape("#{@wdir}/preview.gif")
+    end
+
+    def palette_path
+      Shellwords.escape("#{@wdir}/palette.png")
+    end
+
+    def video_middle
+      Shellwords.escape(`ffmpeg -i #{input_path} 2>&1 | grep Duration | awk '{print $2}' | tr -d , | awk -F ':' '{print ($3+$2*60+$1*3600)/2}'`.chomp)
+    end
+
+    def call_command(command)
+      puts command
+      system(command)
+      raise if $?.to_i != 0
+    end
+
+    def ffmpeg_configuration_for(profile)
+      "-vcodec #{profile["codec"]} -acodec aac -strict -2 -q:a 5 -ac 1 -r 25 -profile:v baseline -vf scale='trunc(oh*a/2)*2:#{profile["resolution"]}' -preset slow -b:v #{profile["bandwidth"]} -maxrate #{profile["bandwidth"]} -pix_fmt yuv420p -flags -global_header -hls_time 10 -hls_list_size 0 #{@wdir}/#{profile["name"]}_#{job_id}.m3u8"
+    end
+
+    def storage
+      @storage ||= Fog::Storage.new({
+        provider:              'AWS',
+        aws_access_key_id:     aws_access_key_id,
+        aws_secret_access_key: aws_secret_access_key,
+        region:                region,
+        endpoint:              endpoint,
+        host:                  host,
+        path_style:            true
+      })
+      @storage.directories.get(bucket)
+    end
+
+    def upload(path, key)
+      open(path) do |file|
+        storage.files.create({
+          key: key,
+          body: file,
+          public: true
+        })
+      end.public_url
+    end
+  end
+end

data/lib/arthropod_hls_video_encoder/version.rb

@@ -0,0 +1,3 @@
+module ArthropodHlsVideoEncoder
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: arthropod_hls_video_encoder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Victor Goya
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: arthropod
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog-aws
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: HLS video encoder using Arthropod
+email:
+- goya.victor@gmail.com
+executables:
+- arthropod_hls_video_encoder
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.md
+- arthropod_hls_video_encoder.gemspec
+- bin/arthropod_hls_video_encoder
+- lib/arthropod_hls_video_encoder/encoder.rb
+- lib/arthropod_hls_video_encoder/version.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: HLS video encoder using Arthropod
+test_files: []