magma_cli 0.0.8

data/lib/magma/app.rb

@@ -0,0 +1,73 @@
+# rubocop:disable Lint/MissingCopEnableDirective, Metrics/LineLength
+
+require 'etc'
+require 'tempfile'
+require 'thor'
+require 'magma/preparer'
+require 'magma/renderer'
+require 'magma/templater'
+
+module Magma
+  # The CLI application
+  class App < Thor
+    class << self
+      def exit_on_failure?
+        true
+      end
+    end
+
+    desc 'render [outfile]', 'Renders an MLT into an HLS video'
+    option :context, type: :hash, default: {}, aliases: 'c', desc: 'Render-time context'
+    option :data_file, type: :string, aliases: 'D', desc: 'Static data file, accepts JSON or YML'
+    option :enable_real_time, type: :boolean, default: false, aliases: 'r', desc: 'Render in real-time'
+    option :format, type: :string, default: 'mp4', aliases: 'f', enum: Renderer::SUPPORTED_FORMATS, desc: 'Encoding format'
+    option :globals, type: :hash, default: {}, aliases: 'g', desc: 'Global data overrides'
+    option :infile, type: :string, aliases: 'i', desc: 'Input file', required: true
+    option :length, type: :numeric, default: 2, desc: 'Set the target length in seconds'
+    option :preset, type: :string, default: 'fast', desc: 'Video preset for encoding'
+    option :real_time, type: :numeric, default: Etc.nprocessors, aliases: 'n', desc: 'Number of cores (real-time)'
+    option :size, type: :string, default: '1280x720', aliases: 's', desc: 'Frame size (width x height)'
+    option :skip_template, type: :boolean, default: false, desc: 'Skips template-filling step'
+    option :vcodec, type: :string, desc: 'Video codec to use for encoding'
+    def render(outfile)
+      if options[:skip_template]
+        Renderer.call outfile, options
+      else
+        template = Tempfile.new [File.basename(outfile), '.mlt']
+
+        begin
+          Preparer.call options[:infile], options.merge(outfile: template)
+          Templater.call template, options.merge(outfile: template)
+          Renderer.call outfile, options.merge(infile: template.path)
+        ensure
+          template.close
+          template.unlink
+        end
+      end
+    end
+
+    desc 'template [file]', 'Fills a template'
+    option :context, type: :hash, default: {}, aliases: 'c', desc: 'Render-time context'
+    option :globals, type: :hash, default: {}, aliases: 'g', desc: 'Global data overrides'
+    option :data_file, type: :string, aliases: 'D', desc: 'Static data file, accepts JSON or YML'
+    option :outfile, type: :string, aliases: 'o', desc: 'Output file'
+    option :overwrite, type: :boolean, default: false, aliases: 'O', desc: 'Overwrite input file'
+    option :print, type: :string, aliases: 'p', desc: 'Print output'
+    def template(infile = nil)
+      Templater.call infile, options
+    end
+
+    desc 'prepare [file]', 'Prepares a template'
+    option :outfile, type: :string, aliases: 'o', desc: 'Output file'
+    option :overwrite, type: :boolean, default: false, aliases: 'O', desc: 'Overwrite input file'
+    option :print, type: :string, aliases: 'p', desc: 'Print output'
+    def prepare(infile = nil)
+      Preparer.call infile, options
+    end
+
+    desc 'version', 'Displays the version'
+    def version
+      puts Magma::VERSION
+    end
+  end
+end

data/lib/magma/common.rb

@@ -0,0 +1,36 @@
+require 'active_support/concern'
+
+module Magma
+  module Common
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def call(*args)
+        new(*args).call
+      end
+    end
+
+    def handle_output!(output, options)
+      output!(output) if options[:print]
+      overwrite!(options[:infile], output) if options[:overwrite]
+      save!(options[:outfile], output) if options[:outfile]
+      output
+    end
+
+    private
+
+    def output!(output)
+      puts output
+    end
+
+    def overwrite!(infile, output)
+      raise('No file to overwrite') unless infile
+      File.write infile, output
+    end
+
+    def save!(outfile, output)
+      raise('No output file specified') unless outfile
+      File.write outfile, output
+    end
+  end
+end

data/lib/magma/config/mlt.rb

@@ -0,0 +1,90 @@
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/hash/deep_merge'
+
+module Magma::Config
+  class MLT < ActiveSupport::HashWithIndifferentAccess
+    VERSION = 1
+    ALLOWED_KEYS = %w[
+      globals
+      scopes
+      type
+      version
+    ].freeze
+
+    attr_reader :data
+
+    def initialize(hash = {})
+      super({
+        globals: {},
+        scopes: {},
+        type: 'mlt',
+        version: VERSION,
+      }.merge(hash))
+      validate!
+    end
+
+    def globals
+      self[:globals]
+    end
+
+    def operations
+      raise 'needs to be resolved' unless resolved?
+      data.select { |key, _value| key[0] == '$' }
+    end
+
+    def resolve(context = {})
+      @data = context.reduce(globals.clone) do |res, (key, value)|
+        scope = scopes.fetch(key, {}).fetch(value, nil)
+        res.tap { |r| r.deep_merge!(scope) if scope }
+      end
+    end
+
+    def resolved?
+      data && true
+    end
+
+    def scopes
+      self[:scopes]
+    end
+
+    def transform(doc)
+      raise 'needs to be resolved' unless resolved?
+      operations.each do |key, obj|
+        # Skip if not prefixed with $
+        leader = key[0]
+        next unless leader == '$'
+
+        # Select node set
+        nodeset = doc.css(key[1..-1])
+
+        case obj
+        when Hash
+          obj.each do |name, value|
+            if value
+              nodeset.attr(name, value)
+            else
+              nodeset.remove_attr(name)
+            end
+          end
+        else
+          text = obj.to_s
+          nodeset.each { |node| node.content = text }
+        end
+      end
+      doc
+    end
+
+    def variables
+      raise 'needs to be resolved' unless resolved?
+      data.reject { |key, _value| key[0] == '$' }
+    end
+
+    private
+
+    def validate!
+      extras = keys - ALLOWED_KEYS
+      raise "extra keys: #{extras}" unless extras.empty?
+      raise 'not type: mlt' unless self[:type] == 'mlt'
+    end
+  end
+end

data/lib/magma/config.rb

@@ -0,0 +1,4 @@
+module Magma
+  module Config
+  end
+end

data/lib/magma/preparer.rb

@@ -0,0 +1,85 @@
+require 'magma/common'
+require 'magma/utils'
+require 'nokogiri'
+require 'securerandom'
+
+module Magma
+  class Preparer
+    include Magma::Common
+    include Magma::Utils
+
+    def initialize(infile, options = {})
+      @infile = infile
+      @options = options
+    end
+
+    def call
+      stabilize_producers!
+      handle_output! xml, options.merge(infile: infile)
+      xml
+    end
+
+    private
+
+    attr_accessor :infile, :options
+
+    def doc
+      @doc ||= Nokogiri::XML(source)
+    end
+
+    def log(message)
+      # Don't print the message if we're out-putting to STDOUT
+      puts message unless options[:print]
+    end
+
+    def resource_path
+      return Dir.pwd unless infile
+      File.dirname File.expand_path(infile)
+    end
+
+    def stabilize_producers! # rubocop:disable Metrics/AbcSize
+      map = Hash[doc.xpath('//producer').map do |producer|
+        id = producer.get_attribute 'id'
+        next nil if uuid? id
+        [producer.get_attribute('id'), uuid]
+      end.compact]
+      return if map.empty?
+
+      doc.traverse do |node|
+        id = node.get_attribute 'id'
+        case node.name
+        when 'producer'
+          if map[id]
+            log "Re-mapping producer##{id} id to #{map[id]}"
+            node.set_attribute('id', map[id])
+
+            node.search('property[name="resource"]').each do |property|
+              if File.file?(File.join(resource_path, property.content))
+                log "Expanding resource '#{resource_path}/#{property.content}'"
+                property.content = File.join(resource_path, property.content)
+              end
+            end
+          end
+        else
+          producer = node.get_attribute 'producer'
+          if map[producer]
+            node.set_attribute('producer', map[producer])
+            log "Re-mapping #{node.name}#{id ? '#' + id : ''} producer to #{map[producer]}"
+          end
+        end
+      end
+    end
+
+    def source
+      @source ||= read_file_or_stdin(infile) || raise('Need a template')
+    end
+
+    def uuid
+      SecureRandom.uuid
+    end
+
+    def xml
+      doc.to_xml
+    end
+  end
+end

data/lib/magma/renderer.rb

@@ -0,0 +1,123 @@
+require 'magma/utils'
+require 'pry'
+
+module Magma
+  class Renderer
+    SUPPORTED_FORMATS = %w[hls mp4].freeze
+    MELT = 'melt'.freeze
+
+    include Magma::Common
+    include Magma::Utils
+
+    def initialize(outfile, options)
+      @outfile = outfile
+      @options = options
+    end
+
+    def call
+      render!
+    end
+
+    private
+
+    attr_accessor :options, :outfile
+
+    # Adds sane defaults to config
+    def add_defaults(options)
+      {
+        hls_list_size: 0,
+        start_number: 0,
+      }.merge(options)
+    end
+
+    # Formats keys to melt arguments
+    def format(options)
+      options.map do |key, value|
+        [{
+          format: :f,
+          size: :s,
+          length: :hls_time,
+        }[key] || key, value].map(&:to_s).join('=')
+      end
+    end
+
+    # Finds and constructs a melt invocation string
+    def melt(args)
+      bin = which(MELT) || raise("#{MELT} not found. Is MLT installed?")
+      "#{bin} #{args}"
+    end
+
+    # Constructs melt arguments from options hash
+    def melt_args(options)
+      [
+        options[:infile],
+        "-consumer avformat:#{outfile}",
+      ].concat(pipe(options, %i[
+        symbolize_keys
+        add_defaults
+        select
+        transform
+        table
+        format
+      ]))
+    end
+
+    # Renders the project
+    def render!
+      cmd = melt melt_args(options.merge(outfile: outfile)).join(' ')
+      puts "Run: #{cmd}"
+      puts
+
+      run(cmd) do |_stdin, _stdout, stderr|
+        stderr.each("\r") do |line|
+          STDOUT.write "\r#{line}"
+        end
+      end
+    end
+
+    # Selects certain options
+    def select(options)
+      # Clone original
+      options = options.clone
+
+      # Handle related options
+      options.delete(:real_time) unless options.delete(:enable_real_time)
+
+      # Reject
+      options.select do |key, value|
+        !value.nil? && %i[
+          format
+          hls_list_size
+          real_time
+          length
+          preset
+          size
+          start_number
+          vcodec
+        ].include?(key)
+      end
+    end
+
+    # Prints a table and passes through the options hash
+    def table(options)
+      lpadding = options.keys.max_by(&:length).length + 2
+      rpadding = options.values.max_by { |v| v.to_s.length }.to_s.length
+      puts 'PROPERTY'.ljust(lpadding) + 'VALUE'
+      puts '=' * (lpadding + rpadding)
+      options.keys.sort.each do |key|
+        puts key.to_s.ljust(lpadding) + options[key].to_s
+      end
+      puts
+      options
+    end
+
+    # Transforms certain options values
+    def transform(options)
+      options.map do |key, value|
+        [key, ({
+          real_time: ->(x) { "-#{x}" },
+        }[key] || proc { |x| x }).call(value)]
+      end.to_h
+    end
+  end
+end

data/lib/magma/templater.rb

@@ -0,0 +1,78 @@
+require 'magma/config/mlt'
+require 'magma/common'
+require 'magma/utils'
+require 'liquid'
+require 'nokogiri'
+
+module Magma
+  class Templater
+    include Magma::Common
+    include Magma::Utils
+
+    def initialize(infile, options)
+      @infile   = infile
+      @options  = options
+    end
+
+    def call
+      pipe source, [
+        :template,
+        :render,
+        :parse,
+        :transform,
+        '.to_xml',
+        ->(result) { handle_output!(result, options.merge(infile: infile)) },
+      ]
+    end
+
+    private
+
+    attr_reader :infile, :options
+
+    def config
+      @config ||= Magma::Config::MLT.new(data).tap do |c|
+        c.resolve(options[:context] || {})
+      end
+    end
+
+    def data
+      @data ||= options[:data_file].nil? ? {} : YAML.safe_load(File.read(options[:data_file]), [], [], true)
+    end
+
+    def source
+      @source ||= read_file_or_stdin(infile) || raise('Need a template')
+    end
+
+    ### Pipeline
+
+    # Renders a Liquid template
+    def render(template)
+      template.render(config.variables.deep_merge(options[:globals]), strict_variables: true).tap do
+        if template.errors&.length&.positive?
+          puts template.errors
+          raise template.errors.map(&:to_s).join('; ')
+        end
+      end
+    end
+
+    # Parses a template into a DOM
+    def parse(template)
+      Nokogiri::XML template
+    end
+
+    # Parses a source into a template
+    def template(source)
+      Liquid::Template.parse(source, error_mode: :strict)
+    end
+
+    # Applies transformations to a document
+    def transform(doc)
+      config.transform doc
+    end
+
+    # Converts a docuemtn into XML
+    def xml(doc)
+      doc.to_xml
+    end
+  end
+end

data/lib/magma/utils.rb

@@ -0,0 +1,57 @@
+require 'open3'
+
+module Magma
+  module Utils
+    # Reads a file or from STDIN if piped
+    def read_file_or_stdin(filename = nil)
+      filename.nil? ? !STDIN.tty? && STDIN.read : File.read(filename)
+    end
+
+    # Returns whether a string is a UUID
+    def uuid?(uuid)
+      uuid_regex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+      uuid_regex =~ uuid.to_s.downcase
+    end
+
+    # Applies methods/procs to an object in order
+    def pipe(object, methods = [])
+      methods.reduce(object) do |acc, method|
+        case method
+        when Proc || method.respond_to?(:call)
+          method.call(acc)
+        when Symbol
+          send(method, acc)
+        when String
+          raise "must start with '.'" unless method[0] == '.'
+          acc.send(method[1..-1].to_sym)
+        else
+          raise "unexpected pipe #{method}"
+        end
+      end
+    end
+
+    # Delegates to Open3
+    def run(*args, &block)
+      return Open3.popen3(*args, &block) if block_given?
+      Open3.popen3(*args)
+    end
+
+    # Symbolizes a hash's keys
+    def symbolize_keys(hash)
+      hash.each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v }
+    end
+
+    # Cross-platform way of finding an executable in the $PATH
+    # Reference: https://stackoverflow.com/a/5471032/3557448
+    def which(cmd)
+      exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+      ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+        exts.each do |ext|
+          exe = File.join(path, "#{cmd}#{ext}")
+          return exe if File.executable?(exe) && !File.directory?(exe)
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/magma/version.rb

@@ -0,0 +1,3 @@
+module Magma
+  VERSION = '0.0.8'.freeze
+end

data/lib/magma.rb

@@ -0,0 +1,6 @@
+require 'magma/app'
+require 'magma/version'
+
+# The main CLI module
+module Magma
+end

data/magma.gemspec

@@ -0,0 +1,30 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'magma/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'magma_cli'
+  spec.version       = Magma::VERSION
+  spec.authors       = ['Allan Reyes']
+  spec.email         = ['allanbreyes@users.noreply.github.com']
+
+  spec.summary       = 'Magma CLI'
+  spec.description   = 'Command-line interface for MLT templating and rendering'
+  spec.homepage      = 'https://github.com/darbylabs/magma'
+  spec.license       = 'LGPL-2.1'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(examples|features|spec|test)/})
+  end
+  spec.executables   = ['magma']
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '~> 5.2'
+  spec.add_dependency 'liquid', '~> 4.0'
+  spec.add_dependency 'nokogiri', '~> 1.8'
+  spec.add_dependency 'thor', '~> 0.20'
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end