distorted 0.6.0 → 0.7.0

data/lib/distorted/media_molecule/image.rb

@@ -0,0 +1,33 @@
+
+require 'set'
+
+require 'distorted/checking_you_out'
+require 'distorted/modular_technology/vips'
+
+
+module Cooltrainer; end
+module Cooltrainer::DistorteD; end
+module Cooltrainer::DistorteD::Molecule; end
+module Cooltrainer::DistorteD::Molecule::Image
+
+
+  # Attributes for our <picture>/<img>.
+  # Automatically enabled as attrs for DD Liquid Tag.
+  # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture#Attributes
+  # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#Attributes
+  # https://developer.mozilla.org/en-US/docs/Web/Performance/Lazy_loading
+  ATTRIBUTES = Set[:alt, :caption, :href, :loading]
+
+  # Defaults for HTML Element attributes.
+  # Not every attr has to be listed here.
+  # Many need no default and just won't render.
+  ATTRIBUTES_DEFAULT = {
+    :loading => :eager,
+  }
+  ATTRIBUTES_VALUES = {
+    :loading => Set[:eager, :lazy],
+  }
+
+  include Cooltrainer::DistorteD::Technology::Vips
+
+end  # Image

data/lib/distorted/media_molecule/pdf.rb

@@ -0,0 +1,44 @@
+require 'set'
+
+require 'hexapdf'
+
+require 'distorted/checking_you_out'
+
+
+module Cooltrainer; end
+module Cooltrainer::DistorteD; end
+module Cooltrainer::DistorteD::Molecule; end
+module Cooltrainer::DistorteD::Molecule::PDF
+
+
+  # https://hexapdf.gettalong.org/documentation/reference/api/HexaPDF/Document/index.html#method-c-new
+  # https://hexapdf.gettalong.org/documentation/reference/api/HexaPDF/index.html#DefaultDocumentConfiguration
+  # https://hexapdf.gettalong.org/documentation/reference/api/HexaPDF/Task/Optimize.html
+  PDF_TYPE = CHECKING::YOU::OUT['application/pdf']
+  LOWER_WORLD = Hash[
+    PDF_TYPE => nil,
+  ]
+  OUTER_LIMITS = Hash[
+    PDF_TYPE => nil,
+  ]
+
+  # TODO: Use MuPDF instead of libvips magick-based PDF loader.
+
+  def self.optimize(src, dest)
+    HexaPDF::Document.open(src) do |doc|
+      doc.task(
+        :optimize,
+        compact: true,
+        object_streams: :generate,
+        xref_streams: :generate,
+        compress_pages: false,
+      )
+      doc.write(dest)
+    end
+  end
+
+  define_method(PDF_TYPE.distorted_file_method) { |dest_root, change|
+    copy_file(change.paths(dest_root).first)
+  }
+
+end  # PDF

data/lib/distorted/media_molecule/svg.rb

@@ -0,0 +1,45 @@
+require 'set'
+
+require 'svg_optimizer'  # https://github.com/fnando/svg_optimizer
+
+require 'distorted/checking_you_out'
+require 'distorted/modular_technology/vips/save'
+
+
+module Cooltrainer; end
+module Cooltrainer::DistorteD; end
+module Cooltrainer::DistorteD::Molecule; end
+module Cooltrainer::DistorteD::Molecule::SVG
+
+  include Cooltrainer::DistorteD::Technology::Vips::Save
+
+  LOWER_WORLD = Hash[
+    CHECKING::YOU::OUT['image/svg+xml'] => Cooltrainer::DistorteD::Technology::Vips::vips_get_options(
+      Cooltrainer::DistorteD::Technology::Vips::vips_foreign_find_load_suffix(".#{CHECKING::YOU::OUT['image/svg+xml'].preferred_extension}")
+    ).merge(Hash[
+      :optimize => Cooltrainer::Compound.new(:optimize, valid: Cooltrainer::BOOLEAN_VALUES, default: false, blurb: 'SvgOptimizer'),
+    ])
+  ]
+
+  # WISHLIST: Support VML for old IE compatibility.
+  #  Example: RaphaëlJS — https://en.wikipedia.org/wiki/Rapha%C3%ABl_(JavaScript_library)
+  OUTER_LIMITS = Hash[
+    CHECKING::YOU::OUT['image/svg+xml'] => nil,
+  ]
+
+  def to_vips_image
+    # NOTE: libvips 8.9 added the `unlimited` argument to svgload.
+    # Loading SVGs >= 10MiB in size will fail on older libvips.
+    # https://github.com/libvips/libvips/commit/55e49831b801e05ddd974b1e2102fda7956c53f5
+    @vips_image ||= Vips::Image.new_from_file(path)
+  end
+
+  define_method(CHECKING::YOU::OUT['image/svg+xml'].distorted_file_method) { |dest_root, change|
+    if change.optimize
+      SvgOptimizer.optimize_file(path, change.paths(dest_root).first, SvgOptimizer::DEFAULT_PLUGINS)
+    else
+      copy_file(change.paths(dest_root).first)
+    end
+  }
+
+end

data/lib/distorted/media_molecule/text.rb

@@ -0,0 +1,203 @@
+require 'set'
+
+require 'charlock_holmes'  # Text file charset detection
+
+require 'distorted/monkey_business/encoding'
+require 'distorted/monkey_business/string'  # String#map
+require 'distorted/modular_technology/pango'
+require 'distorted/modular_technology/ttfunk'
+require 'distorted/modular_technology/vips/save'
+
+require 'distorted/checking_you_out'
+
+
+module Cooltrainer; end
+module Cooltrainer::DistorteD; end
+module Cooltrainer::DistorteD::Molecule; end
+module Cooltrainer::DistorteD::Molecule::Text
+
+  #TODO: Generate separate images per-size to stop text being blurry from resizing.
+
+  include Cooltrainer::DistorteD::Technology::TTFunk
+  include Cooltrainer::DistorteD::Technology::Pango
+  include Cooltrainer::DistorteD::Technology::Vips::Save
+
+  # Track supported fonts by codepage.
+  # Avoid renaming these from the original archives / websites.
+  # Try not to go nuts here bloating the size of our Gem for a
+  # very niche feature, but I want to ensure good coverage too.
+  #
+  # Treat codepage 8859 documents as codepage 1252 to avoid breaking smart-
+  # quotes and other printable chars in 1252 that are control chars in 8859.
+  # https://encoding.spec.whatwg.org/#names-and-labels
+  #
+  # Numeric key for UTF-8 is codepage 65001 like Win32:
+  # https://docs.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
+  FONT_FILENAME = {
+    :anonpro => 'Anonymous Pro.ttf'.freeze,
+    :anonpro_b => 'Anonymous Pro B.ttf'.freeze,
+    :anonpro_bi => 'Anonymous Pro BI.ttf'.freeze,
+    :anonpro_i => 'Anonymous Pro I.ttf'.freeze,
+    :lessperfectdosvga => 'LessPerfectDOSVGA.ttf'.freeze,
+    :moreperfectdisvga => 'MorePerfectDOSVGA.ttf'.freeze,
+    :perfectdosvgawin => 'Perfect DOS VGA 437 Win.ttf'.freeze,
+    :mona => 'mona.ttf'.freeze,
+    :perfectdosvga => 'Perfect DOS VGA 437.ttf'.freeze,
+    :profont => 'ProFontWindows.ttf'.freeze,
+    :profont_b => 'ProFontWindows-Bold.ttf'.freeze,
+  }
+  # Certain fonts are more suitable for certain codepages,
+  # so track each codepage's available fonts…
+  CODEPAGE_FONT = {
+    65001 => [
+      :anonpro,
+      :anonpro_b,
+      :anonpro_bi,
+      :anonpro_i,
+    ],
+    1252 => [
+      :lessperfectdosvga,
+      :moreperfectdosvga,
+      :perfectdosvgawin,
+    ],
+    932 => [
+      :mona,
+    ],
+    850 => [
+      :profont,
+      :profont_b,
+    ],
+    437 => [
+      :perfectdosvga,
+    ],
+  }
+  # …as well as the inverse, the numeric codepage for each font:
+  FONT_CODEPAGE = self::CODEPAGE_FONT.reduce(Hash.new([])) { |memo, (key, values)|
+    values.each { |value| memo[value] = key }
+    memo
+  }
+
+
+  LOWER_WORLD = CHECKING::YOU::IN(/^text\/(plain|x-nfo)/).to_hash.transform_values { |v| Hash[
+    :encoding => Cooltrainer::Compound.new(:encoding, blurb: 'Character encoding used in this document.', default: 'UTF-8'.freeze),
+  ]}
+  OUTER_LIMITS = CHECKING::YOU::IN(/^text\/(plain|x-nfo)/).to_hash.merge(
+    Cooltrainer::DistorteD::Technology::Vips::Save::OUTER_LIMITS.dup.transform_values{ |v| Hash[
+      :spacing => Cooltrainer::Compound.new(:spacing, blurb: 'Document-wide character spacing style.', valid: Set[:monospace, :proportional]),
+      :dpi => Cooltrainer::Compound.new(:dpi, blurb: 'Dots per inch for text rendering.', valid: Integer, default: 144),
+      :font => Cooltrainer::Compound.new(:font, blurb: 'Font to use for text rendering.', valid: self::FONT_FILENAME.keys.to_set),
+    ]}
+  )
+
+  self::LOWER_WORLD.keys.each { |t|
+    define_method(t.distorted_file_method) { |dest_root, change|
+      copy_file(change.paths(dest_root).first)
+    }
+  }
+
+
+  # Return a Pango Markup escaped version of the document.
+  def to_pango
+    # https://developer.gnome.org/glib/stable/glib-Simple-XML-Subset-Parser.html#g-markup-escape-text
+    escaped = text_file_utf8_content.map{ |c|
+      g_markup_escape_char(c)
+    }
+    if font_spacing == :monospace
+      "<tt>" << escaped << "</tt>"
+    else
+      escaped
+    end
+  end
+
+  protected
+
+  def text_file_content
+    # VIPS makes us provide the text content as a single variable,
+    # so we may as well just one-shot File.read() it into memory.
+    # https://kunststube.net/encoding/
+    @text_file_content ||= File.read(path)
+  end
+
+  def text_file_utf8_content
+    CharlockHolmes::Converter.convert(text_file_content, text_file_encoding.to_s, 'UTF-8'.freeze)
+  end
+
+  def text_file_encoding
+    # It's not easy or even possible in some cases to tell the "true" codepage
+    # we should use for any given text document, but using character detection
+    # is worth a shot if the user gave us nothing.
+    #
+    # TODO: Figure out if/how we can get IBM437 files to not be detected as ISO-8859-1
+    @text_file_encoding ||= Encoding::find(
+      abstract(:encoding).to_s ||
+      CharlockHolmes::EncodingDetector.detect(text_file_content)[:encoding] ||
+      'UTF-8'.freeze
+    )
+  end
+
+  def vips_font
+    # Set the shorthand Symbol key for our chosen font.
+    return abstract(:font)&.to_sym || CODEPAGE_FONT[text_file_encoding.code_page].first
+  end
+
+  def to_vips_image
+    # Load font metadata directly from the file so we don't have to
+    # duplicate it here to feed to Vips/Pango.
+    #
+    # irb(main)> font_meta.name.font_name
+    # => ["Perfect DOS VGA 437", "\x00P\x00e\x00r\x00f\x00e\x00c\x00t\x00 \x00D\x00O\x00S\x00 \x00V\x00G\x00A\x00 \x004\x003\x007"]
+    # irb(main)> font_meta.name.font_family
+    # => ["Perfect DOS VGA 437", "\x00P\x00e\x00r\x00f\x00e\x00c\x00t\x00 \x00D\x00O\x00S\x00 \x00V\x00G\x00A\x00 \x004\x003\x007"]
+    # irb(main)> font_meta.name.font_subfamily
+    # => ["Regular", "\x00R\x00e\x00g\x00u\x00l\x00a\x00r"]
+    # irb(main)> font_meta.name.postscript_name
+    # => "PerfectDOSVGA437"
+    # irb(main)> font_meta.line_gap
+    # => 0
+
+    # https://libvips.github.io/libvips/API/current/libvips-create.html#vips-text
+    Vips::Image.text(
+      # This string must be well-escaped Pango Markup:
+      # https://developer.gnome.org/pango/stable/pango-Markup.html
+      # However the official function for escaping text is
+      # not implemented in Ruby GLib, so we have to do it ourselves.
+      to_pango,
+      **{
+        # String absolute path to TTF
+        :fontfile => font_path,
+        # It's not enough to just specify the TTF path;
+        # we must also specify a font family, subfamily, and size.
+        :font => "#{font_name} 16",
+        # Space between lines (in Points).
+        :spacing => to_ttfunk.line_gap,
+        :justify => true,  # Requires libvips 8.8
+        :dpi => abstract(:dpi)&.to_i,
+      },
+    )
+  end
+
+  # Return the String absolute path to the TTF file
+  def font_path
+    File.join(
+      File.dirname(__FILE__),  # molecule
+      '..'.freeze,  # distorted
+      '..'.freeze,  # lib
+      '..'.freeze,  # DistorteD-Floor
+      'font'.freeze,
+      font_codepage.to_s,
+      font_filename,
+    )
+  end
+
+  # Returns the numeric representation of the codepage
+  # covered by our font.
+  def font_codepage
+    FONT_CODEPAGE.dig(vips_font).to_s
+  end
+
+  # Returns the basename (with file extension) of our font.
+  def font_filename
+    FONT_FILENAME.dig(vips_font)
+  end
+
+end  # Text

data/lib/distorted/media_molecule/video.rb

@@ -0,0 +1,18 @@
+require 'set'
+require 'distorted/monkey_business/set'
+
+require 'distorted/checking_you_out'
+
+require 'distorted/modular_technology/gstreamer'
+
+
+module Cooltrainer; end
+module Cooltrainer::DistorteD; end
+module Cooltrainer::DistorteD::Molecule; end
+module Cooltrainer::DistorteD::Molecule::Video
+
+  LOWER_WORLD = CHECKING::YOU::IN('video/mp4').to_hash
+
+  include Cooltrainer::DistorteD::Technology::GStreamer
+
+end  # Video

data/lib/distorted/modular_technology/gstreamer.rb

@@ -0,0 +1,174 @@
+require 'set'
+
+require 'distorted/checking_you_out'
+
+require 'distorted/triple_counter'
+GST_MINIMUM_VER = TripleCounter.new(1, 18, 0)
+
+begin
+  require 'gst'
+  GST_AVAILABLE_VER = TripleCounter.new(*(Gst.version))
+  unless GST_AVAILABLE_VER >= GST_MINIMUM_VER
+    raise LoadError.new(
+      "DistorteD needs GStreamer #{GST_MINIMUM_VER}, but the available version is '#{Gst.version_string}'"
+    )
+  end
+rescue LoadError => le
+  raise unless le.message =~ /libgst/
+
+  # Multiple OS help
+  help = <<~INSTALL
+
+  Please install the GStreamer library for your system, version #{GST_MINIMUM_VER} or later.
+  INSTALL
+
+  # Re-raise with install message
+  raise $!, "#{help}\n#{$!}", $!.backtrace
+end
+
+
+module Cooltrainer; end
+module Cooltrainer::DistorteD; end
+module Cooltrainer::DistorteD::Technology; end
+module Cooltrainer::DistorteD::Technology::GStreamer
+
+  OUTER_LIMITS = CHECKING::YOU::IN(Set[
+    'application/dash+xml',
+    'application/vnd.apple.mpegurl',
+    'video/mp4',
+  ])
+
+
+  def write_video_mp4(dest_root, change)
+    copy_file(change.paths(dest_root).first)
+  end
+
+  def write_application_dash_xml(dest, *a, **k)
+    begin
+      segment_dest = File.join(File.dirname(dest), "#{basename}.dash", '/')
+      #segment_dest = segment_dest.sub("#{@base}/", '')
+      FileUtils.mkdir_p(segment_dest)
+      Jekyll.logger.debug(@tag_name, "Re-muxing #{path} to #{segment_dest}")
+
+      # https://gstreamer.freedesktop.org/documentation/tools/gst-launch.html?gi-language=c#pipeline-description
+      # TODO: Convert this from parse_launch() pipeline notation to Element objects
+      # TODO: Get source video duration/resolution/etc and use it to compute a
+      #  value for `target-duration`.
+      # TODO: Also support urldecodebin for remote media.
+      pipeline, error = Gst.parse_launch("dashsink name=mux  filesrc name=src ! decodebin name=demux ! audioconvert ! avenc_aac ! mux.audio_0 demux. ! videoconvert ! x264enc ! mux.video_0")
+
+      if pipeline.nil?
+        Jekyll.logger.error(@tag_name, "Parse error: #{error.message}")
+        return false
+      end
+
+      filesrc = pipeline.get_by_name('src')
+      filesrc.location = path
+
+      mux = pipeline.get_by_name('mux')
+      mux.mpd_filename = File.basename(dest)
+      mux.target_duration = 3
+      #mux.segment_tpl_path = "#{segment_dest}/#{basename}%05d.mp4"
+      mux.mpd_root_path = segment_dest
+      Jekyll.logger.warn('MPD ROOT PATH', mux.get_property('mpd-root-path'))
+
+      # typedef enum
+      # {
+      #   GST_DASH_SINK_MUXER_TS = 0,
+      #   GST_DASH_SINK_MUXER_MP4 = 1,
+      # } GstDashSinkMuxerType;
+      mux.muxer = 1
+
+      pipeline.play
+
+      # Play until End Of Stream
+      event_loop(pipeline)
+
+      pipeline.stop
+
+    rescue Gst::ParseError::NoSuchElement
+      raise
+    end
+  end
+
+  def write_application_vnd_apple_mpegurl(dest, *a, **k)
+    begin
+      orig_dest = dest
+      orig_path = path
+
+      FileUtils.mkdir_p(File.dirname(orig_dest))
+
+      hls_dest = File.join(File.dirname(orig_dest), basename + '.hls')
+      FileUtils.mkdir_p(hls_dest)
+      Jekyll.logger.debug(@tag_name, "Re-muxing #{orig_path} to #{hls_dest}.")
+
+      #FileUtils.rm(orig_dest) if File.exist?(orig_dest)
+      if not File.file?(orig_dest)
+        FileUtils.cp(orig_path, orig_dest)
+      end
+
+      # https://gstreamer.freedesktop.org/documentation/tools/gst-launch.html?gi-language=c#pipeline-description
+      # TODO: Convert this from parse_launch() pipeline notation to Element objects
+      # TODO: Get source video duration/resolution/etc and use it to compute a
+      #  value for `target-duration`.
+      # TODO: Also support urldecodebin for remote media.
+      pipeline, error = Gst.parse_launch("filesrc name=src ! decodebin name=demux ! videoconvert ! x264enc ! queue2 ! h264parse ! queue2 ! mux.video hlssink2 name=mux max-files=0 playlist-length=0 target-duration=2 demux. ! audioconvert ! faac ! queue2 ! mux.audio")
+
+      if pipeline.nil?
+        Jekyll.logger.error(@tag_name, "Parse error: #{error.message}")
+        return false
+      end
+
+      filesrc = pipeline.get_by_name('src')
+      filesrc.location = orig_path
+
+      hls_playlist = "#{hls_dest}/#{basename}.m3u8"
+      hls = pipeline.get_by_name('mux')
+      hls.location = "#{hls_dest}/#{basename}%05d.ts"
+      hls.playlist_location = hls_playlist
+
+      # TODO: config option for absolute vs relative segment URIs in the playlist.
+      #hls.playlist_root = @url
+
+      # TODO: dashsink support once there is a stable GStreamer release including it:
+      # https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/merge_requests/704
+
+      pipeline.play
+
+      # Play until End Of Stream
+      event_loop(pipeline)
+
+      pipeline.stop
+
+      # HACK HACK HACK: Replace X-ALLOW-CACHE line in playlist with YES.
+      # This property does not seem to be exposed to the outside of hlssink:
+      # https://cgit.freedesktop.org/gstreamer/gst-plugins-bad/tree/ext/hls/gsthlssink.c
+      text = File.read(hls_playlist)
+      File.write(hls_playlist, text.gsub(/^#EXT-X-ALLOW-CACHE:NO$/, '#EXT-X-ALLOW-CACHE:YES'))
+    rescue Gst::ParseError::NoSuchElement
+      raise
+    end
+  end
+
+  def event_loop(pipeline)
+    running = true
+    bus = pipeline.bus
+
+    while running
+      message = bus.poll(Gst::MessageType::ANY, -1)
+
+      case message.type
+      when Gst::MessageType::EOS
+        running = false
+      when Gst::MessageType::WARNING
+        warning, _debug = message.parse_warning
+        Jekyll.logger.warning(@tag_name, warning)
+      when Gst::MessageType::ERROR
+        error, _debug = message.parse_error
+        Jekyll.logger.error(@tag_name, error)
+        running = false
+      end
+    end
+  end
+
+end