distorted 0.5.3 → 0.6.0

data/lib/distorted/molecule/image.rb

@@ -0,0 +1,36 @@
+
+require 'set'
+
+require 'distorted/checking_you_out'
+require 'distorted/modular_technology/vips'
+require 'distorted/injection_of_love'
+
+
+module Cooltrainer
+  module DistorteD
+    module Image
+
+
+      # Attributes for our <picture>/<img>.
+      # Automatically enabled as attrs for DD Liquid Tag.
+      # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture#Attributes
+      # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#Attributes
+      # https://developer.mozilla.org/en-US/docs/Web/Performance/Lazy_loading
+      ATTRIBUTES = Set[:alt, :caption, :href, :loading]
+
+      # Defaults for HTML Element attributes.
+      # Not every attr has to be listed here.
+      # Many need no default and just won't render.
+      ATTRIBUTES_DEFAULT = {
+        :loading => :eager,
+      }
+      ATTRIBUTES_VALUES = {
+        :loading => Set[:eager, :lazy],
+      }
+
+      include Cooltrainer::DistorteD::Technology::Vips
+      include Cooltrainer::DistorteD::InjectionOfLove
+
+    end  # Image
+  end  # DistorteD
+end  # Cooltrainer

data/lib/distorted/molecule/pdf.rb

@@ -0,0 +1,119 @@
+require 'set'
+
+require 'hexapdf'
+
+require 'distorted/checking_you_out'
+require 'distorted/injection_of_love'
+require 'distorted/molecule/C18H27NO3'
+
+
+module Cooltrainer
+  module DistorteD
+    module PDF
+
+
+      include Cooltrainer::DistorteD::Molecule::C18H27NO3
+
+      LOWER_WORLD = CHECKING::YOU::IN("application/pdf")
+
+      # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/object#Attributes
+      # https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdf_open_parameters.pdf
+      PDF_OPEN_PARAMS = Array[
+        # Keep the PDF Open Params in the order they are defined
+        # in the Adobe documentation, since it says they should
+        # be specified in the URL in that same order.
+        # Ruby's Set doesn't guarantee order, so use a plain Array here.
+        :nameddest,
+        :page,
+        :comment,
+        :collab,
+        :zoom,
+        :view,
+        :viewrect,
+        :pagemode,
+        :scrollbar,
+        :search,
+        :toolbar,
+        :statusbar,
+        :messages,
+        :navpanes,
+        :highlight,
+        :fdf,
+      ]
+      ATTRIBUTES = Set[
+        :alt,
+        :caption,
+        :height,  #<object> viewer container height.
+        :width,  # <object> viewer container width.
+      ].merge(PDF_OPEN_PARAMS)
+
+      # "You cannot use the reserved characters =, #, and &.
+      # There is no way to escape these special characters."
+      RESERVED_CHARACTERS_FRAGMENT = '[^=#&]+'.freeze
+
+      FLOAT_INT_FRAGMENT = '[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)'.freeze
+      ZERO_TO_ONE_HUNDRED = /^(([1-9]\d?|1\d{1})([.,]\d{0,1})?|100([.,]0{1})?)$/
+
+      ATTRIBUTES_DEFAULT = {
+        :height => '100%'.freeze,
+        :width => '100%'.freeze,
+        # BEGIN PDF Open Parameters
+        :page => 1,
+        :view => :Fit,
+        :pagemode => :none,
+        :scrollbar => 1,
+        :toolbar => 1,
+        :statusbar => 1,
+        :messages => 0,
+        :navpanes => 1,
+        # END PDF Open Parameters
+      }
+
+      # Adobe's PDF Open Parameters documentation sez:
+      # "Individual parameters, together with their values (separated by & or #),
+      # can be no greater then 32 characters in length."
+      # …but then goes on to show some examples (like `comment`)
+      # that are clearly longer than 32 characters.
+      # Dunno. I'll err on the side of giving you a footgun.
+      ATTRIBUTES_VALUES = {
+        :nameddest => /^#{RESERVED_CHARACTERS_FRAGMENT}$/,
+        :page => /\d/,
+        :comment => /^#{RESERVED_CHARACTERS_FRAGMENT}$/,
+        :collab => /^(DAVFDF|FSFDF|DB)@#{RESERVED_CHARACTERS_FRAGMENT}$/,
+        :zoom => /^#{FLOAT_INT_FRAGMENT}(,#{FLOAT_INT_FRAGMENT},#{FLOAT_INT_FRAGMENT})?$/,
+        :view => /^Fit(H|V|B|BH|BV(,#{FLOAT_INT_FRAGMENT})?)?$/,
+        :viewrect => /^#{FLOAT_INT_FRAGMENT},#{FLOAT_INT_FRAGMENT},#{FLOAT_INT_FRAGMENT},#{FLOAT_INT_FRAGMENT}$/,
+        :pagemode => Set[:none, :thumbs, :bookmarks],
+        :scrollbar => BOOLEAN_ATTR_VALUES,
+        :search => /^#{RESERVED_CHARACTERS_FRAGMENT}(,\s#{RESERVED_CHARACTERS_FRAGMENT})*$/,
+        :toolbar => BOOLEAN_ATTR_VALUES,
+        :statusbar => BOOLEAN_ATTR_VALUES,
+        :messages => BOOLEAN_ATTR_VALUES,
+        :navpanes => BOOLEAN_ATTR_VALUES,
+        :fdf => /^#{RESERVED_CHARACTERS_FRAGMENT}$/,
+      }
+
+      include Cooltrainer::DistorteD::InjectionOfLove
+
+      # TODO: Use MuPDF instead of libvips magick-based PDF loader.
+
+      def self.optimize(src, dest)
+        HexaPDF::Document.open(src) do |doc|
+          doc.task(
+            :optimize,
+            compact: true,
+            object_streams: :generate,
+            xref_streams: :generate,
+            compress_pages: false,
+          )
+          doc.write(dest)
+        end
+      end
+
+      def to_application_pdf(*a, **k, &b)
+        copy_file(*a, **k, &b)
+      end
+
+    end  # PDF
+  end  # DistorteD
+end  # Cooltrainer

data/lib/distorted/molecule/svg.rb

@@ -0,0 +1,60 @@
+require 'set'
+
+require 'svg_optimizer'
+
+require 'distorted/checking_you_out'
+require 'distorted/injection_of_love'
+require 'distorted/molecule/C18H27NO3'
+
+
+module Cooltrainer
+  module DistorteD
+    module SVG
+
+      SUB_TYPE = 'svg'.freeze
+      include Cooltrainer::DistorteD::Molecule::C18H27NO3
+
+      #WISHLIST: Support VML for old IE compatibility.
+      #  Example: RaphaëlJS — https://en.wikipedia.org/wiki/Rapha%C3%ABl_(JavaScript_library)
+      LOWER_WORLD = CHECKING::YOU::IN(/^image\/svg/)
+
+      ATTRIBUTES = Set[
+        :alt,
+        :caption,
+        :href,
+        :loading,
+        :optimize,
+      ]
+      ATTRIBUTES_VALUES = {
+        :optimize => BOOLEAN_ATTR_VALUES,
+      }
+      ATTRIBUTES_DEFAULT = {
+        :optimize => false,
+      }
+
+      include Cooltrainer::DistorteD::Technology::VipsSave
+      include Cooltrainer::DistorteD::InjectionOfLove
+
+      def to_vips_image
+        # TODO: Load-time options for various formats, like SVG's `unlimited`:
+        # "SVGs larger than 10MB are normally blocked for security. Set unlimited to allow SVGs of any size."
+        # https://libvips.github.io/libvips/API/current/VipsForeignSave.html#vips-svgload
+        @vips_image ||= Vips::Image.new_from_file(path)
+      end
+
+      def to_image_svg_xml(dest, *a, **k, &b)
+        if abstract(:optimize)
+          SvgOptimizer.optimize_file(path, dest, SvgOptimizer::DEFAULT_PLUGINS)
+        else
+          copy_file(dest, *a, **k, &b)
+        end
+      end
+
+      def self.optimize(src, dest)
+        # TODO: Make optimizations/plugins configurable
+        SvgOptimizer.optimize_file(src, dest, SvgOptimizer::DEFAULT_PLUGINS)
+      end
+
+    end
+  end
+end

data/lib/distorted/molecule/text.rb

@@ -0,0 +1,225 @@
+require 'set'
+
+require 'charlock_holmes'  # Text file charset detection
+
+require 'distorted/monkey_business/string'  # String#map
+require 'distorted/modular_technology/pango'
+require 'distorted/modular_technology/ttfunk'
+require 'distorted/modular_technology/vips_save'
+
+require 'distorted/checking_you_out'
+require 'distorted/injection_of_love'
+require 'distorted/molecule/image'
+
+
+module Cooltrainer
+  module DistorteD
+    module Text
+
+
+      LOWER_WORLD = CHECKING::YOU::IN(/^text\/(plain|x-nfo)/)
+      OUTER_LIMITS = CHECKING::YOU::IN(/^text\/(plain|x-nfo)/)
+
+      # Track supported fonts by codepage.
+      # Avoid renaming these from the original archives / websites.
+      # Try not to go nuts here bloating the size of our Gem for a
+      # very niche feature, but I want to ensure good coverage too.
+      #
+      # Treat codepage 8859 documents as codepage 1252 to avoid breaking smart-
+      # quotes and other printable chars in 1252 that are control chars in 8859.
+      # https://encoding.spec.whatwg.org/#names-and-labels
+      #
+      # Numeric key for UTF-8 is codepage 65001 like Win32:
+      # https://docs.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
+      FONT_FILENAME = {
+        :anonpro => 'Anonymous Pro.ttf'.freeze,
+        :anonpro_b => 'Anonymous Pro B.ttf'.freeze,
+        :anonpro_bi => 'Anonymous Pro BI.ttf'.freeze,
+        :anonpro_i => 'Anonymous Pro I.ttf'.freeze,
+        :lessperfectdosvga => 'LessPerfectDOSVGA.ttf'.freeze,
+        :moreperfectdisvga => 'MorePerfectDOSVGA.ttf'.freeze,
+        :perfectdosvgawin => 'Perfect DOS VGA 437 Win.ttf'.freeze,
+        :mona => 'mona.ttf'.freeze,
+        :perfectdosvga => 'Perfect DOS VGA 437.ttf'.freeze,
+        :profont => 'ProFontWindows.ttf'.freeze,
+        :profont_b => 'ProFontWindows-Bold.ttf'.freeze,
+      }
+      # Certain fonts are more suitable for certain codepages,
+      # so track each codepage's available fonts…
+      CODEPAGE_FONT = {
+        65001 => [
+          :anonpro,
+          :anonpro_b,
+          :anonpro_bi,
+          :anonpro_i,
+        ],
+        1252 => [
+          :lessperfectdosvga,
+          :moreperfectdosvga,
+          :perfectdosvgawin,
+        ],
+        932 => [
+          :mona,
+        ],
+        850 => [
+          :profont,
+          :profont_b,
+        ],
+        437 => [
+          :perfectdosvga,
+        ],
+      }
+      # …as well as the inverse, the numeric codepage for each font:
+      FONT_CODEPAGE = self::CODEPAGE_FONT.reduce(Hash.new([])) { |memo, (key, values)|
+        values.each { |value| memo[value] = key }
+        memo
+      }
+
+      self::OUTER_LIMITS.each { |t|
+        define_method(t.distorted_method) { |*a, **k, &b|
+          copy_file(*a, **k, &b)
+        }
+      }
+
+      ATTRIBUTES = Set[
+        :alt,
+        :crop,
+        :font,
+        :encoding,
+        :spacing,
+        :dpi,
+      ]
+      ATTRIBUTES_VALUES = {
+        :spacing => Set[:monospace, :proportional],
+        :font => self::FONT_FILENAME.keys.to_set,
+      }
+      ATTRIBUTES_DEFAULT = {
+        :crop => :none,
+        :dpi => 144,
+        :encoding => 'UTF-8'.freeze
+      }
+
+      include Cooltrainer::DistorteD::Technology::TTFunk
+      include Cooltrainer::DistorteD::Technology::Pango
+      include Cooltrainer::DistorteD::Technology::VipsSave
+      include Cooltrainer::DistorteD::InjectionOfLove
+
+      # Using a numeric key for things for simplicity.
+      # TODO: Replace this with Ruby's built-in Encoding class after I have
+      # a better idea what I want to do.
+      def codepage
+        case text_file_encoding
+          when 'UTF-8'.freeze then 65001
+          when 'Shift_JIS'.freeze then 932
+          when 'IBM437'.freeze then 437
+          else 1252
+        end
+      end
+
+      # Return a Pango Markup escaped version of the document.
+      def to_pango
+        # https://developer.gnome.org/glib/stable/glib-Simple-XML-Subset-Parser.html#g-markup-escape-text
+        escaped = text_file_utf8_content.map{ |c|
+          g_markup_escape_char(c)
+        }
+        if font_spacing == :monospace
+          "<tt>" << escaped << "</tt>"
+        else
+          escaped
+        end
+      end
+
+      protected
+
+      def text_file_content
+        # VIPS makes us provide the text content as a single variable,
+        # so we may as well just one-shot File.read() it into memory.
+        # https://kunststube.net/encoding/
+        @text_file_content ||= File.read(path)
+      end
+
+      def text_file_utf8_content
+        CharlockHolmes::Converter.convert(text_file_content, text_file_encoding, 'UTF-8'.freeze)
+      end
+
+      def text_file_encoding
+        # It's not easy or even possible in some cases to tell the "true" codepage
+        # we should use for any given text document, but using character detection
+        # is worth a shot if the user gave us nothing.
+        #
+        # TODO: Figure out if/how we can get IBM437 files to not be detected as ISO-8859-1
+        @text_file_encoding ||= (
+          abstract(:encoding).to_s ||
+          CharlockHolmes::EncodingDetector.detect(text_file_content)[:encoding] ||
+          'UTF-8'.freeze
+        ).to_s
+      end
+
+      def vips_font
+        # Set the shorthand Symbol key for our chosen font.
+        return abstract(:font)&.to_sym || CODEPAGE_FONT[codepage].first
+      end
+
+      def to_vips_image
+        # Load font metadata directly from the file so we don't have to
+        # duplicate it here to feed to Vips/Pango.
+        #
+        # irb(main)> font_meta.name.font_name
+        # => ["Perfect DOS VGA 437", "\x00P\x00e\x00r\x00f\x00e\x00c\x00t\x00 \x00D\x00O\x00S\x00 \x00V\x00G\x00A\x00 \x004\x003\x007"]
+        # irb(main)> font_meta.name.font_family
+        # => ["Perfect DOS VGA 437", "\x00P\x00e\x00r\x00f\x00e\x00c\x00t\x00 \x00D\x00O\x00S\x00 \x00V\x00G\x00A\x00 \x004\x003\x007"]
+        # irb(main)> font_meta.name.font_subfamily
+        # => ["Regular", "\x00R\x00e\x00g\x00u\x00l\x00a\x00r"]
+        # irb(main)> font_meta.name.postscript_name
+        # => "PerfectDOSVGA437"
+        # irb(main)> font_meta.line_gap
+        # => 0
+
+        # https://libvips.github.io/libvips/API/current/libvips-create.html#vips-text
+        Vips::Image.text(
+          # This string must be well-escaped Pango Markup:
+          # https://developer.gnome.org/pango/stable/pango-Markup.html
+          # However the official function for escaping text is
+          # not implemented in Ruby GLib, so we have to do it ourselves.
+          to_pango,
+          **{
+            # String absolute path to TTF
+            :fontfile => font_path,
+            # It's not enough to just specify the TTF path;
+            # we must also specify a font family, subfamily, and size.
+            :font => "#{font_name} 16",
+            # Space between lines (in Points).
+            :spacing => to_ttfunk.line_gap,
+            :justify => true,  # Requires libvips 8.8
+            :dpi => abstract(:dpi)&.to_i,
+          },
+        )
+      end
+
+      # Return the String absolute path to the TTF file
+      def font_path
+        File.join(
+          File.dirname(__FILE__),  # molecule
+          '..'.freeze,  # distorted
+          '..'.freeze,  # lib
+          '..'.freeze,  # DistorteD-Ruby
+          'font'.freeze,
+          font_codepage.to_s,
+          font_filename,
+        )
+      end
+
+      # Returns the numeric representation of the codepage
+      # covered by our font.
+      def font_codepage
+        FONT_CODEPAGE.dig(vips_font).to_s
+      end
+
+      # Returns the basename (with file extension) of our font.
+      def font_filename
+        FONT_FILENAME.dig(vips_font)
+      end
+
+    end  # Text
+  end  # DistorteD
+end  # Cooltrainer

data/lib/distorted/molecule/video.rb

@@ -0,0 +1,195 @@
+#!/usr/bin/env ruby
+# Tell the user to install the shared library if it's missing.
+begin
+  require 'gst'
+rescue LoadError => le
+  raise unless le.message =~ /libgst/
+
+  # Multiple OS help
+  help = <<~INSTALL
+
+  Please install the GStreamer library for your system.
+  INSTALL
+
+  # Re-raise with install message
+  raise $!, "#{help}\n#{$!}", $!.backtrace
+end
+
+require 'set'
+
+require 'distorted/checking_you_out'
+require 'distorted/injection_of_love'
+
+
+module Cooltrainer
+  module DistorteD
+    module Video
+
+      LOWER_WORLD = CHECKING::YOU::IN(/^video\/mp4/)
+
+      # Attributes for our <video>.
+      # Automatically enabled as attrs for DD Liquid Tag.
+      # https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#Attributes
+      ATTRIBUTES = Set[:caption]
+
+      # Defaults for HTML Element attributes.
+      # Not every attr has to be listed here.
+      # Many need no default and just won't render.
+      ATTRIBUTES_DEFAULT = {}
+      ATTRIBUTES_VALUES = {}
+      include Cooltrainer::DistorteD::InjectionOfLove
+
+      attr_accessor :dest
+
+
+      def initialize(src, dest, basename)
+        @src = src
+        @dest = dest
+        @basename = basename
+      end
+
+      def rotate(angle=nil)
+        false
+      end
+
+      def clean
+        false
+      end
+
+      def generate
+        self.generate_hls
+        begin
+          self.generate_dash
+        rescue Gst::ParseError::NoSuchElement
+          # This is going away once the new dashsink2 lands in Gst so :effort:
+        end
+      end
+
+      def generate_dash
+        orig_dest = @dest
+        orig_path = @src
+
+        FileUtils.mkdir_p(File.dirname(orig_dest))
+
+        hls_dest = File.join(File.dirname(orig_dest), @basename + '.dash')
+        FileUtils.mkdir_p(hls_dest)
+        Jekyll.logger.debug(@tag_name, "Re-muxing #{orig_path} to #{hls_dest}.")
+
+        #FileUtils.rm(orig_dest) if File.exist?(orig_dest)
+        if not File.file?(orig_dest)
+          FileUtils.cp(orig_path, orig_dest)
+        end
+
+        # https://gstreamer.freedesktop.org/documentation/tools/gst-launch.html?gi-language=c#pipeline-description
+        # TODO: Convert this from parse_launch() pipeline notation to Element objects
+        # TODO: Get source video duration/resolution/etc and use it to compute a
+        #  value for `target-duration`.
+        # TODO: Also support urldecodebin for remote media.
+        pipeline, error = Gst.parse_launch("filesrc name=src ! decodebin name=demux ! videoconvert ! vaapih264enc ! queue2 ! h264parse ! queue2 ! mux.video dashsink name=mux max-files=0 playlist-length=0 target-duration=2 demux. ! audioconvert ! voaacenc ! queue2 ! mux.audio")
+
+        if pipeline.nil?
+          Jekyll.logger.error(@tag_name, "Parse error: #{error.message}")
+          return false
+        end
+
+        filesrc = pipeline.get_by_name('src')
+        filesrc.location = orig_path
+
+        hls_playlist = "#{hls_dest}/#{@basename}.m3u8"
+        hls = pipeline.get_by_name('mux')
+        hls.location = "#{hls_dest}/#{@basename}%05d.ts"
+        hls.playlist_location = hls_playlist
+
+        # TODO: config option for absolute vs relative segment URIs in the playlist.
+        #hls.playlist_root = @url
+
+        # TODO: dashsink support once there is a stable GStreamer release including it:
+        # https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/merge_requests/704
+
+        pipeline.play
+
+        # Play until End Of Stream
+        event_loop(pipeline)
+
+        pipeline.stop
+
+      end
+
+      def generate_hls
+        orig_dest = @dest
+        orig_path = @src
+
+        FileUtils.mkdir_p(File.dirname(orig_dest))
+
+        hls_dest = File.join(File.dirname(orig_dest), @basename + '.hls')
+        FileUtils.mkdir_p(hls_dest)
+        Jekyll.logger.debug(@tag_name, "Re-muxing #{orig_path} to #{hls_dest}.")
+
+        #FileUtils.rm(orig_dest) if File.exist?(orig_dest)
+        if not File.file?(orig_dest)
+          FileUtils.cp(orig_path, orig_dest)
+        end
+
+        # https://gstreamer.freedesktop.org/documentation/tools/gst-launch.html?gi-language=c#pipeline-description
+        # TODO: Convert this from parse_launch() pipeline notation to Element objects
+        # TODO: Get source video duration/resolution/etc and use it to compute a
+        #  value for `target-duration`.
+        # TODO: Also support urldecodebin for remote media.
+        pipeline, error = Gst.parse_launch("filesrc name=src ! decodebin name=demux ! videoconvert ! vaapih264enc ! queue2 ! h264parse ! queue2 ! mux.video hlssink2 name=mux max-files=0 playlist-length=0 target-duration=2 demux. ! audioconvert ! voaacenc ! queue2 ! mux.audio")
+
+        if pipeline.nil?
+          Jekyll.logger.error(@tag_name, "Parse error: #{error.message}")
+          return false
+        end
+
+        filesrc = pipeline.get_by_name('src')
+        filesrc.location = orig_path
+
+        hls_playlist = "#{hls_dest}/#{@basename}.m3u8"
+        hls = pipeline.get_by_name('mux')
+        hls.location = "#{hls_dest}/#{@basename}%05d.ts"
+        hls.playlist_location = hls_playlist
+
+        # TODO: config option for absolute vs relative segment URIs in the playlist.
+        #hls.playlist_root = @url
+
+        # TODO: dashsink support once there is a stable GStreamer release including it:
+        # https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/merge_requests/704
+
+        pipeline.play
+
+        # Play until End Of Stream
+        event_loop(pipeline)
+
+        pipeline.stop
+
+        # HACK HACK HACK: Replace X-ALLOW-CACHE line in playlist with YES.
+        # This property does not seem to be exposed to the outside of hlssink:
+        # https://cgit.freedesktop.org/gstreamer/gst-plugins-bad/tree/ext/hls/gsthlssink.c
+        text = File.read(hls_playlist)
+        File.write(hls_playlist, text.gsub(/^#EXT-X-ALLOW-CACHE:NO$/, '#EXT-X-ALLOW-CACHE:YES'))
+      end
+
+      def event_loop(pipeline)
+        running = true
+        bus = pipeline.bus
+
+        while running
+          message = bus.poll(Gst::MessageType::ANY, -1)
+
+          case message.type
+          when Gst::MessageType::EOS
+            running = false
+          when Gst::MessageType::WARNING
+            warning, _debug = message.parse_warning
+            Jekyll.logger.warning(@tag_name, warning)
+          when Gst::MessageType::ERROR
+            error, _debug = message.parse_error
+            Jekyll.logger.error(@tag_name, error)
+            running = false
+          end
+        end
+      end
+    end  # Image
+  end  # DistorteD
+end  # Cooltrainer