checking-you-out 0.7.0

data/README.md

@@ -0,0 +1,14 @@
+CHECKING::YOU::OUT is DistorteD's file-identification library, originally a
+`ruby-mime-types` wrapper/monkey-patch — now its own full implementation
+of a `shared-mime-info` parser and in-memory representation.
+
+Designed with speed and memory-efficiency in mind. See `bin/benchmark` for more.
+
+Prior Art:
+https://github.com/mime-types/ruby-mime-types
+https://github.com/hanklords/shared-mime-info
+https://github.com/mimemagicrb/mimemagic
+https://github.com/discourse/mini_mime
+https://github.com/rails/marcel/
+https://github.com/blackwinter/ruby-filemagic/
+https://github.com/glongman/ffiruby-filemagic/

data/bin/are-we-unallocated-yet

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+
+require 'memory_profiler'
+
+$: << File.expand_path('../lib', __dir__)
+result = MemoryProfiler.report do
+  require 'checking-you-out'
+end.pretty_print

data/bin/benchmark

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'benchmark/ips'
+require 'memory_profiler'
+
+# Add our development CYO to LOAD_PATH so we can run it without Bundler.
+$: << File.expand_path('../../lib', __FILE__)
+
+puts
+puts "Memory stats for requiring mime/types/columnar"
+result = MemoryProfiler.report do
+  require 'mime/types/columnar'
+end
+
+# Note: Some creative accounting is necessary to accurately count the number of unique `Mime::Type`s
+# included in this library's data, since it treats aliases as separate types, e.g.:
+#  "doc"=>[#<MIME::Type: application/msword>, #<MIME::Type: application/word>, #<MIME::Type: application/x-msword>, #<MIME::Type: application/x-word>],
+#  "rtf"=>[#<MIME::Type: application/rtf>, #<MIME::Type: application/x-rtf>, #<MIME::Type: text/rtf>, #<MIME::Type: text/x-rtf>],
+#  "csv"=>[#<MIME::Type: text/comma-separated-values>, #<MIME::Type: text/csv>],
+#
+# I recognize that there is some overlap between `:preferred_extension`s where the types are *not*
+# the same thing, but this is the best way I have to get a count anywhere close to correct.
+# You can see for yourself with `MIME::Types.to_a.group_by(&:preferred_extension)` to double-check me.
+#
+# There are also many many `MIME::Type`s with no recorded extname at all, and I am intentionally
+# not counting them because this library doesn't do content-matching, meaning without that filename
+# metadata there's no other way we could identify files of that type with this library.
+#   irb> MIME::Types.to_a.group_by(&:preferred_extension)[nil].size => 1350
+puts "Total known file extensions: #{MIME::Types.to_a.group_by(&:preferred_extension).size}"
+puts "Total allocated: #{result.total_allocated_memsize} bytes (#{result.total_allocated} objects)"
+puts "Total retained:  #{result.total_retained_memsize} bytes (#{result.total_retained} objects)"
+
+puts
+puts "Memory stats for requiring mini_mime"
+result = MemoryProfiler.report do
+  require 'mini_mime'
+end
+
+puts "Total known file extensions: #{File.readlines(MiniMime::Configuration.ext_db_path).size}"
+puts "Total allocated: #{result.total_allocated_memsize} bytes (#{result.total_allocated} objects)"
+puts "Total retained:  #{result.total_retained_memsize} bytes (#{result.total_retained} objects)"
+
+puts
+puts "Memory stats for requiring CHECKING-YOU-OUT"
+result = MemoryProfiler.report do
+  require 'checking-you-out'
+end
+
+puts "Total known file extensions: #{CHECKING::YOU::OUT::instance_variable_get(:@after_forever).size}"
+puts "Total allocated: #{result.total_allocated_memsize} bytes (#{result.total_allocated} objects)"
+puts "Total retained:  #{result.total_retained_memsize} bytes (#{result.total_retained} objects)"
+puts
+
+Benchmark.ips do |bm|
+  bm.report 'cached content_type lookup MiniMime' do
+    MiniMime.lookup_by_filename("a.txt").content_type
+  end
+
+  bm.report 'content_type lookup MIME::Types' do
+    MIME::Types.type_for("a.txt")[0].content_type
+  end
+
+  bm.report 'content_type lookup CHECKING::YOU::OUT' do
+    CHECKING::YOU::OUT::from_postfix("txt")
+  end
+end

data/bin/checking-you-out

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/checking-you-out'
+#require 'profile'
+
+abort("Be with ye") if ARGV.empty?
+
+puts CHECKING::YOU::OUT(ARGV.first)

data/bin/repl

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "checking-you-out"
+require "irb"
+require "objspace"
+IRB.start(__FILE__)

data/bin/test-my-best

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/checking-you-out'
+require_relative '../TEST-MY-BEST/test-my-best'

data/lib/checking-you-out.rb

@@ -0,0 +1,40 @@
+require 'pathname' unless defined? ::Pathname
+
+# Silence warning for Ractor use in Ruby 3.0.
+# See https://ruby-doc.org/core/Warning.html#method-c-5B-5D for more.
+# TODO: Remove this when Ractors are "stable".
+Warning[:experimental] = false
+
+
+class CHECKING; end
+require_relative 'checking-you-out/inner_spirit' unless defined? ::CHECKING::YOU::IN
+
+
+# I'm not trying to be an exact clone of `shared-mime-info`, but I think its "Recommended checking order"
+# is pretty sane: https://specifications.freedesktop.org/shared-mime-info-spec/latest/
+#
+# In addition to the above, CYO() supports IETF-style Media Type strings like "application/xhtml+xml"
+# and supports `stat`-less testing of `.extname`-style Strings.
+class CHECKING::YOU
+  def self.OUT(unknown_identifier, so_deep: true)
+    case unknown_identifier
+    when ::Pathname
+      TEST_EXTANT_PATHNAME.call(unknown_identifier)
+    when ::String
+      case
+      when unknown_identifier.count(-?/) == 1 then  # TODO: Additional String validation here.
+        ::CHECKING::YOU::OUT::from_ietf_media_type(unknown_identifier)
+      when unknown_identifier.start_with?(-?.) && unknown_identifier.count(-?.) == 1 then
+        ::CHECKING::YOU::OUT::from_pathname(unknown_identifier)
+      else
+        if File::exist?(File::expand_path(unknown_identifier)) and so_deep then
+          TEST_EXTANT_PATHNAME.call(Pathname.new(File::expand_path(unknown_identifier)))
+        else
+          LEGENDARY_HEAVY_GLOW.call(::CHECKING::YOU::OUT::from_glob(unknown_identifier), :weight) || ::CHECKING::YOU::OUT::from_postfix(unknown_identifier)
+        end
+      end
+    when ::CHECKING::YOU::IN
+      unknown_identifier.out
+    end
+  end
+end

data/lib/checking-you-out/auslandsgesprach.rb

@@ -0,0 +1,253 @@
+
+# https://github.com/jarib/ffi-xattr
+require 'ffi-xattr'
+
+
+module CHECKING::YOU::IN::AUSLANDSGESPRÄCH
+
+  # Parse IETF Media-Type String → `::CHECKING::YOU::IN`
+  FROM_IETF_TYPE = proc {
+    # Keep these allocated instead of fragmenting our heap, since this will be called very frequently.
+    scratch = Array.allocate
+    hold = Array.allocate
+    my_base = ::CHECKING::YOU::IN::allocate
+
+    # Clear out the contents of the above temporary vars,
+    # called to ensure we never leak the contents of one parse into another.
+    the_bomb = proc {
+      scratch.clear
+      hold.clear
+      my_base.members.each { |gentleman|
+        my_base[gentleman] = nil
+      }
+    }
+
+    # Take a single codepoint from a reversed-then-NULL-terminated IETF Type String,
+    # e.g. "ttub=traf;lmbe+fnb.ppg3.dnv/noitacilppa#{-?\u{0}}".
+    #
+    #
+    # I switched from `String#each_char` to `String#each_codepoint` to avoid allocating single-character Strings
+    # before they could be deduplicated with `-zig`. The Integer codepoints, on the other hand,
+    # will always be the same `object_id` for the same codepoint:
+    #
+    # rb(main):162:0> -"あああ".each_char { |c| p c.object_id }
+    # 420
+    # 440
+    # 460
+    #
+    # rb(main):163:0> -"あああ".each_codepoint { |c| p c.object_id }
+    # 4709
+    # 4709
+    # 4709
+    #
+    #
+    # Putting the codepoints back together with `Array#pack` is the fastest way I've found,
+    # but it is mildly annoying that it won't intern the packed Strings in the C `pack_pack` code,
+    # which lives here: https://github.com/ruby/ruby/blob/master/pack.c
+    #
+    # This means that interning them here in Ruby-land forces us to eat the spurious allocation
+    # and is generally slower than it needs to be, e.g. `memory_profiler` report without post-allocation interning:
+    #
+    #   Retained String Report
+    #   -----------------------------------
+    #       1038  "application"
+    #       1037  <internal:pack>:135
+    #
+    #        171  "text"
+    #        170  <internal:pack>:135
+    #
+    #        140  "image"
+    #        139  <internal:pack>:135
+    #   […]
+    #   -----------------------------------
+    #
+    # …vs `memory_profiler` report *with* post-allocation interning, where the duplicate packed Strings
+    # are now "Allocated" instead of "Retained", i.e. they will be GCed:
+    #
+    #   Allocated String Report
+    #   -----------------------------------
+    #       2863  ""
+    #       2818  /home/okeeblow/Works/DistorteD/CHECKING-YOU-OUT/lib/checking-you-out/party_starter.rb:90
+    #
+    #       2271  "application"
+    #       2269  <internal:pack>:135
+    #   […]
+    #   -----------------------------------
+    #
+    # This ends up being a difference of ~2000 Objects for us, comparing the same before/after as above:
+    #
+    #   [okeeblow@emi#CHECKING-YOU-OUT] ./bin/are-we-unallocated-yet|grep Total
+    #   Total allocated: 18178861 bytes (318640 objects)
+    #   Total retained:  1999675 bytes (26815 objects)
+    #   [okeeblow@emi#CHECKING-YOU-OUT] ./bin/are-we-unallocated-yet|grep Total
+    #   Total allocated: 18231779 bytes (319963 objects)
+    #   Total retained:  1926675 bytes (24996 objects)
+    move_zig = proc { |zig|
+      case zig
+      when 0 then  # NULL
+        my_base[:phylum] = scratch.reverse!.pack(-'U*').-@
+      when 61 then  # =
+        # TODO: Implement Fragment-based Type variations
+        hold.push(*scratch)
+        scratch.clear
+      when 59 then  # ;
+        # TODO: Implement Fragment-based Type variations
+        scratch.clear
+        hold.clear
+      when 43 then  # +
+        #TODO: Implement tagged parent Types e.g. `+xml`
+        scratch.clear
+      when 47 then  # /
+        # When this character is encountered in a reversed Type String, `scratch` will contain the facet
+        # which lets us determine if this Type belongs to a vendor tree, to the e`x`perimental tree, etc.
+        my_base[:kingdom] = case
+        when scratch[-3..] == (-'dnv').codepoints then
+          scratch.pop(3);
+          # https://datatracker.ietf.org/doc/html/rfc6838#section-3.2
+          # We will be in a vendor tree, but let's additionally inspect `hold` to count its facets.
+          # There will be only a single facet for vendor-tree types like `application/vnd.wordperfect`.
+          # There will be multiple facets for vendor-tree types like `application/vnd.tcpdump.pcap`.
+          #
+          # If we have multiple facets, split the (reversed) last facet out and use it as the vendor-tree name,
+          # e.g. for `application/vnd/tcpdump.pcap` we will use `tcpdump` as the tree naame instead of `vnd`,
+          # in fact not even storing the `vnd` at all.
+          #
+          # This increases the likelihood of `hold`'s remainder fitting inside a single RValue,
+          # e.g. for yuge Types like `application/vnd.oasis.opendocument.graphics` we will store `oasis`
+          # and `opendocument.graphics` (fits!) instead of `vnd` and `oasis.opendocument.graphics` (doesn't fit!).
+          #
+          # The dropped `vnd` will be reconstructed by `CYO#to_s` when it detects a non-standard tree name.
+          hold.rindex(46) ? -hold.slice!(hold.rindex(46)..).reverse!.tap(&:pop).pack(-'U*') : -'vnd'
+        when scratch[-3..] == (-'srp').codepoints then
+          # https://datatracker.ietf.org/doc/html/rfc6838#section-3.3
+          # "Media types created experimentally or as part of products that are not distributed commercially".
+          # This is mostly an early-Internet legacy and there are only a few of these in `shared-mime-info`,
+          # e.g. `audio/prs.sid` for the C=64 Sound Interface Device audio format,
+          # but they can still be registered.
+          scratch.pop(3); -'prs'
+        when scratch[-5..] == (-'-sm-x').codepoints then
+          # Microsoft formats like `text/x-ms-regedit`.
+          # I'm treating this separately from the IETF `x-` tree just because there are so many of them,
+          # and it's nice to keep Winders formats logically-grouped.
+          scratch.pop(5); -'x-ms'
+        when scratch[-2..] == (-'-x').codepoints then
+          # Deprecated experimental tree (`x-`): https://datatracker.ietf.org/doc/html/rfc6648
+          # I'm giving this deprecated tree the canonical `x` tree in CYO because it has legacy dating back
+          # to the mid '70s and has many many many more Types than post-2012 `x.` tree,
+          # RE: https://datatracker.ietf.org/doc/html/rfc6648#appendix-A
+          scratch.pop(2); -?x
+        when scratch.one? && scratch.last == 100 then  # x
+          # Faceted experimental tree (`x.`): https://datatracker.ietf.org/doc/html/rfc6838#section-3.4
+          # There are only a few of these since "use of both `x-` and `x.` forms is discouraged",
+          # e.g. `model/x.stl-binary`, and there aren't likely to be many more.
+          scratch.pop; -'kayo-dot'
+        else
+          # Otherwise we are in the "standards" tree: https://datatracker.ietf.org/doc/html/rfc6838#section-3.1
+          -'possum'
+        end.-@
+        # Everything remaining in `hold` and `scratch` will comprise the most-specific Type component.
+        hold.push(*scratch)
+        my_base[:genus] = hold.reverse!.pack(-'U*').-@
+        scratch.clear
+        hold.clear
+      when 46 then  # .
+        hold << 46 unless hold.empty?
+        hold.push(*scratch)
+        scratch.clear
+      else
+        scratch << zig
+      end
+    }
+
+    # 𝘐𝘛'𝘚 𝘠𝘖𝘜 !!
+    cats = ->(gentlemen) {
+      gentlemen.each_codepoint.reverse_each(&move_zig)
+      move_zig.call(0)
+      return my_base.dup.freeze.tap(&the_bomb)
+    }
+    -> (gentlemen) {
+      return cats.call((gentlemen.encoding == Encoding::UTF_8) ? gentlemen : gentlemen.encode(Encoding::UTF_8))
+    }
+  }.call
+
+  # Call the above singleton Proc to do the thing.
+  def from_ietf_media_type(ietf_string)
+    return if ietf_string.nil?
+    FROM_IETF_TYPE.call(ietf_string)
+  end
+end
+
+module CHECKING::YOU::IN::INLANDGESPRÄCH
+  # Non-IETF-tree as a CY(I|O)'s `kingdom` signifies the need for a leading `vnd.` facet
+  # when reconstructing the Media-Type String.
+  IETF_TREES = [
+    # Current top-level registries are shown here: https://www.iana.org/assignments/media-types/media-types.xhtml
+    # The latest addition reflected here is `font` from RFC 8081: https://datatracker.ietf.org/doc/html/rfc8081
+    -'application',
+    -'audio',
+    -'example',
+    -'font',
+    -'image',
+    -'message',
+    -'model',
+    -'multipart',
+    -'text',
+    -'video',
+  ]
+
+  # Reconstruct an IETF Media-Type String from a loaded CYI/CYO's `#members`
+  def to_s
+    # TODO: Fragments (e.g. `;what=ever`), and syntax identifiers (e.g. `+xml`)
+    -(String.new(encoding: Encoding::UTF_8, capacity: 128) << self.phylum.to_s << -'/' << case
+    when self.kingdom == -'kayo-dot' then -'x.'
+    when self.kingdom == -?x then -'x-'
+    when self.kingdom == -'x-ms' then -'x-ms-'
+    when self.kingdom == -'prs' then -'prs.'
+    when self.kingdom == -'vnd' then -'vnd.'
+    when self.kingdom == -'possum' then nil.to_s
+    when !IETF_TREES.include?(self.kingdom.to_s) then 'vnd.' << self.kingdom.to_s << -'.'
+    else self.kingdom.to_s << -'.'
+    end << self.genus.to_s)
+  end
+
+  # Pretty-print objects using our custom `#:to_s`
+  def inspect
+    "#<#{self.class.to_s} #{self.to_s}>"
+  end
+end
+
+module CHECKING::YOU::OUT::AUSLANDSGESPRÄCH
+
+  def from_ietf_media_type(ietf_string)
+    return if ietf_string.nil?
+    self.new(super)
+  end
+
+  # CHECK OUT a filesystem path.
+  # This might be a String, or might be an instance of the actual stdlib class `Pathname`:
+  # https://ruby-doc.org/stdlib/libdoc/pathname/rdoc/Pathname.html
+  def from_xattr(pathname)
+    # T0DO: Handle relative paths and all the other corner cases that could be here when given String.
+
+    # Check the filesystem extended attributes for manually-defined types.
+    #
+    # The freedesktop-dot-org specification is `user.mime_type`:
+    # https://www.freedesktop.org/wiki/CommonExtendedAttributes/
+    #
+    # At least one other application I can find (lighttpd a.k.a. "lighty")
+    # will use `Content-Type` just like would be found in an HTTP header:
+    # https://redmine.lighttpd.net/projects/1/wiki/Mimetype_use-xattrDetails
+    #
+    # Both of these should contain IETF-style `media/sub`-type Strings,
+    # but they are technically freeform and must be assumed to contain anything.
+    # It's very very unlikely that anybody will ever use one of these at all,
+    # but hey how cool is it that we will support it if they do? :)
+    #
+    # T0DO: Figure out if NTFS has anything to offer us since `ffi-xattr` does support Winders.
+    # https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-fscc/a82e9105-2405-4e37-b2c3-28c773902d85
+    from_ietf_media_type(
+      Xattr.new(pathname).to_h.slice('user.mime_type', 'Content-Type').values.first
+    )
+  end
+
+end

data/lib/checking-you-out/ghost_revival.rb

@@ -0,0 +1,71 @@
+
+require_relative 'ghost_revival/mr_mime' unless defined? ::CHECKING::YOU::MrMIME
+require_relative 'ghost_revival/xross_infection' unless defined? ::CHECKING::YOU::OUT::XROSS_INFECTION
+
+module CHECKING::YOU::OUT::GHOST_REVIVAL
+
+  # Filename for the main fdo `shared-mime-info` source XML.
+  # We will look for this file in system `XDG_DATA_DIRS` and use our own bundled copy
+  # if the system version is missing or outdated.
+  FDO_MIMETYPES_FILENAME = -'freedesktop.org.xml'
+
+  # For now, unconditionally load all available files at startup.
+  # TODO: Support partial on-the-fly loading à la `mini_mime`.
+  def self.extended(otra)
+    # Init a handler that will be passed to multiple instances of Ox::sax_parse()
+    # for our multiple data files. I will just read them sequentially for now.
+    handler = ::CHECKING::YOU::MrMIME.new
+
+    # CYO bundles a copy of `freedesktop.org.xml` from `shared-mime-info` but will prefer a system-level copy
+    # if one is available and not out of date. This flag will be disabled if we find a suitable copy,
+    # otherwise our bundled copy will be loaded after we finish scanning the PATHs givin in our environment.
+    load_bundled_fdo_xml = true
+
+    # Search `XDG_DATA_DIRS` for any additional `shared-mime-info`-format data files we can load,
+    # hopefully including the all-important `freedesktop.org.xml`.
+    ::CHECKING::YOU::OUT::XROSS_INFECTION::XDG.DATA.push(
+      # Append out Gem-local path to the very end (lowest priority)
+      ::CHECKING::YOU::OUT::GEM_ROOT.call
+    ).map {
+      # Add path fragments for finding `shared-mime-info` package files.
+      # This same subdir path applies when searching *any* `PATH` for `shared-mime-info` XML,
+      # e.g. '/usr/share' + 'mime/packages' <-- this part
+      # For consistency the same path is used for our local data under the Gem root.
+      _1.join(-'mime', -'packages')
+    }.flat_map {
+      # Find all XML files under all subdirectories of all given `Pathname`s.
+      #
+      # `#glob` follows the same conventions as `File::fnmatch?`:
+      # https://ruby-doc.org/core-3.0.2/File.html#method-c-fnmatch
+      #
+      # `EXTGLOB` enables the brace-delimited glob syntax, used here to allow an optional `'.in'` extname
+      # as found on the `'freedesktop.org.xml.in'` bundled with our Gem since I don't want to rename
+      # the file from the XDG repo even though that extname means they don't want us to use that file directly.
+      _1.glob(File.join(-'**', -'*.xml{.in,}'), File::FNM_EXTGLOB)
+    }.each_with_object(::CHECKING::YOU::OUT::GEM_ROOT.call) { |xml_path, gem_root|
+
+      # Load the bundled `shared-mime-info` database if the system-level one exists but is out of date
+      # compared to our Gem. Using `String#include?` here since the system-level file will be
+      # `'freedesktop.org.xml'` but the bundled copy will be `'freedesktop.org.xml.in'`.
+      if xml_path.basename.to_s.include?(FDO_MIMETYPES_FILENAME)
+        # `Pathname#ascend` returns an `Enumerator` of `Pathname`s up one level at a time until reaching fs root.
+        # If *any* of these are equal to `GEM_ROOT` then we have found the bundled copy, otherwise system copy.
+        if (xml_path.ascend { break true if _1 ==  gem_root} || false) then
+          # Found bundled copy.
+          # A new-enough system-level copy will disable this flag to prevent loading outdated bundled data.
+          next unless load_bundled_fdo_xml
+        else
+          # Found system-level copy.
+          # Use this if it's newer than our Gem, and set a flag to prevent loading the bundled copy if so.
+          next if ::CHECKING::YOU::OUT::GEM_PACKAGE_TIME.call > xml_path.mtime
+          load_bundled_fdo_xml = false
+        end
+      end
+
+      # If we made it here (no `next` above) then we should load the file.
+      handler.open(xml_path)
+
+    }  # each_with_object
+
+  end  # def self.extended
+end  # module CHECKING::YOU::OUT::GHOST_REVIVAL