libmagic 0.5.11

data/test/files/us-ascii.txt

@@ -0,0 +1,3 @@
+Just some plain old ASCII text.
+
+And some more.

data/test/files/utf-8.txt

@@ -0,0 +1,3 @@
+Lots of unicode content in sequence to check that we don't accidentally cut multibyte characters: 漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字漢字
+Some unicode content like this: µM
+And more just for fun: 漢字

data/test/files/windows-1252.txt

@@ -0,0 +1,2 @@
+MoleculeICKdComment
+ABC-�Archive Mine Collaborate� is our trademark� said Barry on pages �

data/test/test_magic.rb

@@ -0,0 +1,213 @@
+# coding: utf-8
+require "rubygems"
+require "lib/libmagic"
+require "test/unit"
+require "stringio"
+
+class MagicTest < Test::Unit::TestCase
+  def test_public_interface_is_limited
+    assert_equal(%w(file_charset file_charset! file_mime_type io_charset string_charset string_mime_type).map { |m| m.to_sym },
+                 (Magic.public_methods - Magic.instance_methods - FFI::Library.methods - FFI::Library.instance_methods).sort.map { |m| m.to_sym })
+  end
+
+  def test_file_mime_type_for_utf8_file
+    # regex necessary because some versions of file return the semicolon and some don't
+    assert(Magic.file_mime_type(absolute_path("utf-8.txt")) =~ /text\/plain;? charset=utf-8/)
+  end
+
+  def test_string_mime_type_for_utf8_text
+    # regex necessary because some versions of file return the semicolon and some don't
+    assert(Magic.string_mime_type("Some truly Unicode characters like: 불거기") =~ /text\/plain;? charset=utf-8/)
+  end
+
+  def test_string_charset_for_ascii_text
+    assert_equal("us-ascii", Magic.string_charset("Just ASCII"))
+  end
+
+  def test_string_charset_for_utf8_text
+    assert_equal("utf-8", Magic.string_charset("Some truly Unicode characters like: 불거기"))
+  end
+
+  def test_string_charset_for_iso_8859_1_text
+    assert_equal("iso-8859-1", Magic.string_charset("A\240B\240C"))
+  end
+
+  (128..159).each do |windows_char|
+    eval <<-EOMETHOD
+      def test_string_charset_for_string_with_windows_char_#{windows_char}_returns_unknown
+        assert_equal("unknown", Magic.string_charset("over, and over, and over\\#{windows_char.to_s(8)}"))
+      end
+    EOMETHOD
+  end
+
+  # although this is redundant, it's nice to document the one weird case we've found explicitly
+  def test_string_charset_for_string_with_windows_ellipsis_returns_unknown
+    # windows 1252 ellipsis is 133 = 0205
+    # this is a weird case found from a file Sylvia had problem slurping
+    assert_equal("unknown", Magic.string_charset("over, and over, and over\205"))
+  end
+
+  def test_string_charset_for_string_with_utf8_Angstrom_returns_utf8_not_unknown
+    string = File.open(absolute_path("utf-8.csv.gz")) do |io|
+      uncompressed_io = Zlib::GzipReader.new(io)
+      uncompressed_io.read
+    end
+    assert_equal("utf-8", Magic.string_charset(string))
+  end
+
+  def test_io_charset_for_ascii_file
+    assert_equal("us-ascii", Magic.io_charset(File.open((absolute_path("us-ascii.txt")))))
+  end
+
+  def test_io_charset_for_utf8_file
+    assert_equal("utf-8", Magic.io_charset(File.open(absolute_path("utf-8.txt"))))
+  end
+
+  require "zlib"
+  def test_io_charset_for_gzipped_utf8_file
+    File.open(absolute_path("utf-8.csv.gz")) do |io|
+      uncompressed_io = Zlib::GzipReader.new(io)
+      assert_equal("utf-8", Magic.io_charset(uncompressed_io))
+    end
+  end
+
+  def test_io_charset_for_iso_8859_1_file
+    assert_equal("iso-8859-1", Magic.io_charset(File.open(absolute_path("iso-8859-1.txt"))))
+  end
+
+  def test_io_charset_for_windows_1252_file
+    # unfortunately, unknown means some kind of extended ascii
+    assert_equal("unknown", Magic.io_charset(File.open(absolute_path("windows-1252.txt"))))
+  end
+
+  def test_io_charset_for_macintosh_file
+    # unfortunately, unknown means some kind of extended ascii
+    assert_equal("unknown", Magic.io_charset(File.open(absolute_path("macintosh.txt"))))
+  end
+
+  def test_io_charset_for_csv_file_that_looked_like_ppm_image
+    assert_equal("us-ascii", Magic.io_charset(File.open(absolute_path("file_with_text_that_looked_like_ppm_image.csv"))))
+  end
+
+  def test_file_charset_for_ascii_file
+    assert_equal("us-ascii", Magic.file_charset(absolute_path("us-ascii.txt")))
+  end
+
+  def test_file_charset_for_large_CSV_file_that_libmagic_thinks_is_pascal_sourcecode
+    assert_equal("us-ascii", Magic.file_charset(absolute_path("part_of_ki_file.csv")))
+  end
+
+  def test_file_charset_for_utf8_file
+    assert_equal("utf-8", Magic.file_charset(absolute_path("utf-8.txt")))
+  end
+
+  def test_file_charset_for_iso_8859_1_file
+    assert_equal("iso-8859-1", Magic.file_charset(absolute_path("iso-8859-1.txt")))
+  end
+
+  def test_file_charset_for_windows_1252_file
+    # unfortunately, unknown means some kind of extended ascii
+    assert_equal("unknown", Magic.file_charset(absolute_path("windows-1252.txt")))
+  end
+
+  def test_file_charset_for_macintosh_file
+    # unfortunately, unknown means some kind of extended ascii
+    assert_equal("unknown", Magic.file_charset(absolute_path("macintosh.txt")))
+  end
+
+  def test_file_charset_for_csv_file_that_looked_like_ppm_image
+    assert_equal("us-ascii", Magic.file_charset(absolute_path("file_with_text_that_looked_like_ppm_image.csv")))
+  end
+
+  def test_file_charset_raises_if_file_does_not_exist
+    # for this, we don't use assert_raise
+    begin
+      Magic.file_charset("some file that does not exist.txt")
+      fail "Did not raise"
+    rescue Exception => expected
+      # ruby 1.9 and 1.8 return different exceptions
+      assert(expected.message =~ /(some file that does not exist.txt|NULL pointer)/i)
+    end
+  end
+
+  def test_file_charset_bang_exhaustively_checks_file_contents
+    t1 = Time.now
+    assert_equal("iso-8859-1", Magic.file_charset!(absolute_path("huge_file_with_one_special_character.csv")))
+    puts "took #{Time.now - t1} seconds"
+  end
+
+  def test_file_charset_bang_returns_correct_value_for_us_ascii_file
+    assert_equal("us-ascii", Magic.file_charset!(absolute_path("us-ascii.txt")))
+  end
+
+  def test_file_charset_bang_returns_correct_value_for_windows_1252_file
+    assert_equal("unknown", Magic.file_charset!(absolute_path("windows-1252.txt")))
+  end
+
+  def test_file_charset_bang_returns_correct_value_for_UTF8_file
+    assert_equal("utf-8", Magic.file_charset!(absolute_path("utf-8.txt")))
+  end
+
+  def test_file_charset_bang_handles_special_character_at_the_end_of_the_file
+    assert_equal("us-ascii", Magic.file_charset(absolute_path("huge_file_with_one_special_character_at_the_end.csv")))
+    assert_equal("iso-8859-1", Magic.file_charset!(absolute_path("huge_file_with_one_special_character_at_the_end.csv")))
+  end
+
+  def test_collect_special_characters_is_empty_when_there_are_no_special_characters
+    assert_special_chars_equal("", "")
+    assert_special_chars_equal("", "hello")
+    assert_special_chars_equal("", "12345678901234567890")
+  end
+
+  def test_collect_special_characters_returns_characters_with_context
+    assert_special_chars_equal("µ", "µ")
+    assert_special_chars_equal("321µ123", "321µ123")
+    assert_special_chars_equal("µ123", "µ123")
+    assert_special_chars_equal("321µ", "321µ")
+    assert_special_chars_equal("0987654321\xC21234567890", "0987654321\xC21234567890")
+    assert_special_chars_equal("0987654321µ1234567890", "XXX0987654321µ1234567890XXX")
+    assert_special_chars_equal("µ1234567890", "µ1234567890XXX")
+    assert_special_chars_equal("0987654321µ", "XXX0987654321µ")
+  end
+
+  def test_collect_special_characters_does_not_duplicate_context
+    assert_special_chars_equal("0987654321µaaaaaµ1234567890", "XXX0987654321µaaaaaµ1234567890XXX")
+  end
+
+  def test_collect_special_characters_works_with_multiple_characters
+    assert_special_chars_equal(
+      "0987654321µaaaaaµ12345678900987654321µ1234567890",
+      "XXX0987654321µaaaaaµ1234567890XXXXXX0987654321µ1234567890XXX"
+    )
+  end
+
+  def test_collect_special_characters_works_when_reading_multiple_chunks_to_the_buffer
+    default_chunk_size = 2 ** 15
+    assert_equal(default_chunk_size, Magic::CHUNK_SIZE)
+    begin
+      Magic.send(:remove_const, "CHUNK_SIZE")
+      Magic.send(:const_set, "CHUNK_SIZE", 2)
+      assert_special_chars_equal("µ", "µ")
+      assert_special_chars_equal("µ123", "µ123")
+      assert_special_chars_equal("321µ", "321µ")
+      assert_special_chars_equal("321µ123", "321µ123")
+      assert_special_chars_equal("0987654321µ", "XXX0987654321µ")
+      assert_special_chars_equal(
+        "0987654321µaaaaaµ12345678900987654321µ1234567890",
+        "XXX0987654321µaaaaaµ1234567890XXXXXX0987654321µ1234567890XXX"
+      )
+    ensure
+      Magic.send(:remove_const, "CHUNK_SIZE") if Magic.const_defined?("CHUNK_SIZE")
+      Magic.const_set("CHUNK_SIZE", default_chunk_size)
+    end
+    assert_equal(default_chunk_size, Magic::CHUNK_SIZE)
+  end
+
+  def assert_special_chars_equal(expected_output, input)
+    assert_equal(expected_output.force_encoding(Encoding::BINARY), Magic.send(:collect_special_characters, StringIO.new(input)))
+  end
+
+  def absolute_path(test_file_name)
+    "#{ENV["PWD"]}/test/files/#{test_file_name}"
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: libmagic
+version: !ruby/object:Gem::Version
+  version: 0.5.11
+platform: ruby
+authors:
+- Moses Hohman
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-06-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+description: Ruby wrapper for the Unix file/libmagic utility, which can guess mime
+  types and character sets.
+email:
+- moses@moseshohman.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- History.txt
+- Manifest.txt
+- README.txt
+files:
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/custom-magic
+- lib/custom-magic.mime
+- lib/libmagic.rb
+- libmagic.gemspec
+- test/files/file_with_text_that_looked_like_ppm_image.csv
+- test/files/huge_file_with_one_special_character.csv
+- test/files/huge_file_with_one_special_character_at_the_end.csv
+- test/files/iso-8859-1.txt
+- test/files/macintosh.txt
+- test/files/part_of_ki_file.csv
+- test/files/us-ascii.txt
+- test/files/utf-8.txt
+- test/files/windows-1252.txt
+- test/test_magic.rb
+homepage: https://github.com/cdd/libmagic
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/cdd/libmagic
+post_install_message:
+rdoc_options:
+- "--main"
+- README.txt
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: Ruby wrapper for the Unix file/libmagic utility, which can guess mime types
+  and character sets.
+test_files: []