magika 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0fd99e8ceb2f0563b0afb70bebb2caf0b5aedcd776a05a512f47e39d86eac5b
-  data.tar.gz: b1699a79078fde7fa3c007924d2603b1bf44849fd540fd166010240219710e22
+  metadata.gz: 1242b81f1cb54d8d60b1849246ebdecfc288fad8e4521c98e2d51ceb00d6d699
+  data.tar.gz: 0aacd82439630b208f1d87c912276a3aff364b8acb3514678d5a06f55fef1cb0
 SHA512:
-  metadata.gz: 27b7cd2007f40ca98dae78691fdb1bd4360f7f194b86e7c128500b652ea093a556c26d1db50ad7fa8297a1ea6e511fefdb6dd56ae5dafe35cd2e0f685f3abbd0
-  data.tar.gz: 8d069b640a54ac91690e67a2efb4b619987c3a9804a9098d3b179cf85ffee4aa7453e0bb939337c6fd4126b77e2cb20ad553435ee4b32474796ed0740609061d
+  metadata.gz: bfabdaeb971e6564385dbde08c0b5fb7269b2ad97f0e02eebf7010ea95349997e5fadb8e9c7d828c2f69ee724b5e7e165576e67e5e798ccda696be0452b321ac
+  data.tar.gz: 716c087f89004d48096592fc27c6120ee14ed0a9fcb1b78fe3561e4acaf1e7f861544de5a3c9fcb9f7e46bc779e4643ee4e55dc9b76f30c2aab95837331d446f

data/CHANGELOG.md

@@ -1,4 +1,12 @@
-0.1.0
+0.0.2
+=====
+
+* Pass Numo::NArray to OnnxRuntiem
+* Fix features type
+* Rename method: Magika#identify -> Magika#identify_path
+* Add Magika#identify_io
+
+0.0.1
 =====
 
 * Initial release

data/README.md

@@ -12,7 +12,10 @@ SYNOPSIS
 require "magika"
 
 magika = Magika.new
-magika.identify("path/to/file") # => #<Magika::FileType label=onnx score=0.9998390674591064 application/octet-stream "Open Neural Network Exchange" (archive)>,
+magika.identify_path("path/to/file") # => #<Magika::FileType label=onnx score=0.9998390674591064 application/octet-stream "Open Neural Network Exchange" (archive)>,
+
+require "striongio"
+magika.identify_io(StringIO.new("some text content")) # => #<Magika::FileType label=txt score=0.8081967830657959 text/plain "Generic text document" (text)>
 ```
 
 INSTALLATION

data/lib/magika.rb

@@ -11,8 +11,7 @@ class Magika
 
   MODEL_CONFIG = JSON.load_file(MODEL_CONFIG_PATH, symbolize_names: true)
 
-  TARGET_LABELS_SPACE = MODEL_CONFIG[:target_labels_space]
-  TARGET_LABELS_SPACE.collect!(&:to_sym)
+  TARGET_LABELS_SPACE = Numo::RObject[MODEL_CONFIG[:target_labels_space].collect!(&:to_sym)]
   MIN_FILE_SIZE_FOR_DL = MODEL_CONFIG[:min_file_size_for_dl]
   VERSION_MAJOR = MODEL_CONFIG[:version_major]
 
@@ -20,13 +19,23 @@ class Magika
     @session = OnnxRuntime::InferenceSession.new(MODEL_PATH)
   end
 
-  def identify(path)
-    features, label = extract_features_or_label(path)
+  def identify_path(path)
+    stat = File.lstat(path)
+    if stat.directory?
+      return FileType.new(:directory, 1.0)
+    end
+    if stat.symlink?
+      return FileType.new(:symlink, 1.0)
+    end
+    File.open(path, "rb", encoding: Encoding::BINARY) {|file|
+      identify_io(file)
+    }
+  end
+
+  def identify_io(io)
+    features, label = extract_features_or_label(io)
     if features
-      target_label = @session.run([:target_label], {bytes: features.reshape(1, FEATURES_SIZE)}, output_type: :numo)[0]
-      index = target_label.argmax
-      score = target_label[index]
-      FileType.new(TARGET_LABELS_SPACE[index], score)
+      FileType.from_inference(infer_from_features(features))
     else
       FileType.new(label, 1.0)
     end
@@ -41,75 +50,68 @@ class Magika
   BLOCK_SIZE = MODEL_CONFIG[:block_size]
   PADDING_TOKEN = MODEL_CONFIG[:padding_token]
 
-  def extract_features_or_label(path)
-    stat = File.lstat(path)
-    if stat.directory?
-      return nil, :directory
-    end
-    if stat.symlink?
-      return nil, :symlink
-    end
-    file_size = stat.size
-    if file_size == 0
+  def extract_features_or_label(io)
+    io_size = io.size
+    if io_size == 0
       return nil, :empty
     end
-    if file_size < MIN_FILE_SIZE_FOR_DL
-      return small_content_result(path)
+    if io_size < MIN_FILE_SIZE_FOR_DL
+      return nil, detect_small_content_label(io.read)
     end
-    features = extract_features(path)
+    features = extract_features_from_io(io)
     if features[MIN_FILE_SIZE_FOR_DL - 1] == PADDING_TOKEN
-      return small_content_result(path)
+      return nil, detect_small_content_label(features[0...io_size].to_binary)
     else
       return features, nil
     end
   end
 
-  def small_content_result(path)
-    content = File.read(path, encoding: Encoding::BINARY)
-    begin
-      content.encode! Encoding::UTF_8
-      return nil, :txt
-    rescue EncodingError
-      return nil, :unknown
+  def detect_small_content_label(content)
+    if content.force_encoding(Encoding::UTF_8).valid_encoding?
+      return :txt
+    else
+      return :unknown
     end
   end
 
-  def extract_features(path)
+  def extract_features_from_io(io)
     features = Numo::Int32.new(FEATURES_SIZE).fill(PADDING_TOKEN)
-    File.open(path, "rb", encoding: Encoding::BINARY) do |file|
-      file_size = file.size
-      return features if file_size == 0
-
-      block_size = [BLOCK_SIZE, file_size].min
-
-      beg_block = file.read(block_size)
-      # String#lstrip strips null characters, which reduces accuracy
-      beg_str = beg_block.sub(/\A\s+/, "").freeze
-      unless beg_str.empty?
-        beg_content = Numo::Int8.from_binary(beg_str)
-        beg_size = [BEG_SIZE, beg_content.size].min
-        features[0...beg_size] = beg_content[0...beg_size]
-      end
+    io_size = io.size
+    return features if io_size == 0
+
+    block_size = [BLOCK_SIZE, io_size].min
+
+    beg_block = io.read(block_size)
+    # String#lstrip strips null characters, which reduces accuracy
+    beg_str = beg_block.sub(/\A\s+/, "").freeze
+    unless beg_str.empty?
+      beg_content = Numo::UInt8.from_binary(beg_str)
+      beg_size = [BEG_SIZE, beg_content.size].min
+      features[0...beg_size] = beg_content[0...beg_size]
+    end
 
-      # Skip middle feature
-
-      end_block = if block_size == file_size
-                    beg_block
-                  else
-                    file.seek(-block_size, IO::SEEK_END)
-                    file.read(block_size)
-                  end
-      # String#rstrip strips null characters, which reduces accuracy
-      end_str = end_block.reverse.sub(/\A\s+/, "").reverse.freeze
-      unless end_str.empty?
-        end_content = Numo::Int8.from_binary(end_str)
-        end_size = [END_SIZE, end_content.size].min
-        features[-end_size..] = end_content[-end_size..]
-      end
+    # Skip middle feature
+
+    end_block = if block_size == io_size
+                  beg_block
+                else
+                  io.seek(-block_size, IO::SEEK_END)
+                  io.read(block_size)
+                end
+    # String#rstrip strips null characters, which reduces accuracy
+    end_str = end_block.reverse.sub(/\A\s+/, "").reverse.freeze
+    unless end_str.empty?
+      end_content = Numo::UInt8.from_binary(end_str)
+      end_size = [END_SIZE, end_content.size].min
+      features[-end_size..] = end_content[-end_size..]
     end
     features
   end
 
+  def infer_from_features(features)
+    @session.run([:target_label], {bytes: OnnxRuntime::OrtValue.from_numo(features.reshape(1, FEATURES_SIZE))}, output_type: :numo)[0]
+  end
+
   class FileType
     CONTENT_TYPES_PATH = File.join(ASSETS_DIR, "content_types_kb.min.json")
     CONTENT_TYPES = JSON.load_file(CONTENT_TYPES_PATH, symbolize_names: true)
@@ -118,6 +120,14 @@ class Magika
 
     attr_reader :score
 
+    class << self
+      def from_inference(inference)
+      index = inference.argmax
+      score = inference[index]
+      FileType.new(TARGET_LABELS_SPACE[index], score)
+      end
+    end
+
     def initialize(label, score)
       @inferred_label = label
       @score = score

data/magika.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
   s.name = "magika"
-  s.version = "0.0.1"
+  s.version = "0.0.2"
   s.authors = ["Kitaiti Makoto"]
   s.email = ["KitaitiMakoto@gmail.com"]
   s.summary = "Determines file content types using AI"
-  s.description = "Determines file content types using original Msgika's ONNX model"
+  s.description = "Determines file content types using original Magika's ONNX model"
   s.license = "Apache-2.0"
   s.homepage = "https://gitlab.com/KitaitiMakoto/magika-rb"
 
@@ -25,4 +25,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "test-unit"
   s.add_development_dependency "test-unit-notify"
   s.add_development_dependency "terminal-notifier" if RUBY_PLATFORM.match?(/darwin/)
+  s.add_development_dependency "simplecov"
 end

data/test/fixtures/padded-long-whitespace.txt

@@ -0,0 +1 @@
+                                       hello

data/test/helper.rb

@@ -1,3 +1,8 @@
+require "simplecov"
+SimpleCov.start do
+  add_filter /test|vendor/
+end
+
 require "test/unit"
 require "test/unit/notify"
 require "terminal-notifier" if RUBY_PLATFORM.match?(/darwin/)

data/test/test_magika.rb

@@ -1,4 +1,5 @@
 require_relative "helper"
+require "stringio"
 
 class TestMagika < TestBase
   def test_initialize
@@ -6,16 +7,28 @@ class TestMagika < TestBase
     assert_instance_of Magika, magika
   end
 
-
   data("empty file", [fixture_file_path("empty"), :empty])
   data("small text file", [fixture_file_path("hello.txt"), :txt])
+  data("padded long whitespace text file", [fixture_file_path("padded-long-whitespace.txt"), :txt])
   data("small binary fall", [fixture_file_path("small.bin"), :unknown])
   data("ONNX model file", [Magika::MODEL_PATH, :onnx])
   data("Ruby file", [__FILE__, :ruby])
   data("directory", [__dir__, :directory])
   data("symlink", [fixture_file_path("symlink"), :symlink])
-  def test_identify(data)
+  def test_identify_path(data)
+    path, label = data
+    assert_equal label, magika.identify_path(path).label
+  end
+
+  data("empty file", [fixture_file_path("empty"), :empty])
+  data("small text file", [fixture_file_path("hello.txt"), :txt])
+  data("small binary fall", [fixture_file_path("small.bin"), :unknown])
+  data("ONNX model file", [Magika::MODEL_PATH, :onnx])
+  def test_identify_io(data)
     path, label = data
-    assert_equal label, magika.identify(path).label
+    File.open(path, "rb", encoding: Encoding::BINARY) do |io|
+      assert_equal label, magika.identify_io(io).label
+    end
+    assert_equal label, magika.identify_io(StringIO.new(File.binread(path))).label
   end
 end

data/test/test_package.rb

@@ -28,7 +28,7 @@ class TestPackage < Test::Unit::TestCase
 
       lib_dir = File.join(gem_dir, "lib")
 
-      assert_equal "onnx", `ruby -I #{lib_dir.shellescape} -r magika -e 'print Magika.new.identify("#{model_path.shellescape}").label'`
+      assert_equal "onnx", `ruby -I #{lib_dir.shellescape} -r magika -e 'print Magika.new.identify_path("#{model_path.shellescape}").label'`
     end
   end
 end

data/test/test_smoke.rb

@@ -5,16 +5,16 @@ class TestSmoke < TestBase
     data do
       JSON.load(`magika --recursive --json vendor/bundle`).collect {|result|
         path = result["path"]
-        label = result["result"]["value"]["output"]["label"]
+        label = result["result"]["value"]["output"]["label"].to_sym
         score = result["result"]["value"]["score"]
         [path, [path, label, score]]
       }.to_h
     end
     def test_smoke(data)
       path, label, score = data
-      result = magika.identify(path)
-      assert_equal label.to_sym, result.label
-      # assert_in_delta score, result.score
+      result = magika.identify_path(path)
+      assert_equal label, result.label
+      assert_in_delta score, result.score
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: magika
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Kitaiti Makoto
@@ -107,7 +107,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Determines file content types using original Msgika's ONNX model
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Determines file content types using original Magika's ONNX model
 email:
 - KitaitiMakoto@gmail.com
 executables: []
@@ -126,6 +140,7 @@ files:
 - magika.gemspec
 - test/fixtures/empty
 - test/fixtures/hello.txt
+- test/fixtures/padded-long-whitespace.txt
 - test/fixtures/small.bin
 - test/fixtures/symlink
 - test/helper.rb