format_parser 0.25.2 → 0.25.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 290b2369d2fe089202a76ab9d7b659f8de58fa44b9a40718130105ae7026036a
-  data.tar.gz: 53e983a8639cc42377ab50d7364b5099cf1f3308f8108f92b6194040546ea2e8
+  metadata.gz: 2d0b2c07289221019c42f9546eee65b4ccd5c49aadc3c16f7e4192f356821bcb
+  data.tar.gz: 7163ca3bfac79fe5539979e5723902db8fe530ee721d09ba3b18cf635280ece6
 SHA512:
-  metadata.gz: 5f58620ab165b1c47a8a18fe82081d48cb0b285b4f9638146c31d1bb2f839247b36750a7f85be5a9ea14b8db3fc2ca175b86ab4f8b29f87bd1ea10caed57746c
-  data.tar.gz: b9e87723a7cc1d5ecf04a23c28cbfc433c120337275aa51d76396a9a1371bb1b683b22c18ab4e84d8870a76048e26f7c5d7f90b4b5f9b868bf5a0dcb6771640c
+  metadata.gz: dfb72a909878a032d6f832aa0a52bd0521df31faacff2f5d070e9b4f555d420ad11abecfc30e2074f7897250b530543a4892b60d37037f67b87dd8c9b10c2b8b
+  data.tar.gz: 24b69b8ad67b5d4461f63055b379af270a936172660d6350cc98ee2caa2c47ff3b7d94bf353de673ba331ba273b1e86090f83ef282933f261978250562aa22cd

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.25.3
+* MP3: Fix parser to not skip the first bytes if it's not an ID3 header
+
+## 0.25.2
+* Hotfix Moov parser
+
 ## 0.25.1
 * MOV: Fix error "negative length"
 * MOV: Fix reading dimensions in multi-track files

data/CONTRIBUTING.md

@@ -83,32 +83,59 @@ of software. Ideally, this file is going to be something you have produced yours
 and you are permitted to share under the MIT license provisions.
 
 When writing a parser, please try to ensure it returns a usable result as soon as possible,
-or no result as soon as possible (once you know the file is not fit for your specific parser).
+or `nil` as soon as possible (once you know the file is not fit for your specific parser).
 Bear in mind that we enforce read budgets per-parser, so you will not be allowed to perform
 too many reads, or perform reads which are too large.
 
-In order to create new parsers, it is recommended to make a well-named class with an instance method `call`.
+In order to create new parsers, make a well-named class with an instance method `call`,
+and to register a single instance of that class as the parser - so that only one object needs to be stored
+in memory when parsing multiple inputs. In that case your object must be **thread-safe and stateless** - this
+is really important since FormatParser is thread-safe and multiple parsing procedures may be in progress
+concurrently against the same parser object. You can also create a Proc if your parser is fairly trivial.
 
-`call` accepts the IO-ish object as an argument, parses data that it reads from it,
-and then returns the metadata for the file (if it could recover any) or `nil` if it couldn't. All files pass
-through all parsers by default, so if you are dealing with a file that is not "your" format - return `nil` from
-your method or `break` your Proc as early as possible. A blank `return` works fine too.
+If it will be difficult to have your parser thread-safe you can register your class itself as
+the parser and define the `self.call` method to parse using a fresh instance every time, allowing
+object-level state:
+
+```ruby
+class MyParser
+  def self.call(io)
+    new.call(io)
+  end
+
+  def call(io)
+    @state = ...
+  end
+```
+
+`call` accepts a single argument - an IO-ish object which is guaranteed to respond to the same methods as the
+ones defined in `IOConstraint` - that is, it is a strict subset of a standard Ruby IO object. *All reads from
+this IO object are guaranteed to be returned in binary encoding.* The IO will be at offset of 0 when your parsing
+proc receives it and there will be no concurrent calls to that object until your proc returns.
 
-The IO will at the minimum support the subset of the IO API defined in `IOConstraint`
+Your parsing procedure may read from this IO object, and should return either a `Result`-like object with
+the file metadata (if it could recover any) or `nil` if it couldn't. All files pass
+through all parsers by default, so if you are dealing with a file that is not "your" format - return `nil` from
+your method or `break` your Proc as early as possible. A blank `return` works fine too as it actually returns `nil`.
 
-Your parser has to be registered using `FormatParser.register_parser` with the information on the formats
-and file natures it provides.
+Your parser then needs to be registered using `FormatParser.register_parser` with the information on the formats
+and file natures it provides. This allows FormatParser to skip your parser if, say, the user only want to parse for
+`:image` nature files but your parser parses `:audio`.
 
-Down below you can find the most basic parser implementation:
+Down below you can find the most basic parser implementation which parses an imaginary `IMGA` file format:
 
 ```ruby
 MyParser = ->(io) {
-  # ... do some parsing with `io`
+  # ... Read the magic bytes from the start of IO - the IO is
+  # guaranteed to be fed to you at offset 0, start-of-file.
   magic_bytes = io.read(4)
+
   # breaking the block returns `nil` to the caller signaling "no match"
   break if magic_bytes != 'IMGA'
 
+  # Our file format stores the width and height as 2 32-bit unsigned integers
   parsed_witdh, parsed_height = io.read(8).unpack('VV')
+
   # ...and return the FileInformation::Image object with the metadata.
   FormatParser::Image.new(
     format: :imga,
@@ -135,8 +162,8 @@ class MyParser
     # ... do some parsing with `io`
     # The instance will be discarded after parsing, so using instance variables
     # is permitted - they are not shared between calls to `call`
-    @magic_bytes = io.read(4)
-    break if @magic_bytes != 'IMGA'
+    magic_bytes = io.read(4)
+    break if magic_bytes != 'IMGA'
     parsed_witdh, parsed_height = io.read(8).unpack('VV')
     FormatParser::Image.new(
       format: :imga,
@@ -145,23 +172,33 @@ class MyParser
     )
   end
 
-  FormatParser.register_parser self, natures: :image, formats: :bmp
+  # Note that we register an instance of the class, not the class. It is the
+  # instance that responds to `call()` and we can do this because our object
+  # is stateless.
+  FormatParser.register_parser new, natures: :image, formats: :bmp
 end
 ```
 
-### Calling convention for preparing parsers
+If your parser supports file types which have a known filename extension, you can add a method to it called `likely_match?`,
+add this method on the object you register itself. For example, for the ZIP parser we use:
+
+```ruby
+  def likely_match?(filename)
+    filename =~ /\.(zip|docx|keynote|numbers|pptx|xlsx)$/i
+  end
+```
 
-A parser that gets registered using `register_parser` must be either:
+If your parser matches the filename it is going to be applied *earlier*, saving time. Since most FormatParser users are
+likely to only want the first result of the parsing, the sooner your parser gets applied - the sooner you can return the result,
+avoiding unnecessary reads.
 
-1) An object that can be `call()`-ed itself, with an argument that conforms to `IOConstraint`
-2) An object that responds to `new` and returns something that can be `call()`-ed with with an argument that conforms to `IOConstraint`.
+### Calling convention for preparing parsers
 
-The second opton is recommended  for most cases.
+A parser that gets registered using `register_parser` must be an object that can be `call()`-ed, with an argument that conforms to `IOConstraint`
 
 FormatParser is made to be used in threaded environments, and if you use instance variables
-you need your parser to be isolated from it's siblings in other threads - therefore you can pass
-a Class on registration to have your parser instantiated for each `call()`, anew.
-
+you need your parser to be isolated from it's siblings in other threads - create a copy for one-off use inside
+your `call` method.
 
 ## Pull requests
 

data/lib/format_parser.rb

@@ -36,7 +36,7 @@ module FormatParser
   # Register a parser object to be used to perform file format detection. Each parser FormatParser
   # provides out of the box registers itself using this method.
   #
-  # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
+  # @param callable_parser[#call] an object that responds to #call for parsing an IO
   # @param formats[Array<Symbol>] file formats that the parser provides
   # @param natures[Array<Symbol>] file natures that the parser provides
   # @param priority[Integer] whether the parser has to be applied first or later. Parsers that offer the safest
@@ -45,39 +45,39 @@ module FormatParser
   #   with a lower priority value will be applied first, and if a single result is requested, will also return
   #   first.
   # @return void
-  def self.register_parser(callable_or_responding_to_new, formats:, natures:, priority: LEAST_PRIORITY)
+  def self.register_parser(callable_parser, formats:, natures:, priority: LEAST_PRIORITY)
     parser_provided_formats = Array(formats)
     parser_provided_natures = Array(natures)
     PARSER_MUX.synchronize do
       @parsers ||= Set.new
-      @parsers << callable_or_responding_to_new
+      @parsers << callable_parser
       @parsers_per_nature ||= {}
       parser_provided_natures.each do |provided_nature|
         @parsers_per_nature[provided_nature] ||= Set.new
-        @parsers_per_nature[provided_nature] << callable_or_responding_to_new
+        @parsers_per_nature[provided_nature] << callable_parser
       end
       @parsers_per_format ||= {}
       parser_provided_formats.each do |provided_format|
         @parsers_per_format[provided_format] ||= Set.new
-        @parsers_per_format[provided_format] << callable_or_responding_to_new
+        @parsers_per_format[provided_format] << callable_parser
       end
       @parser_priorities ||= {}
-      @parser_priorities[callable_or_responding_to_new] = priority
+      @parser_priorities[callable_parser] = priority
     end
   end
 
   # Deregister a parser object (makes FormatParser forget this parser existed). Is mostly used in
   # tests, but can also be used to forcibly disable some formats completely.
   #
-  # @param callable_or_responding_to_new[#call, #new] an object that either responds to #new or to #call
+  # @param callable_parser[#==] an object that is identity-equal to any other registered parser
   # @return void
-  def self.deregister_parser(callable_or_responding_to_new)
+  def self.deregister_parser(callable_parser)
     # Used only in tests
     PARSER_MUX.synchronize do
-      (@parsers || []).delete(callable_or_responding_to_new)
-      (@parsers_per_nature || {}).values.map { |e| e.delete(callable_or_responding_to_new) }
-      (@parsers_per_format || {}).values.map { |e| e.delete(callable_or_responding_to_new) }
-      (@parser_priorities || {}).delete(callable_or_responding_to_new)
+      (@parsers || []).delete(callable_parser)
+      (@parsers_per_nature || {}).values.map { |e| e.delete(callable_parser) }
+      (@parsers_per_format || {}).values.map { |e| e.delete(callable_parser) }
+      (@parser_priorities || {}).delete(callable_parser)
     end
   end
 

data/lib/format_parser/version.rb

@@ -1,3 +1,3 @@
 module FormatParser
-  VERSION = '0.25.2'
+  VERSION = '0.25.3'
 end

data/lib/parsers/mp3_parser.rb

@@ -73,6 +73,8 @@ class FormatParser::MP3Parser
     id3v1 = ID3Extraction.attempt_id3_v1_extraction(io)
     tags = [id3v1, ID3Extraction.attempt_id3_v2_extraction(io)].compact
 
+    io.seek(0) if tags.empty?
+
     # Compute how many bytes are occupied by the actual MPEG frames
     ignore_bytes_at_tail = id3v1 ? 128 : 0
     ignore_bytes_at_head = io.pos

data/spec/parsers/mp3_parser_spec.rb

@@ -166,6 +166,18 @@ describe FormatParser::MP3Parser do
     expect(parsed.artist). to eq('wetransfer')
   end
 
+  it 'does not skip the first bytes if it is not a id3 tag header' do
+    fpath = fixtures_dir + '/MP3/no_id3_tags.mp3'
+
+    parsed = subject.call(File.open(fpath, 'rb'))
+
+    expect(parsed).not_to be_nil
+
+    expect(parsed.nature).to eq(:audio)
+    expect(parsed.format).to eq(:mp3)
+    expect(parsed.audio_sample_rate_hz).to eq(44100)
+  end
+
   describe '#as_json' do
     it 'converts all hash keys to string when stringify_keys: true' do
       fpath = fixtures_dir + '/MP3/Cassy.mp3'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: format_parser
 version: !ruby/object:Gem::Version
-  version: 0.25.2
+  version: 0.25.3
 platform: ruby
 authors:
 - Noah Berman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2020-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ks