format_parser 0.10.0 → 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +5 -2
- data/exe/format_parser_inspect +1 -1
- data/lib/format_parser.rb +26 -12
- data/lib/format_parser/version.rb +1 -1
- data/lib/image.rb +20 -0
- data/lib/parsers/dpx_parser.rb +39 -125
- data/lib/parsers/dpx_parser/dpx_structs.rb +229 -0
- data/lib/parsers/exif_parser.rb +41 -39
- data/lib/parsers/jpeg_parser.rb +10 -26
- data/lib/parsers/tiff_parser.rb +12 -7
- data/lib/read_limits_config.rb +6 -0
- data/spec/attributes_json_spec.rb +15 -0
- data/spec/format_parser_spec.rb +70 -54
- data/spec/parsers/dpx_parser_spec.rb +33 -22
- data/spec/parsers/exif_parser_spec.rb +8 -6
- data/spec/parsers/jpeg_parser_spec.rb +13 -1
- data/spec/parsers/tiff_parser_spec.rb +1 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ab01944664740856d704875f0a74d1b0540379c8e351176e38d3b5bf119e813f
|
4
|
+
data.tar.gz: b0736ffd074eb3fb49586d52799dd687c72f0a09b6e2d4109b1cbd26a8eac293
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 570e9fcef6a08ad4e800c84d8452d985900f8442d9071477e6cd465b533677f7c39b6ae51c8f94d0ab0e90305ce790c395c3ce54ecda46800921b3311adc23b8
|
7
|
+
data.tar.gz: 0662cc268f0fc1fc61ac97896811e505f1ae251a2a15fe9027de2414107adb1414b6ce4f3a1111be1215e8e48323330ce8b08b98e521bf7d9ec4a26c8d29f01d
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.11.0
|
2
|
+
* Add `Image#display_width_px` and `Image#display_height_px` for EXIF/aspect corrected display dimensions, and provide
|
3
|
+
those values from a few parsers already. Also make full EXIF data available for JPEG/TIFF in `intrinsics[:exif]`
|
4
|
+
* Adds `limits_config` option to `FormatParser.parse()` for tweaking buffers and read limits externally
|
5
|
+
|
1
6
|
## 0.10.0
|
2
7
|
* Adds the `format_parser_inspect` binary for parsing a file from the commandline
|
3
8
|
and returning results in JSON
|
data/README.md
CHANGED
@@ -41,8 +41,8 @@ Pass an IO object that responds to `read` and `seek` to `FormatParser` and the f
|
|
41
41
|
match = FormatParser.parse(File.open("myimage.jpg", "rb"))
|
42
42
|
match.nature #=> :image
|
43
43
|
match.format #=> :jpg
|
44
|
-
match.
|
45
|
-
match.
|
44
|
+
match.display_width_px #=> 320
|
45
|
+
match.display_height_px #=> 240
|
46
46
|
match.orientation #=> :top_left
|
47
47
|
```
|
48
48
|
|
@@ -122,6 +122,9 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
|
|
122
122
|
### FDX
|
123
123
|
- fixture.fdx was created by one of the project maintainers and is MIT licensed
|
124
124
|
|
125
|
+
### DPX
|
126
|
+
- DPX files were created by one of the project maintainers and may be used with the library for the purposes of testing
|
127
|
+
|
125
128
|
### MOOV
|
126
129
|
- bmff.mp4 is borrowed from the [bmff](https://github.com/zuku/bmff) project
|
127
130
|
- Test_Circular MOV files were created by one of the project maintainers and are MIT licensed
|
data/exe/format_parser_inspect
CHANGED
@@ -6,7 +6,7 @@ require 'optparse'
|
|
6
6
|
|
7
7
|
options = {results: :first}
|
8
8
|
OptionParser.new do |opts|
|
9
|
-
opts.banner = 'Usage: format_parser_inspect --
|
9
|
+
opts.banner = 'Usage: format_parser_inspect --all my_file.jpg my_other_file.png'
|
10
10
|
opts.on('-a', '--all', 'Return all results instead of just the first one') do |_v|
|
11
11
|
options[:results] = :all
|
12
12
|
end
|
data/lib/format_parser.rb
CHANGED
@@ -98,23 +98,17 @@ module FormatParser
|
|
98
98
|
# is ambiguous. The default is `:first` which returns the first matching result. Other
|
99
99
|
# possible values are `:all` to get all possible results and an Integer to return
|
100
100
|
# at most N results.
|
101
|
+
# @param limits_config[ReadLimitsConfig] the configuration object for various read/cache limits. The default
|
102
|
+
# one should be good for most cases.
|
101
103
|
# @return [Array<Result>, Result, nil] either an Array of results, a single parsing result or `nil`if
|
102
104
|
# no useful metadata could be recovered from the file
|
103
|
-
def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first)
|
104
|
-
# We need to apply various limits so that parsers do not over-read, do not cause too many HTTP
|
105
|
-
# requests to be dispatched and so on. These should be _balanced_ with one another- for example,
|
106
|
-
# we cannot tell a parser that it is limited to reading 1024 bytes while at the same time
|
107
|
-
# limiting the size of the cache pages it may slurp in to less than that amount, since
|
108
|
-
# it can quickly become frustrating. The limits configurator computes these limits
|
109
|
-
# for us, in a fairly balanced way, based on one setting.
|
110
|
-
limit_config = FormatParser::ReadLimitsConfig.new(MAX_BYTES_READ_PER_PARSER)
|
111
|
-
|
105
|
+
def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first, limits_config: default_limits_config)
|
112
106
|
# Limit the number of cached _pages_ we may fetch. This allows us to limit the number
|
113
107
|
# of page faults (page cache misses) a parser may incur
|
114
|
-
read_limiter_under_cache = FormatParser::ReadLimiter.new(io, max_reads:
|
108
|
+
read_limiter_under_cache = FormatParser::ReadLimiter.new(io, max_reads: limits_config.max_pagefaults_per_parser)
|
115
109
|
|
116
110
|
# Then configure a layer of caching on top of that
|
117
|
-
cached_io = Care::IOWrapper.new(read_limiter_under_cache, page_size:
|
111
|
+
cached_io = Care::IOWrapper.new(read_limiter_under_cache, page_size: limits_config.cache_page_size)
|
118
112
|
|
119
113
|
# How many results has the user asked for? Used to determinate whether an array
|
120
114
|
# is returned or not.
|
@@ -133,7 +127,12 @@ module FormatParser
|
|
133
127
|
parsers = parsers_for(natures, formats)
|
134
128
|
|
135
129
|
# Limit how many operations the parser can perform
|
136
|
-
limited_io = ReadLimiter.new(
|
130
|
+
limited_io = ReadLimiter.new(
|
131
|
+
cached_io,
|
132
|
+
max_bytes: limits_config.max_read_bytes_per_parser,
|
133
|
+
max_reads: limits_config.max_reads_per_parser,
|
134
|
+
max_seeks: limits_config.max_seeks_per_parser
|
135
|
+
)
|
137
136
|
|
138
137
|
results = parsers.lazy.map do |parser|
|
139
138
|
# Reset all the read limits, per parser
|
@@ -157,6 +156,21 @@ module FormatParser
|
|
157
156
|
cached_io.clear if cached_io
|
158
157
|
end
|
159
158
|
|
159
|
+
# We need to apply various limits so that parsers do not over-read, do not cause too many HTTP
|
160
|
+
# requests to be dispatched and so on. These should be _balanced_ with one another- for example,
|
161
|
+
# we cannot tell a parser that it is limited to reading 1024 bytes while at the same time
|
162
|
+
# limiting the size of the cache pages it may slurp in to less than that amount, since
|
163
|
+
# it can quickly become frustrating. The limits configurator computes these limits
|
164
|
+
# for us, in a fairly balanced way, based on one setting.
|
165
|
+
#
|
166
|
+
# This method returns a ReadLimitsConfig object preset from the `MAX_BYTES_READ_PER_PARSER`
|
167
|
+
# default.
|
168
|
+
#
|
169
|
+
# @return [ReadLimitsConfig]
|
170
|
+
def self.default_limits_config
|
171
|
+
FormatParser::ReadLimitsConfig.new(MAX_BYTES_READ_PER_PARSER)
|
172
|
+
end
|
173
|
+
|
160
174
|
def self.execute_parser_and_capture_expected_exceptions(parser, limited_io)
|
161
175
|
parser_name_for_instrumentation = parser.class.to_s.split('::').last
|
162
176
|
Measurometer.instrument('format_parser.parser.%s' % parser_name_for_instrumentation) do
|
data/lib/image.rb
CHANGED
@@ -15,6 +15,24 @@ module FormatParser
|
|
15
15
|
# Number of pixels vertically in the pixel buffer
|
16
16
|
attr_accessor :height_px
|
17
17
|
|
18
|
+
# Image width when displayed and taking into account things like camera
|
19
|
+
# orientation tags and non-square pixels/anamorphicity. The dimensions
|
20
|
+
# used for display are always computed by _squashing_, not by _stretching_.
|
21
|
+
#
|
22
|
+
# If the display width/height are not specified, the sizes of the
|
23
|
+
# pixel buffer will get returned instead (values of the `width_px`/`height_px`
|
24
|
+
# attributes)
|
25
|
+
attr_accessor :display_width_px
|
26
|
+
|
27
|
+
# Image height when displayed and taking into account things like camera
|
28
|
+
# orientation tags and non-square pixels/anamorphicity. The dimensions
|
29
|
+
# used for display are always computed by _squashing_, not by _stretching_.
|
30
|
+
#
|
31
|
+
# If the display width/height are not specified, the sizes of the
|
32
|
+
# pixel buffer will get returned instead (values of the `width_px`/`height_px`
|
33
|
+
# attributes)
|
34
|
+
attr_accessor :display_height_px
|
35
|
+
|
18
36
|
# Whether the file has multiple frames (relevant for image files and video)
|
19
37
|
attr_accessor :has_multiple_frames
|
20
38
|
|
@@ -49,6 +67,8 @@ module FormatParser
|
|
49
67
|
# Only permits assignments via defined accessors
|
50
68
|
def initialize(**attributes)
|
51
69
|
attributes.map { |(k, v)| public_send("#{k}=", v) }
|
70
|
+
@display_width_px ||= @width_px
|
71
|
+
@display_height_px ||= @height_px
|
52
72
|
end
|
53
73
|
|
54
74
|
def nature
|
data/lib/parsers/dpx_parser.rb
CHANGED
@@ -1,143 +1,57 @@
|
|
1
|
+
require 'delegate'
|
2
|
+
|
1
3
|
class FormatParser::DPXParser
|
2
4
|
include FormatParser::IOUtils
|
5
|
+
require_relative 'dpx_parser/dpx_structs'
|
6
|
+
BE_MAGIC = 'SDPX'
|
7
|
+
LE_MAGIC = BE_MAGIC.reverse
|
3
8
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
:x4, # u32 :ditto_key, :desc => 'Whether the basic headers stay the same through the sequence (1 means they do)'
|
10
|
-
:x4, # u32 :generic_size, :desc => 'Generic header length'
|
11
|
-
:x4, # u32 :industry_size, :desc => 'Industry header length'
|
12
|
-
:x4, # u32 :user_size, :desc => 'User header length'
|
13
|
-
:x100, # char :filename, 100, :desc => 'Original filename'
|
14
|
-
:x24, # char :timestamp, 24, :desc => 'Creation timestamp'
|
15
|
-
:x100, # char :creator, 100, :desc => 'Creator application'
|
16
|
-
:x200, # char :project, 200, :desc => 'Project name'
|
17
|
-
:x200, # char :copyright, 200, :desc => 'Copyright'
|
18
|
-
:x4, # u32 :encrypt_key, :desc => 'Encryption key'
|
19
|
-
:x104, # blanking :reserve, 104
|
20
|
-
].join
|
21
|
-
|
22
|
-
FILM_INFO = [
|
23
|
-
:x2, # char :id, 2, :desc => 'Film mfg. ID code (2 digits from film edge code)'
|
24
|
-
:x2, # char :type, 2, :desc => 'Film type (2 digits from film edge code)'
|
25
|
-
:x2, # char :offset, 2, :desc => 'Offset in perfs (2 digits from film edge code)'
|
26
|
-
:x6, # char :prefix, 6, :desc => 'Prefix (6 digits from film edge code'
|
27
|
-
:x4, # char :count, 4, :desc => 'Count (4 digits from film edge code)'
|
28
|
-
:x32, # char :format, 32, :desc => 'Format (e.g. Academy)'
|
29
|
-
:x4, # u32 :frame_position, :desc => 'Frame position in sequence'
|
30
|
-
:x4, # u32 :sequence_extent, :desc => 'Sequence length'
|
31
|
-
:x4, # u32 :held_count, :desc => 'For how many frames the frame is held'
|
32
|
-
:x4, # r32 :frame_rate, :desc => 'Frame rate'
|
33
|
-
:x4, # r32 :shutter_angle, :desc => 'Shutter angle'
|
34
|
-
:x4, # char :frame_id, 32, :desc => 'Frame identification (keyframe)'
|
35
|
-
:x4, # char :slate, 100, :desc => 'Slate information'
|
36
|
-
:x4, # blanking :reserve, 56
|
37
|
-
].join
|
38
|
-
|
39
|
-
IMAGE_ELEMENT = [
|
40
|
-
:x4, # u32 :data_sign, :desc => 'Data sign (0=unsigned, 1=signed). Core is unsigned', :req => true
|
41
|
-
#
|
42
|
-
:x4, # u32 :low_data, :desc => 'Reference low data code value'
|
43
|
-
:x4, # r32 :low_quantity, :desc => 'Reference low quantity represented'
|
44
|
-
:x4, # u32 :high_data, :desc => 'Reference high data code value (1023 for 10bit per channel)'
|
45
|
-
:x4, # r32 :high_quantity, :desc => 'Reference high quantity represented'
|
46
|
-
#
|
47
|
-
:x1, # u8 :descriptor, :desc => 'Descriptor for this image element (ie Video or Film), by enum', :req => true
|
48
|
-
# TODO - colirimetry information might be handy to recover,
|
49
|
-
# as well as "bit size per element" (how many bits _per component_ we have) -
|
50
|
-
# this will be different for, say, 8-bit DPX files versus 10-bit etc.
|
51
|
-
:x1, # u8 :transfer, :desc => 'Transfer function (ie Linear), by enum', :req => true
|
52
|
-
:x1, # u8 :colorimetric, :desc => 'Colorimetric (ie YcbCr), by enum', :req => true
|
53
|
-
:x1, # u8 :bit_size, :desc => 'Bit size for element (ie 10)', :req => true
|
54
|
-
#
|
55
|
-
:x2, # u16 :packing, :desc => 'Packing (0=Packed into 32-bit words, 1=Filled to 32-bit words))', :req => true
|
56
|
-
:x2, # u16 :encoding, :desc => "Encoding (0=None, 1=RLE)", :req => true
|
57
|
-
:x4, # u32 :data_offset, :desc => 'Offset to data for this image element', :req => true
|
58
|
-
:x4, # u32 :end_of_line_padding, :desc => "End-of-line padding for this image element"
|
59
|
-
:x4, # u32 :end_of_image_padding, :desc => "End-of-line padding for this image element"
|
60
|
-
:x32, # char :description, 32
|
61
|
-
].join
|
9
|
+
class ByteOrderHintIO < SimpleDelegator
|
10
|
+
def initialize(io, is_little_endian)
|
11
|
+
super(io)
|
12
|
+
@little_endian = is_little_endian
|
13
|
+
end
|
62
14
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
:N1, # u32 :lines_per_element, :desc => 'Line count', :req => true
|
68
|
-
IMAGE_ELEMENT * 8, # 8 IMAGE_ELEMENT structures
|
69
|
-
:x52, # blanking :reserve, 52
|
70
|
-
].join
|
15
|
+
def le?
|
16
|
+
@little_endian
|
17
|
+
end
|
18
|
+
end
|
71
19
|
|
72
|
-
|
73
|
-
:x4, # u32 :x_offset
|
74
|
-
:x4, # u32 :y_offset
|
75
|
-
#
|
76
|
-
:x4, # r32 :x_center
|
77
|
-
:x4, # r32 :y_center
|
78
|
-
#
|
79
|
-
:x4, # u32 :x_size, :desc => 'Original X size'
|
80
|
-
:x4, # u32 :y_size, :desc => 'Original Y size'
|
81
|
-
#
|
82
|
-
:x100, # char :filename, 100, :desc => "Source image filename"
|
83
|
-
:x24, # char :timestamp, 24, :desc => "Source image/tape timestamp"
|
84
|
-
:x32, # char :device, 32, :desc => "Input device or tape"
|
85
|
-
:x32, # char :serial, 32, :desc => "Input device serial number"
|
86
|
-
#
|
87
|
-
:x4, # array :border, :u16, 4, :desc => 'Border validity: XL, XR, YT, YB'
|
88
|
-
:x4,
|
89
|
-
:x4,
|
90
|
-
:x4,
|
20
|
+
private_constant :ByteOrderHintIO
|
91
21
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
#
|
97
|
-
:x28, # blanking :reserve, 28
|
98
|
-
].join
|
22
|
+
def call(io)
|
23
|
+
io = FormatParser::IOConstraint.new(io)
|
24
|
+
magic = safe_read(io, 4)
|
25
|
+
return unless [BE_MAGIC, LE_MAGIC].include?(magic)
|
99
26
|
|
100
|
-
|
101
|
-
FILE_INFO,
|
102
|
-
IMAGE_INFO,
|
103
|
-
ORIENTATION_INFO,
|
104
|
-
].join
|
27
|
+
io.seek(0)
|
105
28
|
|
106
|
-
|
29
|
+
dpx_structure = DPX.read_and_unpack(ByteOrderHintIO.new(io, magic == LE_MAGIC))
|
107
30
|
|
108
|
-
|
109
|
-
|
110
|
-
'v' => 2, # 16bit uints
|
111
|
-
'n' => 2,
|
112
|
-
'V' => 4, # 32bit uints
|
113
|
-
'N' => 4,
|
114
|
-
'C' => 1,
|
115
|
-
'x' => 1,
|
116
|
-
}
|
117
|
-
pattern.scan(/[^\d]\d+/).map do |pattern|
|
118
|
-
unpack_code = pattern[0]
|
119
|
-
num_repetitions = pattern[1..-1].to_i
|
120
|
-
bytes_per_element.fetch(unpack_code) * num_repetitions
|
121
|
-
end.inject(&:+)
|
122
|
-
}
|
31
|
+
w = dpx_structure.fetch(:image).fetch(:pixels_per_line)
|
32
|
+
h = dpx_structure.fetch(:image).fetch(:lines_per_element)
|
123
33
|
|
124
|
-
|
125
|
-
|
126
|
-
|
34
|
+
pixel_aspect_w = dpx_structure.fetch(:orientation).fetch(:horizontal_pixel_aspect)
|
35
|
+
pixel_aspect_h = dpx_structure.fetch(:orientation).fetch(:vertical_pixel_aspect)
|
36
|
+
pixel_aspect = pixel_aspect_w / pixel_aspect_h.to_f
|
127
37
|
|
128
|
-
|
129
|
-
io = FormatParser::IOConstraint.new(io)
|
130
|
-
magic = io.read(4)
|
38
|
+
image_aspect = w / h.to_f * pixel_aspect
|
131
39
|
|
132
|
-
|
40
|
+
display_w = w
|
41
|
+
display_h = h
|
42
|
+
if image_aspect > 1
|
43
|
+
display_h = (display_w / image_aspect).round
|
44
|
+
else
|
45
|
+
display_w = (display_h * image_aspect).round
|
46
|
+
end
|
133
47
|
|
134
|
-
unpack_pattern = DPX_INFO
|
135
|
-
unpack_pattern = DPX_INFO_LE if magic == LE_MAGIC
|
136
|
-
_num_elements, pixels_per_line, num_lines, *_ = safe_read(io, HEADER_SIZE).unpack(unpack_pattern)
|
137
48
|
FormatParser::Image.new(
|
138
49
|
format: :dpx,
|
139
|
-
width_px:
|
140
|
-
height_px:
|
50
|
+
width_px: w,
|
51
|
+
height_px: h,
|
52
|
+
display_width_px: display_w,
|
53
|
+
display_height_px: display_h,
|
54
|
+
intrinsics: dpx_structure,
|
141
55
|
)
|
142
56
|
end
|
143
57
|
|
@@ -0,0 +1,229 @@
|
|
1
|
+
class FormatParser::DPXParser
|
2
|
+
# A teeny-tiny rewording of depix (https://rubygems.org/gems/depix)
|
3
|
+
class Binstr
|
4
|
+
TO_LITTLE_ENDIAN = {
|
5
|
+
'N' => 'V',
|
6
|
+
'n' => 'v',
|
7
|
+
}
|
8
|
+
|
9
|
+
class Capture < Struct.new(:pattern, :bytes)
|
10
|
+
include FormatParser::IOUtils
|
11
|
+
def read_and_unpack(io)
|
12
|
+
platform_byte_order_pattern = io.le? ? TO_LITTLE_ENDIAN.fetch(pattern, pattern) : pattern
|
13
|
+
safe_read(io, bytes).unpack(platform_byte_order_pattern).first
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.fields
|
18
|
+
@fields ||= []
|
19
|
+
@fields
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.char(field_name, length, **_kwargs)
|
23
|
+
fields << [field_name, Capture.new('Z%d' % length, length)]
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.u8(field_name, **_kwargs)
|
27
|
+
fields << [field_name, Capture.new('c', 1)]
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.u16(field_name, **_kwargs)
|
31
|
+
fields << [field_name, Capture.new('n', 2)]
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.u32(field_name, **_kwargs)
|
35
|
+
fields << [field_name, Capture.new('N', 4)]
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.r32(field_name, **_kwargs)
|
39
|
+
fields << [field_name, Capture.new('e', 4)]
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.blanking(field_name, length, **_kwargs)
|
43
|
+
fields << [field_name, Capture.new('x%d' % length, length)]
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.array(field_name, nested_struct_descriptor_or_symbol, n_items, **_kwargs)
|
47
|
+
if nested_struct_descriptor_or_symbol.is_a?(Symbol)
|
48
|
+
n_items.times do |i|
|
49
|
+
public_send(nested_struct_descriptor_or_symbol, '%s_%d' % [field_name, i])
|
50
|
+
end
|
51
|
+
else
|
52
|
+
n_items.times do |i|
|
53
|
+
fields << ['%s_%d' % [field_name, i], nested_struct_descriptor_or_symbol]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.inner(field_name, nested_struct_descriptor, **_kwargs)
|
59
|
+
fields << [field_name, nested_struct_descriptor]
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.cleanup(v)
|
63
|
+
case v
|
64
|
+
when String
|
65
|
+
v.scrub
|
66
|
+
when Float
|
67
|
+
v.nan? ? nil : v
|
68
|
+
else
|
69
|
+
v
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.read_and_unpack(io)
|
74
|
+
fields.each_with_object({}) do |(field_name, capture), h|
|
75
|
+
maybe_value = cleanup(capture.read_and_unpack(io))
|
76
|
+
h[field_name] = maybe_value unless maybe_value.nil?
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
class FileInfo < Binstr
|
82
|
+
char :magic, 4, desc: 'Endianness (SDPX is big endian)', req: true
|
83
|
+
u32 :image_offset, desc: 'Offset to image data in bytes', req: true
|
84
|
+
char :version, 8, desc: 'Version of header format', req: true
|
85
|
+
|
86
|
+
u32 :file_size, desc: 'Total image size in bytes', req: true
|
87
|
+
u32 :ditto_key, desc: 'Whether the basic headers stay the same through the sequence (1 means they do)'
|
88
|
+
u32 :generic_size, desc: 'Generic header length'
|
89
|
+
u32 :industry_size, desc: 'Industry header length'
|
90
|
+
u32 :user_size, desc: 'User header length'
|
91
|
+
|
92
|
+
char :filename, 100, desc: 'Original filename'
|
93
|
+
char :timestamp, 24, desc: 'Creation timestamp'
|
94
|
+
char :creator, 100, desc: 'Creator application'
|
95
|
+
char :project, 200, desc: 'Project name'
|
96
|
+
char :copyright, 200, desc: 'Copyright'
|
97
|
+
|
98
|
+
u32 :encrypt_key, desc: 'Encryption key'
|
99
|
+
blanking :reserve, 104
|
100
|
+
end
|
101
|
+
|
102
|
+
class FilmInfo < Binstr
|
103
|
+
char :id, 2, desc: 'Film mfg. ID code (2 digits from film edge code)'
|
104
|
+
char :type, 2, desc: 'Film type (2 digits from film edge code)'
|
105
|
+
char :offset, 2, desc: 'Offset in perfs (2 digits from film edge code)'
|
106
|
+
char :prefix, 6, desc: 'Prefix (6 digits from film edge code'
|
107
|
+
char :count, 4, desc: 'Count (4 digits from film edge code)'
|
108
|
+
char :format, 32, desc: 'Format (e.g. Academy)'
|
109
|
+
|
110
|
+
u32 :frame_position, desc: 'Frame position in sequence'
|
111
|
+
u32 :sequence_extent, desc: 'Sequence length'
|
112
|
+
u32 :held_count, desc: 'For how many frames the frame is held'
|
113
|
+
|
114
|
+
r32 :frame_rate, desc: 'Frame rate'
|
115
|
+
r32 :shutter_angle, desc: 'Shutter angle'
|
116
|
+
|
117
|
+
char :frame_id, 32, desc: 'Frame identification (keyframe)'
|
118
|
+
char :slate, 100, desc: 'Slate information'
|
119
|
+
blanking :reserve, 56
|
120
|
+
end
|
121
|
+
|
122
|
+
class ImageElement < Binstr
|
123
|
+
u32 :data_sign, desc: 'Data sign (0=unsigned, 1=signed). Core is unsigned', req: true
|
124
|
+
|
125
|
+
u32 :low_data, desc: 'Reference low data code value'
|
126
|
+
r32 :low_quantity, desc: 'Reference low quantity represented'
|
127
|
+
u32 :high_data, desc: 'Reference high data code value (1023 for 10bit per channel)'
|
128
|
+
r32 :high_quantity, desc: 'Reference high quantity represented'
|
129
|
+
|
130
|
+
u8 :descriptor, desc: 'Descriptor for this image element (ie Video or Film), by enum', req: true
|
131
|
+
u8 :transfer, desc: 'Transfer function (ie Linear), by enum', req: true
|
132
|
+
u8 :colorimetric, desc: 'Colorimetric (ie YcbCr), by enum', req: true
|
133
|
+
u8 :bit_size, desc: 'Bit size for element (ie 10)', req: true
|
134
|
+
|
135
|
+
u16 :packing, desc: 'Packing (0=Packed into 32-bit words, 1=Filled to 32-bit words))', req: true
|
136
|
+
u16 :encoding, desc: 'Encoding (0=None, 1=RLE)', req: true
|
137
|
+
u32 :data_offset, desc: 'Offset to data for this image element', req: true
|
138
|
+
u32 :end_of_line_padding, desc: 'End-of-line padding for this image element'
|
139
|
+
u32 :end_of_image_padding, desc: 'End-of-line padding for this image element'
|
140
|
+
char :description, 32
|
141
|
+
end
|
142
|
+
|
143
|
+
class OrientationInfo < Binstr
|
144
|
+
u32 :x_offset
|
145
|
+
u32 :y_offset
|
146
|
+
|
147
|
+
r32 :x_center
|
148
|
+
r32 :y_center
|
149
|
+
|
150
|
+
u32 :x_size, desc: 'Original X size'
|
151
|
+
u32 :y_size, desc: 'Original Y size'
|
152
|
+
|
153
|
+
char :filename, 100, desc: 'Source image filename'
|
154
|
+
char :timestamp, 24, desc: 'Source image/tape timestamp'
|
155
|
+
char :device, 32, desc: 'Input device or tape'
|
156
|
+
char :serial, 32, desc: 'Input device serial number'
|
157
|
+
|
158
|
+
array :border, :u16, 4, desc: 'Border validity: XL, XR, YT, YB'
|
159
|
+
u32 :horizontal_pixel_aspect, desc: 'Aspect (H)'
|
160
|
+
u32 :vertical_pixel_aspect, desc: 'Aspect (V)'
|
161
|
+
|
162
|
+
blanking :reserve, 28
|
163
|
+
end
|
164
|
+
|
165
|
+
class TelevisionInfo < Binstr
|
166
|
+
u32 :time_code, desc: 'Timecode, formatted as HH:MM:SS:FF in the 4 higher bits of each 8bit group'
|
167
|
+
u32 :user_bits, desc: 'Timecode UBITs'
|
168
|
+
u8 :interlace, desc: 'Interlace (0 = noninterlaced; 1 = 2:1 interlace'
|
169
|
+
|
170
|
+
u8 :field_number, desc: 'Field number'
|
171
|
+
u8 :video_signal, desc: 'Video signal (by enum)'
|
172
|
+
u8 :padding, desc: 'Zero (for byte alignment)'
|
173
|
+
|
174
|
+
r32 :horizontal_sample_rate, desc: 'Horizontal sampling Hz'
|
175
|
+
r32 :vertical_sample_rate, desc: 'Vertical sampling Hz'
|
176
|
+
r32 :frame_rate, desc: 'Frame rate'
|
177
|
+
r32 :time_offset, desc: 'From sync pulse to first pixel'
|
178
|
+
r32 :gamma, desc: 'Gamma'
|
179
|
+
r32 :black_level, desc: 'Black pedestal code value'
|
180
|
+
r32 :black_gain, desc: 'Black gain code value'
|
181
|
+
r32 :break_point, desc: 'Break point (?)'
|
182
|
+
r32 :white_level, desc: 'White level'
|
183
|
+
r32 :integration_times, desc: 'Integration times (S)'
|
184
|
+
blanking :reserve, 4 # As long as a real
|
185
|
+
end
|
186
|
+
|
187
|
+
class UserInfo < Binstr
|
188
|
+
char :id, 32, desc: 'Name of the user data tag'
|
189
|
+
u32 :user_data_ptr
|
190
|
+
end
|
191
|
+
|
192
|
+
class ImageInfo < Binstr
|
193
|
+
u16 :orientation, desc: 'To which orientation descriptor this relates', req: true
|
194
|
+
u16 :number_elements, desc: 'How many elements to scan', req: true
|
195
|
+
|
196
|
+
u32 :pixels_per_line, desc: 'Pixels per horizontal line', req: true
|
197
|
+
u32 :lines_per_element, desc: 'Line count', req: true
|
198
|
+
|
199
|
+
array :image_elements, ImageElement, 8, desc: 'Image elements'
|
200
|
+
|
201
|
+
blanking :reserve, 52
|
202
|
+
|
203
|
+
# Only expose the elements present
|
204
|
+
def image_elements #:nodoc:
|
205
|
+
@image_elements[0...number_elements]
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
# This is the main structure represinting headers of one DPX file
|
210
|
+
class DPX < Binstr
|
211
|
+
inner :file, FileInfo, desc: 'File information', req: true
|
212
|
+
inner :image, ImageInfo, desc: 'Image information', req: true
|
213
|
+
inner :orientation, OrientationInfo, desc: 'Orientation', req: true
|
214
|
+
inner :film, FilmInfo, desc: 'Film industry info', req: true
|
215
|
+
inner :television, TelevisionInfo, desc: 'TV industry info', req: true
|
216
|
+
blanking :user, 32 + 4, desc: 'User info', req: true
|
217
|
+
|
218
|
+
def self.read_and_unpack(io)
|
219
|
+
super.tap do |h|
|
220
|
+
num_elems = h[:image][:number_elements]
|
221
|
+
num_elems.upto(8) do |n|
|
222
|
+
h[:image].delete("image_elements_#{n}")
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
private_constant :Binstr, :FileInfo, :FilmInfo, :ImageElement, :OrientationInfo, :TelevisionInfo, :UserInfo, :ImageInfo
|
229
|
+
end
|
data/lib/parsers/exif_parser.rb
CHANGED
@@ -1,8 +1,31 @@
|
|
1
1
|
require 'exifr/tiff'
|
2
2
|
require 'delegate'
|
3
3
|
|
4
|
-
|
5
|
-
|
4
|
+
module FormatParser::EXIFParser
|
5
|
+
ORIENTATIONS = [
|
6
|
+
:top_left,
|
7
|
+
:top_right,
|
8
|
+
:bottom_right,
|
9
|
+
:bottom_left,
|
10
|
+
:left_top,
|
11
|
+
:right_top,
|
12
|
+
:right_bottom,
|
13
|
+
:left_bottom
|
14
|
+
]
|
15
|
+
ROTATED_ORIENTATIONS = ORIENTATIONS - [
|
16
|
+
:bottom_left,
|
17
|
+
:bottom_right,
|
18
|
+
:top_left,
|
19
|
+
:top_right
|
20
|
+
]
|
21
|
+
module MethodsMethodFix
|
22
|
+
# Fix a little bug in EXIFR which breaks delegators
|
23
|
+
# https://github.com/remvee/exifr/pull/55
|
24
|
+
def methods(*)
|
25
|
+
super() # no args
|
26
|
+
end
|
27
|
+
end
|
28
|
+
EXIFR::TIFF.prepend(MethodsMethodFix)
|
6
29
|
|
7
30
|
# EXIFR kindly requests the presence of a few more methods than what our IOConstraint
|
8
31
|
# is willing to provide, but they can be derived from the available ones
|
@@ -30,47 +53,26 @@ class FormatParser::EXIFParser
|
|
30
53
|
alias_method :getbyte, :readbyte
|
31
54
|
end
|
32
55
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
attr_accessor :exif_data, :orientation, :width, :height
|
38
|
-
|
39
|
-
ORIENTATIONS = [
|
40
|
-
:top_left,
|
41
|
-
:top_right,
|
42
|
-
:bottom_right,
|
43
|
-
:bottom_left,
|
44
|
-
:left_top,
|
45
|
-
:right_top,
|
46
|
-
:right_bottom,
|
47
|
-
:left_bottom
|
48
|
-
]
|
56
|
+
class EXIFResult < SimpleDelegator
|
57
|
+
def rotated?
|
58
|
+
ROTATED_ORIENTATIONS.include?(orientation)
|
59
|
+
end
|
49
60
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
@orientation = nil
|
54
|
-
@height = nil
|
55
|
-
@width = nil
|
56
|
-
end
|
61
|
+
def to_json(*maybe_coder)
|
62
|
+
__getobj__.to_hash.to_json(*maybe_coder)
|
63
|
+
end
|
57
64
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
@exif_data = raw_exif_data
|
63
|
-
@orientation = orientation_parser(raw_exif_data)
|
64
|
-
@width = @exif_data.width
|
65
|
-
@height = @exif_data.height
|
65
|
+
def orientation
|
66
|
+
value = __getobj__.orientation.to_i
|
67
|
+
ORIENTATIONS.fetch(value - 1)
|
68
|
+
end
|
66
69
|
end
|
67
70
|
|
68
|
-
|
69
|
-
|
70
|
-
@orientation = ORIENTATIONS[value - 1] if valid_orientation?(value)
|
71
|
-
end
|
71
|
+
# Squash exifr's invalid date warning since we do not use that data.
|
72
|
+
EXIFR.logger = Logger.new(nil)
|
72
73
|
|
73
|
-
def
|
74
|
-
(
|
74
|
+
def exif_from_tiff_io(constrained_io)
|
75
|
+
raw_exif_data = EXIFR::TIFF.new(IOExt.new(constrained_io))
|
76
|
+
raw_exif_data ? EXIFResult.new(raw_exif_data) : nil
|
75
77
|
end
|
76
78
|
end
|
data/lib/parsers/jpeg_parser.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
class FormatParser::JPEGParser
|
2
2
|
include FormatParser::IOUtils
|
3
|
+
include FormatParser::EXIFParser
|
3
4
|
|
4
5
|
class InvalidStructure < StandardError
|
5
6
|
end
|
@@ -16,8 +17,6 @@ class FormatParser::JPEGParser
|
|
16
17
|
@buf = FormatParser::IOConstraint.new(io)
|
17
18
|
@width = nil
|
18
19
|
@height = nil
|
19
|
-
@orientation = nil
|
20
|
-
@intrinsics = {}
|
21
20
|
scan
|
22
21
|
end
|
23
22
|
|
@@ -66,13 +65,17 @@ class FormatParser::JPEGParser
|
|
66
65
|
|
67
66
|
# Return at the earliest possible opportunity
|
68
67
|
if @width && @height
|
69
|
-
|
68
|
+
result = FormatParser::Image.new(
|
70
69
|
format: :jpg,
|
71
70
|
width_px: @width,
|
72
71
|
height_px: @height,
|
73
|
-
|
74
|
-
|
72
|
+
display_width_px: @exif_data&.rotated? ? @height : @width,
|
73
|
+
display_height_px: @exif_data&.rotated? ? @width : @height,
|
74
|
+
orientation: @exif_data&.orientation,
|
75
|
+
intrinsics: {exif: @exif_data},
|
75
76
|
)
|
77
|
+
|
78
|
+
return result
|
76
79
|
end
|
77
80
|
|
78
81
|
nil # We could not parse anything
|
@@ -138,27 +141,8 @@ class FormatParser::JPEGParser
|
|
138
141
|
exif_data = safe_read(@buf, app1_frame_content_length - EXIF_MAGIC_STRING.bytesize)
|
139
142
|
|
140
143
|
FormatParser::Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_sent_to_exif_parser', exif_data.bytesize)
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
@exif_output = scanner.exif_data
|
145
|
-
@orientation = scanner.orientation unless scanner.orientation.nil?
|
146
|
-
@intrinsics[:exif_pixel_x_dimension] = @exif_output.pixel_x_dimension
|
147
|
-
@intrinsics[:exif_pixel_y_dimension] = @exif_output.pixel_y_dimension
|
148
|
-
# Save these two for later, when we decide to provide display width /
|
149
|
-
# display height in addition to pixel buffer width / height. These two
|
150
|
-
# are _different concepts_. Imagine you have an image shot with a camera
|
151
|
-
# in portrait orientation, and the camera has an anamorphic lens. That
|
152
|
-
# anamorpohic lens is a smart lens, and therefore transmits pixel aspect
|
153
|
-
# ratio to the camera, and the camera encodes that aspect ratio into the
|
154
|
-
# image metadata. If we want to know what size our _pixel buffer_ will be,
|
155
|
-
# and how to _read_ the pixel data (stride/interleaving) - we need the
|
156
|
-
# pixel buffer dimensions. If we want to know what aspect and dimensions
|
157
|
-
# our file is going to have _once displayed_ and _once pixels have been
|
158
|
-
# brought to the right orientation_ we need to work with **display dimensions**
|
159
|
-
# which can be remarkably different from the pixel buffer dimensions.
|
160
|
-
@exif_width = scanner.width
|
161
|
-
@exif_height = scanner.height
|
144
|
+
|
145
|
+
@exif_data = exif_from_tiff_io(StringIO.new(exif_data))
|
162
146
|
rescue EXIFR::MalformedTIFF
|
163
147
|
# Not a JPEG or the Exif headers contain invalid data, or
|
164
148
|
# an APP1 marker was detected in a file that is not a JPEG
|
data/lib/parsers/tiff_parser.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
class FormatParser::TIFFParser
|
2
2
|
include FormatParser::IOUtils
|
3
|
+
include FormatParser::EXIFParser
|
3
4
|
|
4
5
|
MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
|
5
6
|
MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
|
@@ -14,16 +15,20 @@ class FormatParser::TIFFParser
|
|
14
15
|
# The TIFF scanner in EXIFR is plenty good enough,
|
15
16
|
# so why don't we use it? It does all the right skips
|
16
17
|
# in all the right places.
|
17
|
-
|
18
|
-
|
19
|
-
|
18
|
+
exif_data = exif_from_tiff_io(io)
|
19
|
+
return unless exif_data
|
20
|
+
|
21
|
+
w = exif_data.image_width
|
22
|
+
h = exif_data.image_length
|
20
23
|
|
21
24
|
FormatParser::Image.new(
|
22
25
|
format: :tif,
|
23
|
-
width_px:
|
24
|
-
height_px:
|
25
|
-
|
26
|
-
|
26
|
+
width_px: w,
|
27
|
+
height_px: h,
|
28
|
+
display_width_px: exif_data.rotated? ? h : w,
|
29
|
+
display_height_px: exif_data.rotated? ? w : h,
|
30
|
+
orientation: exif_data.orientation,
|
31
|
+
intrinsics: {exif: exif_data},
|
27
32
|
)
|
28
33
|
rescue EXIFR::MalformedTIFF
|
29
34
|
nil
|
data/lib/read_limits_config.rb
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
# We need to apply various limits so that parsers do not over-read, do not cause too many HTTP
|
2
|
+
# requests to be dispatched and so on. These should be _balanced_ with one another- for example,
|
3
|
+
# we cannot tell a parser that it is limited to reading 1024 bytes while at the same time
|
4
|
+
# limiting the size of the cache pages it may slurp in to less than that amount, since
|
5
|
+
# it can quickly become frustrating. ReadLimitsConfig computes these limits
|
6
|
+
# for us, in a fairly balanced way, based on one setting.
|
1
7
|
class FormatParser::ReadLimitsConfig
|
2
8
|
MAX_PAGE_FAULTS = 16
|
3
9
|
|
@@ -42,6 +42,21 @@ describe FormatParser::AttributesJSON do
|
|
42
42
|
expect(readback[:some_infinity]).to be_nil
|
43
43
|
end
|
44
44
|
|
45
|
+
it 'converts NaN to nil' do
|
46
|
+
anon_class = Class.new do
|
47
|
+
include FormatParser::AttributesJSON
|
48
|
+
attr_accessor :some_nan
|
49
|
+
def some_nan
|
50
|
+
(1.0 / 0.0).to_f
|
51
|
+
end
|
52
|
+
end
|
53
|
+
instance = anon_class.new
|
54
|
+
output = JSON.dump(instance)
|
55
|
+
readback = JSON.parse(output, symbolize_names: true)
|
56
|
+
expect(readback).to have_key(:some_nan)
|
57
|
+
expect(readback[:some_nan]).to be_nil
|
58
|
+
end
|
59
|
+
|
45
60
|
it 'provides a default implementation of to_json as well' do
|
46
61
|
anon_class = Class.new do
|
47
62
|
include FormatParser::AttributesJSON
|
data/spec/format_parser_spec.rb
CHANGED
@@ -1,17 +1,31 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe FormatParser do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
describe '.parse' do
|
5
|
+
it 'returns nil when trying to parse an empty IO' do
|
6
|
+
d = StringIO.new('')
|
7
|
+
expect(FormatParser.parse(d)).to be_nil
|
8
|
+
end
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
10
|
+
it 'returns nil when parsing an IO no parser can make sense of' do
|
11
|
+
d = StringIO.new(Random.new.bytes(1))
|
12
|
+
expect(FormatParser.parse(d)).to be_nil
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'uses the passed ReadLimitsConfig and applies limits in it' do
|
16
|
+
conf = FormatParser::ReadLimitsConfig.new(16)
|
17
|
+
d = StringIO.new(Random.new.bytes(64 * 1024))
|
18
|
+
expect(FormatParser.parse(d, limits_config: conf)).to be_nil
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'parses our fixtures without raising any errors' do
|
22
|
+
Dir.glob(fixtures_dir + '/**/*.*').sort.each do |fixture_path|
|
23
|
+
File.open(fixture_path, 'rb') do |file|
|
24
|
+
FormatParser.parse(file)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
13
28
|
|
14
|
-
describe 'with fuzzing' do
|
15
29
|
it "returns either a valid result or a nil for all fuzzed inputs at seed #{RSpec.configuration.seed}" do
|
16
30
|
r = Random.new(RSpec.configuration.seed)
|
17
31
|
1024.times do
|
@@ -19,54 +33,54 @@ describe FormatParser do
|
|
19
33
|
FormatParser.parse(random_blob) # If there is an error in one of the parsers the example will raise too
|
20
34
|
end
|
21
35
|
end
|
22
|
-
end
|
23
36
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
37
|
+
it 'fails gracefully when a parser module reads more and more causing page faults and prevents too many reads on the source' do
|
38
|
+
exploit = ->(io) {
|
39
|
+
loop {
|
40
|
+
skip = 16 * 1024
|
41
|
+
io.read(1)
|
42
|
+
io.seek(io.pos + skip)
|
43
|
+
}
|
30
44
|
}
|
31
|
-
|
32
|
-
FormatParser.register_parser exploit, natures: :document, formats: :exploit
|
45
|
+
FormatParser.register_parser exploit, natures: :document, formats: :exploit
|
33
46
|
|
34
|
-
|
35
|
-
|
47
|
+
sample_io = StringIO.new(Random.new.bytes(1024 * 1024 * 8))
|
48
|
+
allow(sample_io).to receive(:read).and_call_original
|
36
49
|
|
37
|
-
|
50
|
+
result = FormatParser.parse(sample_io, formats: [:exploit])
|
38
51
|
|
39
|
-
|
40
|
-
|
52
|
+
expect(sample_io).to have_received(:read).at_most(16).times
|
53
|
+
expect(result).to be_nil
|
41
54
|
|
42
|
-
|
43
|
-
|
55
|
+
FormatParser.deregister_parser(exploit)
|
56
|
+
end
|
44
57
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
58
|
+
describe 'when multiple results are requested' do
|
59
|
+
let(:blob) { StringIO.new(Random.new.bytes(512 * 1024)) }
|
60
|
+
let(:audio) { FormatParser::Audio.new(format: :aiff, num_audio_channels: 1) }
|
61
|
+
let(:image) { FormatParser::Image.new(format: :dpx, width_px: 1, height_px: 1) }
|
49
62
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
subject { FormatParser.parse(blob, results: :all) }
|
63
|
+
context 'with :result=> :all (multiple results)' do
|
64
|
+
before do
|
65
|
+
expect_any_instance_of(FormatParser::AIFFParser).to receive(:call).and_return(audio)
|
66
|
+
expect_any_instance_of(FormatParser::DPXParser).to receive(:call).and_return(image)
|
67
|
+
end
|
57
68
|
|
58
|
-
|
59
|
-
it { is_expected.to include(audio) }
|
60
|
-
end
|
69
|
+
subject { FormatParser.parse(blob, results: :all) }
|
61
70
|
|
62
|
-
|
63
|
-
|
64
|
-
expect_any_instance_of(FormatParser::DPXParser).to receive(:call).and_return(image)
|
71
|
+
it { is_expected.to include(image) }
|
72
|
+
it { is_expected.to include(audio) }
|
65
73
|
end
|
66
74
|
|
67
|
-
|
75
|
+
context 'without :result=> :all (first result)' do
|
76
|
+
before do
|
77
|
+
expect_any_instance_of(FormatParser::DPXParser).to receive(:call).and_return(image)
|
78
|
+
end
|
79
|
+
|
80
|
+
subject { FormatParser.parse(blob) }
|
68
81
|
|
69
|
-
|
82
|
+
it { is_expected.to eq(image) }
|
83
|
+
end
|
70
84
|
end
|
71
85
|
end
|
72
86
|
|
@@ -76,19 +90,15 @@ describe FormatParser do
|
|
76
90
|
result = FormatParser.parse_file_at(path)
|
77
91
|
expect(result.nature).to eq(:audio)
|
78
92
|
end
|
79
|
-
end
|
80
93
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
FormatParser.parse(file)
|
86
|
-
end
|
87
|
-
end
|
94
|
+
it 'passes keyword arguments to parse()' do
|
95
|
+
path = fixtures_dir + '/WAV/c_M1F1-Alaw-AFsp.wav'
|
96
|
+
expect(FormatParser).to receive(:parse).with(an_instance_of(File), foo: :bar)
|
97
|
+
FormatParser.parse_file_at(path, foo: :bar)
|
88
98
|
end
|
89
99
|
end
|
90
100
|
|
91
|
-
describe 'parsers_for' do
|
101
|
+
describe '.parsers_for' do
|
92
102
|
it 'raises on an invalid request' do
|
93
103
|
expect {
|
94
104
|
FormatParser.parsers_for([:image], [:fdx])
|
@@ -111,7 +121,7 @@ describe FormatParser do
|
|
111
121
|
end
|
112
122
|
end
|
113
123
|
|
114
|
-
describe '
|
124
|
+
describe '.register_parser and .deregister_parser' do
|
115
125
|
it 'registers a parser for a certain nature and format' do
|
116
126
|
some_parser = ->(_io) { 'I parse EXRs! Whee!' }
|
117
127
|
|
@@ -130,4 +140,10 @@ describe FormatParser do
|
|
130
140
|
}.to raise_error(/No parsers provide/)
|
131
141
|
end
|
132
142
|
end
|
143
|
+
|
144
|
+
describe '.default_limits_config' do
|
145
|
+
it 'returns a ReadLimitsConfig object' do
|
146
|
+
expect(FormatParser.default_limits_config).to be_kind_of(FormatParser::ReadLimitsConfig)
|
147
|
+
end
|
148
|
+
end
|
133
149
|
end
|
@@ -1,29 +1,40 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe FormatParser::DPXParser do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
expect(parsed.format).to eq(:dpx)
|
12
|
-
|
13
|
-
# If we have an error in the struct offsets these values are likely to become
|
14
|
-
# the maximum value of a 4-byte uint, which is way higher
|
15
|
-
expect(parsed.width_px).to be_kind_of(Integer)
|
16
|
-
expect(parsed.width_px).to be_between(0, 2048)
|
17
|
-
expect(parsed.height_px).to be_kind_of(Integer)
|
18
|
-
expect(parsed.height_px).to be_between(0, 4000)
|
19
|
-
end
|
20
|
-
end
|
4
|
+
Dir.glob(fixtures_dir + '/dpx/*.*').each do |dpx_path|
|
5
|
+
it "is able to parse #{File.basename(dpx_path)}" do
|
6
|
+
parsed = subject.call(File.open(dpx_path, 'rb'))
|
7
|
+
|
8
|
+
expect(parsed).not_to be_nil
|
9
|
+
expect(parsed.nature).to eq(:image)
|
10
|
+
expect(parsed.format).to eq(:dpx)
|
21
11
|
|
22
|
-
|
23
|
-
|
24
|
-
parsed
|
25
|
-
expect(parsed.width_px).to
|
26
|
-
expect(parsed.height_px).to
|
12
|
+
# If we have an error in the struct offsets these values are likely to become
|
13
|
+
# the maximum value of a 4-byte uint, which is way higher
|
14
|
+
expect(parsed.width_px).to be_kind_of(Integer)
|
15
|
+
expect(parsed.width_px).to be_between(0, 2048)
|
16
|
+
expect(parsed.height_px).to be_kind_of(Integer)
|
17
|
+
expect(parsed.height_px).to be_between(0, 4000)
|
27
18
|
end
|
28
19
|
end
|
20
|
+
|
21
|
+
it 'correctly reads display dimensions corrected for the pixel aspect from the DPX header' do
|
22
|
+
fi = File.open(fixtures_dir + '/dpx/aspect_237_example.dpx', 'rb')
|
23
|
+
parsed = subject.call(fi)
|
24
|
+
|
25
|
+
expect(parsed.width_px).to eq(1920)
|
26
|
+
expect(parsed.height_px).to eq(1080)
|
27
|
+
|
28
|
+
expect(parsed.display_width_px).to eq(1920)
|
29
|
+
expect(parsed.display_height_px).to eq(810)
|
30
|
+
|
31
|
+
expect(parsed.display_width_px / parsed.display_height_px.to_f).to be_within(0.01).of(2.37)
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'does not explode on invalid inputs' do
|
35
|
+
invalid = StringIO.new('SDPX' + (' ' * 64))
|
36
|
+
expect {
|
37
|
+
subject.call(invalid)
|
38
|
+
}.to raise_error(FormatParser::IOUtils::InvalidRead)
|
39
|
+
end
|
29
40
|
end
|
@@ -1,17 +1,19 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe FormatParser::EXIFParser do
|
4
|
+
let(:subject) do
|
5
|
+
Object.new.tap { |o| o.extend FormatParser::EXIFParser }
|
6
|
+
end
|
7
|
+
|
4
8
|
describe 'is able to correctly parse orientation for all the TIFF EXIF examples from FastImage' do
|
5
9
|
Dir.glob(fixtures_dir + '/exif-orientation-testimages/tiff-*/*.tif').each do |tiff_path|
|
6
10
|
filename = File.basename(tiff_path)
|
7
11
|
it "is able to parse #{filename}" do
|
8
|
-
|
9
|
-
|
10
|
-
expect(
|
11
|
-
|
12
|
-
expect(parser.orientation).to be_kind_of(Symbol)
|
12
|
+
result = subject.exif_from_tiff_io(File.open(tiff_path, 'rb'))
|
13
|
+
expect(result).not_to be_nil
|
14
|
+
expect(result.orientation).to be_kind_of(Symbol)
|
13
15
|
# Filenames in this dir correspond with the orientation of the file
|
14
|
-
expect(filename.include
|
16
|
+
expect(filename).to include(result.orientation.to_s)
|
15
17
|
end
|
16
18
|
end
|
17
19
|
end
|
@@ -26,6 +26,8 @@ describe FormatParser::JPEGParser do
|
|
26
26
|
expect(parsed.orientation).to be_kind_of(Symbol)
|
27
27
|
expect(parsed.width_px).to be > 0
|
28
28
|
expect(parsed.height_px).to be > 0
|
29
|
+
expect(parsed.display_width_px).to eq(1240)
|
30
|
+
expect(parsed.display_height_px).to eq(1754)
|
29
31
|
end
|
30
32
|
|
31
33
|
bottom_left_path = fixtures_dir + '/exif-orientation-testimages/jpg/bottom_left.jpg'
|
@@ -33,20 +35,30 @@ describe FormatParser::JPEGParser do
|
|
33
35
|
expect(parsed.orientation).to eq(:bottom_left)
|
34
36
|
expect(parsed.width_px).to eq(1240)
|
35
37
|
expect(parsed.height_px).to eq(1754)
|
38
|
+
expect(parsed.display_width_px).to eq(1240)
|
39
|
+
expect(parsed.display_height_px).to eq(1754)
|
40
|
+
expect(parsed.intrinsics[:exif]).not_to be_nil
|
36
41
|
|
37
42
|
top_right_path = fixtures_dir + '/exif-orientation-testimages/jpg/right_bottom.jpg'
|
38
43
|
parsed = subject.call(File.open(top_right_path, 'rb'))
|
39
44
|
expect(parsed.orientation).to eq(:right_bottom)
|
40
45
|
expect(parsed.width_px).to eq(1754)
|
41
46
|
expect(parsed.height_px).to eq(1240)
|
47
|
+
expect(parsed.display_width_px).to eq(1240)
|
48
|
+
expect(parsed.display_height_px).to eq(1754)
|
49
|
+
expect(parsed.intrinsics[:exif]).not_to be_nil
|
42
50
|
end
|
43
51
|
|
44
52
|
it 'gives true pixel dimensions priority over pixel dimensions in EXIF tags' do
|
45
53
|
jpeg_path = fixtures_dir + '/JPEG/divergent_pixel_dimensions_exif.jpg'
|
46
54
|
result = subject.call(File.open(jpeg_path, 'rb'))
|
55
|
+
|
47
56
|
expect(result.width_px).to eq(1920)
|
48
57
|
expect(result.height_px).to eq(1280)
|
49
|
-
|
58
|
+
|
59
|
+
exif = result.intrinsics.fetch(:exif)
|
60
|
+
expect(exif.pixel_x_dimension).to eq(8214)
|
61
|
+
expect(exif.pixel_y_dimension).to eq(5476)
|
50
62
|
end
|
51
63
|
|
52
64
|
it 'reads an example with many APP1 markers at the beginning of which none are EXIF' do
|
@@ -44,6 +44,7 @@ describe FormatParser::TIFFParser do
|
|
44
44
|
expect(parsed).not_to be_nil
|
45
45
|
expect(parsed.width_px).to eq(320)
|
46
46
|
expect(parsed.height_px).to eq(240)
|
47
|
+
expect(parsed.intrinsics[:exif]).not_to be_nil
|
47
48
|
end
|
48
49
|
|
49
50
|
describe 'correctly extracts dimensions from various TIFF flavors of the same file' do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: format_parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Noah Berman
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-04-
|
12
|
+
date: 2018-04-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: ks
|
@@ -177,6 +177,7 @@ files:
|
|
177
177
|
- lib/parsers/bmp_parser.rb
|
178
178
|
- lib/parsers/cr2_parser.rb
|
179
179
|
- lib/parsers/dpx_parser.rb
|
180
|
+
- lib/parsers/dpx_parser/dpx_structs.rb
|
180
181
|
- lib/parsers/exif_parser.rb
|
181
182
|
- lib/parsers/fdx_parser.rb
|
182
183
|
- lib/parsers/flac_parser.rb
|