format_parser 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +5 -2
- data/exe/format_parser_inspect +1 -1
- data/lib/format_parser.rb +26 -12
- data/lib/format_parser/version.rb +1 -1
- data/lib/image.rb +20 -0
- data/lib/parsers/dpx_parser.rb +39 -125
- data/lib/parsers/dpx_parser/dpx_structs.rb +229 -0
- data/lib/parsers/exif_parser.rb +41 -39
- data/lib/parsers/jpeg_parser.rb +10 -26
- data/lib/parsers/tiff_parser.rb +12 -7
- data/lib/read_limits_config.rb +6 -0
- data/spec/attributes_json_spec.rb +15 -0
- data/spec/format_parser_spec.rb +70 -54
- data/spec/parsers/dpx_parser_spec.rb +33 -22
- data/spec/parsers/exif_parser_spec.rb +8 -6
- data/spec/parsers/jpeg_parser_spec.rb +13 -1
- data/spec/parsers/tiff_parser_spec.rb +1 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ab01944664740856d704875f0a74d1b0540379c8e351176e38d3b5bf119e813f
|
4
|
+
data.tar.gz: b0736ffd074eb3fb49586d52799dd687c72f0a09b6e2d4109b1cbd26a8eac293
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 570e9fcef6a08ad4e800c84d8452d985900f8442d9071477e6cd465b533677f7c39b6ae51c8f94d0ab0e90305ce790c395c3ce54ecda46800921b3311adc23b8
|
7
|
+
data.tar.gz: 0662cc268f0fc1fc61ac97896811e505f1ae251a2a15fe9027de2414107adb1414b6ce4f3a1111be1215e8e48323330ce8b08b98e521bf7d9ec4a26c8d29f01d
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.11.0
|
2
|
+
* Add `Image#display_width_px` and `Image#display_height_px` for EXIF/aspect corrected display dimensions, and provide
|
3
|
+
those values from a few parsers already. Also make full EXIF data available for JPEG/TIFF in `intrinsics[:exif]`
|
4
|
+
* Adds `limits_config` option to `FormatParser.parse()` for tweaking buffers and read limits externally
|
5
|
+
|
1
6
|
## 0.10.0
|
2
7
|
* Adds the `format_parser_inspect` binary for parsing a file from the commandline
|
3
8
|
and returning results in JSON
|
data/README.md
CHANGED
@@ -41,8 +41,8 @@ Pass an IO object that responds to `read` and `seek` to `FormatParser` and the f
|
|
41
41
|
match = FormatParser.parse(File.open("myimage.jpg", "rb"))
|
42
42
|
match.nature #=> :image
|
43
43
|
match.format #=> :jpg
|
44
|
-
match.
|
45
|
-
match.
|
44
|
+
match.display_width_px #=> 320
|
45
|
+
match.display_height_px #=> 240
|
46
46
|
match.orientation #=> :top_left
|
47
47
|
```
|
48
48
|
|
@@ -122,6 +122,9 @@ Unless specified otherwise in this section the fixture files are MIT licensed an
|
|
122
122
|
### FDX
|
123
123
|
- fixture.fdx was created by one of the project maintainers and is MIT licensed
|
124
124
|
|
125
|
+
### DPX
|
126
|
+
- DPX files were created by one of the project maintainers and may be used with the library for the purposes of testing
|
127
|
+
|
125
128
|
### MOOV
|
126
129
|
- bmff.mp4 is borrowed from the [bmff](https://github.com/zuku/bmff) project
|
127
130
|
- Test_Circular MOV files were created by one of the project maintainers and are MIT licensed
|
data/exe/format_parser_inspect
CHANGED
@@ -6,7 +6,7 @@ require 'optparse'
|
|
6
6
|
|
7
7
|
options = {results: :first}
|
8
8
|
OptionParser.new do |opts|
|
9
|
-
opts.banner = 'Usage: format_parser_inspect --
|
9
|
+
opts.banner = 'Usage: format_parser_inspect --all my_file.jpg my_other_file.png'
|
10
10
|
opts.on('-a', '--all', 'Return all results instead of just the first one') do |_v|
|
11
11
|
options[:results] = :all
|
12
12
|
end
|
data/lib/format_parser.rb
CHANGED
@@ -98,23 +98,17 @@ module FormatParser
|
|
98
98
|
# is ambiguous. The default is `:first` which returns the first matching result. Other
|
99
99
|
# possible values are `:all` to get all possible results and an Integer to return
|
100
100
|
# at most N results.
|
101
|
+
# @param limits_config[ReadLimitsConfig] the configuration object for various read/cache limits. The default
|
102
|
+
# one should be good for most cases.
|
101
103
|
# @return [Array<Result>, Result, nil] either an Array of results, a single parsing result or `nil`if
|
102
104
|
# no useful metadata could be recovered from the file
|
103
|
-
def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first)
|
104
|
-
# We need to apply various limits so that parsers do not over-read, do not cause too many HTTP
|
105
|
-
# requests to be dispatched and so on. These should be _balanced_ with one another- for example,
|
106
|
-
# we cannot tell a parser that it is limited to reading 1024 bytes while at the same time
|
107
|
-
# limiting the size of the cache pages it may slurp in to less than that amount, since
|
108
|
-
# it can quickly become frustrating. The limits configurator computes these limits
|
109
|
-
# for us, in a fairly balanced way, based on one setting.
|
110
|
-
limit_config = FormatParser::ReadLimitsConfig.new(MAX_BYTES_READ_PER_PARSER)
|
111
|
-
|
105
|
+
def self.parse(io, natures: @parsers_per_nature.keys, formats: @parsers_per_format.keys, results: :first, limits_config: default_limits_config)
|
112
106
|
# Limit the number of cached _pages_ we may fetch. This allows us to limit the number
|
113
107
|
# of page faults (page cache misses) a parser may incur
|
114
|
-
read_limiter_under_cache = FormatParser::ReadLimiter.new(io, max_reads:
|
108
|
+
read_limiter_under_cache = FormatParser::ReadLimiter.new(io, max_reads: limits_config.max_pagefaults_per_parser)
|
115
109
|
|
116
110
|
# Then configure a layer of caching on top of that
|
117
|
-
cached_io = Care::IOWrapper.new(read_limiter_under_cache, page_size:
|
111
|
+
cached_io = Care::IOWrapper.new(read_limiter_under_cache, page_size: limits_config.cache_page_size)
|
118
112
|
|
119
113
|
# How many results has the user asked for? Used to determinate whether an array
|
120
114
|
# is returned or not.
|
@@ -133,7 +127,12 @@ module FormatParser
|
|
133
127
|
parsers = parsers_for(natures, formats)
|
134
128
|
|
135
129
|
# Limit how many operations the parser can perform
|
136
|
-
limited_io = ReadLimiter.new(
|
130
|
+
limited_io = ReadLimiter.new(
|
131
|
+
cached_io,
|
132
|
+
max_bytes: limits_config.max_read_bytes_per_parser,
|
133
|
+
max_reads: limits_config.max_reads_per_parser,
|
134
|
+
max_seeks: limits_config.max_seeks_per_parser
|
135
|
+
)
|
137
136
|
|
138
137
|
results = parsers.lazy.map do |parser|
|
139
138
|
# Reset all the read limits, per parser
|
@@ -157,6 +156,21 @@ module FormatParser
|
|
157
156
|
cached_io.clear if cached_io
|
158
157
|
end
|
159
158
|
|
159
|
+
# We need to apply various limits so that parsers do not over-read, do not cause too many HTTP
|
160
|
+
# requests to be dispatched and so on. These should be _balanced_ with one another- for example,
|
161
|
+
# we cannot tell a parser that it is limited to reading 1024 bytes while at the same time
|
162
|
+
# limiting the size of the cache pages it may slurp in to less than that amount, since
|
163
|
+
# it can quickly become frustrating. The limits configurator computes these limits
|
164
|
+
# for us, in a fairly balanced way, based on one setting.
|
165
|
+
#
|
166
|
+
# This method returns a ReadLimitsConfig object preset from the `MAX_BYTES_READ_PER_PARSER`
|
167
|
+
# default.
|
168
|
+
#
|
169
|
+
# @return [ReadLimitsConfig]
|
170
|
+
def self.default_limits_config
|
171
|
+
FormatParser::ReadLimitsConfig.new(MAX_BYTES_READ_PER_PARSER)
|
172
|
+
end
|
173
|
+
|
160
174
|
def self.execute_parser_and_capture_expected_exceptions(parser, limited_io)
|
161
175
|
parser_name_for_instrumentation = parser.class.to_s.split('::').last
|
162
176
|
Measurometer.instrument('format_parser.parser.%s' % parser_name_for_instrumentation) do
|
data/lib/image.rb
CHANGED
@@ -15,6 +15,24 @@ module FormatParser
|
|
15
15
|
# Number of pixels vertically in the pixel buffer
|
16
16
|
attr_accessor :height_px
|
17
17
|
|
18
|
+
# Image width when displayed and taking into account things like camera
|
19
|
+
# orientation tags and non-square pixels/anamorphicity. The dimensions
|
20
|
+
# used for display are always computed by _squashing_, not by _stretching_.
|
21
|
+
#
|
22
|
+
# If the display width/height are not specified, the sizes of the
|
23
|
+
# pixel buffer will get returned instead (values of the `width_px`/`height_px`
|
24
|
+
# attributes)
|
25
|
+
attr_accessor :display_width_px
|
26
|
+
|
27
|
+
# Image height when displayed and taking into account things like camera
|
28
|
+
# orientation tags and non-square pixels/anamorphicity. The dimensions
|
29
|
+
# used for display are always computed by _squashing_, not by _stretching_.
|
30
|
+
#
|
31
|
+
# If the display width/height are not specified, the sizes of the
|
32
|
+
# pixel buffer will get returned instead (values of the `width_px`/`height_px`
|
33
|
+
# attributes)
|
34
|
+
attr_accessor :display_height_px
|
35
|
+
|
18
36
|
# Whether the file has multiple frames (relevant for image files and video)
|
19
37
|
attr_accessor :has_multiple_frames
|
20
38
|
|
@@ -49,6 +67,8 @@ module FormatParser
|
|
49
67
|
# Only permits assignments via defined accessors
|
50
68
|
def initialize(**attributes)
|
51
69
|
attributes.map { |(k, v)| public_send("#{k}=", v) }
|
70
|
+
@display_width_px ||= @width_px
|
71
|
+
@display_height_px ||= @height_px
|
52
72
|
end
|
53
73
|
|
54
74
|
def nature
|
data/lib/parsers/dpx_parser.rb
CHANGED
@@ -1,143 +1,57 @@
|
|
1
|
+
require 'delegate'
|
2
|
+
|
1
3
|
class FormatParser::DPXParser
|
2
4
|
include FormatParser::IOUtils
|
5
|
+
require_relative 'dpx_parser/dpx_structs'
|
6
|
+
BE_MAGIC = 'SDPX'
|
7
|
+
LE_MAGIC = BE_MAGIC.reverse
|
3
8
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
:x4, # u32 :ditto_key, :desc => 'Whether the basic headers stay the same through the sequence (1 means they do)'
|
10
|
-
:x4, # u32 :generic_size, :desc => 'Generic header length'
|
11
|
-
:x4, # u32 :industry_size, :desc => 'Industry header length'
|
12
|
-
:x4, # u32 :user_size, :desc => 'User header length'
|
13
|
-
:x100, # char :filename, 100, :desc => 'Original filename'
|
14
|
-
:x24, # char :timestamp, 24, :desc => 'Creation timestamp'
|
15
|
-
:x100, # char :creator, 100, :desc => 'Creator application'
|
16
|
-
:x200, # char :project, 200, :desc => 'Project name'
|
17
|
-
:x200, # char :copyright, 200, :desc => 'Copyright'
|
18
|
-
:x4, # u32 :encrypt_key, :desc => 'Encryption key'
|
19
|
-
:x104, # blanking :reserve, 104
|
20
|
-
].join
|
21
|
-
|
22
|
-
FILM_INFO = [
|
23
|
-
:x2, # char :id, 2, :desc => 'Film mfg. ID code (2 digits from film edge code)'
|
24
|
-
:x2, # char :type, 2, :desc => 'Film type (2 digits from film edge code)'
|
25
|
-
:x2, # char :offset, 2, :desc => 'Offset in perfs (2 digits from film edge code)'
|
26
|
-
:x6, # char :prefix, 6, :desc => 'Prefix (6 digits from film edge code'
|
27
|
-
:x4, # char :count, 4, :desc => 'Count (4 digits from film edge code)'
|
28
|
-
:x32, # char :format, 32, :desc => 'Format (e.g. Academy)'
|
29
|
-
:x4, # u32 :frame_position, :desc => 'Frame position in sequence'
|
30
|
-
:x4, # u32 :sequence_extent, :desc => 'Sequence length'
|
31
|
-
:x4, # u32 :held_count, :desc => 'For how many frames the frame is held'
|
32
|
-
:x4, # r32 :frame_rate, :desc => 'Frame rate'
|
33
|
-
:x4, # r32 :shutter_angle, :desc => 'Shutter angle'
|
34
|
-
:x4, # char :frame_id, 32, :desc => 'Frame identification (keyframe)'
|
35
|
-
:x4, # char :slate, 100, :desc => 'Slate information'
|
36
|
-
:x4, # blanking :reserve, 56
|
37
|
-
].join
|
38
|
-
|
39
|
-
IMAGE_ELEMENT = [
|
40
|
-
:x4, # u32 :data_sign, :desc => 'Data sign (0=unsigned, 1=signed). Core is unsigned', :req => true
|
41
|
-
#
|
42
|
-
:x4, # u32 :low_data, :desc => 'Reference low data code value'
|
43
|
-
:x4, # r32 :low_quantity, :desc => 'Reference low quantity represented'
|
44
|
-
:x4, # u32 :high_data, :desc => 'Reference high data code value (1023 for 10bit per channel)'
|
45
|
-
:x4, # r32 :high_quantity, :desc => 'Reference high quantity represented'
|
46
|
-
#
|
47
|
-
:x1, # u8 :descriptor, :desc => 'Descriptor for this image element (ie Video or Film), by enum', :req => true
|
48
|
-
# TODO - colirimetry information might be handy to recover,
|
49
|
-
# as well as "bit size per element" (how many bits _per component_ we have) -
|
50
|
-
# this will be different for, say, 8-bit DPX files versus 10-bit etc.
|
51
|
-
:x1, # u8 :transfer, :desc => 'Transfer function (ie Linear), by enum', :req => true
|
52
|
-
:x1, # u8 :colorimetric, :desc => 'Colorimetric (ie YcbCr), by enum', :req => true
|
53
|
-
:x1, # u8 :bit_size, :desc => 'Bit size for element (ie 10)', :req => true
|
54
|
-
#
|
55
|
-
:x2, # u16 :packing, :desc => 'Packing (0=Packed into 32-bit words, 1=Filled to 32-bit words))', :req => true
|
56
|
-
:x2, # u16 :encoding, :desc => "Encoding (0=None, 1=RLE)", :req => true
|
57
|
-
:x4, # u32 :data_offset, :desc => 'Offset to data for this image element', :req => true
|
58
|
-
:x4, # u32 :end_of_line_padding, :desc => "End-of-line padding for this image element"
|
59
|
-
:x4, # u32 :end_of_image_padding, :desc => "End-of-line padding for this image element"
|
60
|
-
:x32, # char :description, 32
|
61
|
-
].join
|
9
|
+
class ByteOrderHintIO < SimpleDelegator
|
10
|
+
def initialize(io, is_little_endian)
|
11
|
+
super(io)
|
12
|
+
@little_endian = is_little_endian
|
13
|
+
end
|
62
14
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
:N1, # u32 :lines_per_element, :desc => 'Line count', :req => true
|
68
|
-
IMAGE_ELEMENT * 8, # 8 IMAGE_ELEMENT structures
|
69
|
-
:x52, # blanking :reserve, 52
|
70
|
-
].join
|
15
|
+
def le?
|
16
|
+
@little_endian
|
17
|
+
end
|
18
|
+
end
|
71
19
|
|
72
|
-
|
73
|
-
:x4, # u32 :x_offset
|
74
|
-
:x4, # u32 :y_offset
|
75
|
-
#
|
76
|
-
:x4, # r32 :x_center
|
77
|
-
:x4, # r32 :y_center
|
78
|
-
#
|
79
|
-
:x4, # u32 :x_size, :desc => 'Original X size'
|
80
|
-
:x4, # u32 :y_size, :desc => 'Original Y size'
|
81
|
-
#
|
82
|
-
:x100, # char :filename, 100, :desc => "Source image filename"
|
83
|
-
:x24, # char :timestamp, 24, :desc => "Source image/tape timestamp"
|
84
|
-
:x32, # char :device, 32, :desc => "Input device or tape"
|
85
|
-
:x32, # char :serial, 32, :desc => "Input device serial number"
|
86
|
-
#
|
87
|
-
:x4, # array :border, :u16, 4, :desc => 'Border validity: XL, XR, YT, YB'
|
88
|
-
:x4,
|
89
|
-
:x4,
|
90
|
-
:x4,
|
20
|
+
private_constant :ByteOrderHintIO
|
91
21
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
#
|
97
|
-
:x28, # blanking :reserve, 28
|
98
|
-
].join
|
22
|
+
def call(io)
|
23
|
+
io = FormatParser::IOConstraint.new(io)
|
24
|
+
magic = safe_read(io, 4)
|
25
|
+
return unless [BE_MAGIC, LE_MAGIC].include?(magic)
|
99
26
|
|
100
|
-
|
101
|
-
FILE_INFO,
|
102
|
-
IMAGE_INFO,
|
103
|
-
ORIENTATION_INFO,
|
104
|
-
].join
|
27
|
+
io.seek(0)
|
105
28
|
|
106
|
-
|
29
|
+
dpx_structure = DPX.read_and_unpack(ByteOrderHintIO.new(io, magic == LE_MAGIC))
|
107
30
|
|
108
|
-
|
109
|
-
|
110
|
-
'v' => 2, # 16bit uints
|
111
|
-
'n' => 2,
|
112
|
-
'V' => 4, # 32bit uints
|
113
|
-
'N' => 4,
|
114
|
-
'C' => 1,
|
115
|
-
'x' => 1,
|
116
|
-
}
|
117
|
-
pattern.scan(/[^\d]\d+/).map do |pattern|
|
118
|
-
unpack_code = pattern[0]
|
119
|
-
num_repetitions = pattern[1..-1].to_i
|
120
|
-
bytes_per_element.fetch(unpack_code) * num_repetitions
|
121
|
-
end.inject(&:+)
|
122
|
-
}
|
31
|
+
w = dpx_structure.fetch(:image).fetch(:pixels_per_line)
|
32
|
+
h = dpx_structure.fetch(:image).fetch(:lines_per_element)
|
123
33
|
|
124
|
-
|
125
|
-
|
126
|
-
|
34
|
+
pixel_aspect_w = dpx_structure.fetch(:orientation).fetch(:horizontal_pixel_aspect)
|
35
|
+
pixel_aspect_h = dpx_structure.fetch(:orientation).fetch(:vertical_pixel_aspect)
|
36
|
+
pixel_aspect = pixel_aspect_w / pixel_aspect_h.to_f
|
127
37
|
|
128
|
-
|
129
|
-
io = FormatParser::IOConstraint.new(io)
|
130
|
-
magic = io.read(4)
|
38
|
+
image_aspect = w / h.to_f * pixel_aspect
|
131
39
|
|
132
|
-
|
40
|
+
display_w = w
|
41
|
+
display_h = h
|
42
|
+
if image_aspect > 1
|
43
|
+
display_h = (display_w / image_aspect).round
|
44
|
+
else
|
45
|
+
display_w = (display_h * image_aspect).round
|
46
|
+
end
|
133
47
|
|
134
|
-
unpack_pattern = DPX_INFO
|
135
|
-
unpack_pattern = DPX_INFO_LE if magic == LE_MAGIC
|
136
|
-
_num_elements, pixels_per_line, num_lines, *_ = safe_read(io, HEADER_SIZE).unpack(unpack_pattern)
|
137
48
|
FormatParser::Image.new(
|
138
49
|
format: :dpx,
|
139
|
-
width_px:
|
140
|
-
height_px:
|
50
|
+
width_px: w,
|
51
|
+
height_px: h,
|
52
|
+
display_width_px: display_w,
|
53
|
+
display_height_px: display_h,
|
54
|
+
intrinsics: dpx_structure,
|
141
55
|
)
|
142
56
|
end
|
143
57
|
|
@@ -0,0 +1,229 @@
|
|
1
|
+
class FormatParser::DPXParser
|
2
|
+
# A teeny-tiny rewording of depix (https://rubygems.org/gems/depix)
|
3
|
+
class Binstr
|
4
|
+
TO_LITTLE_ENDIAN = {
|
5
|
+
'N' => 'V',
|
6
|
+
'n' => 'v',
|
7
|
+
}
|
8
|
+
|
9
|
+
class Capture < Struct.new(:pattern, :bytes)
|
10
|
+
include FormatParser::IOUtils
|
11
|
+
def read_and_unpack(io)
|
12
|
+
platform_byte_order_pattern = io.le? ? TO_LITTLE_ENDIAN.fetch(pattern, pattern) : pattern
|
13
|
+
safe_read(io, bytes).unpack(platform_byte_order_pattern).first
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.fields
|
18
|
+
@fields ||= []
|
19
|
+
@fields
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.char(field_name, length, **_kwargs)
|
23
|
+
fields << [field_name, Capture.new('Z%d' % length, length)]
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.u8(field_name, **_kwargs)
|
27
|
+
fields << [field_name, Capture.new('c', 1)]
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.u16(field_name, **_kwargs)
|
31
|
+
fields << [field_name, Capture.new('n', 2)]
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.u32(field_name, **_kwargs)
|
35
|
+
fields << [field_name, Capture.new('N', 4)]
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.r32(field_name, **_kwargs)
|
39
|
+
fields << [field_name, Capture.new('e', 4)]
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.blanking(field_name, length, **_kwargs)
|
43
|
+
fields << [field_name, Capture.new('x%d' % length, length)]
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.array(field_name, nested_struct_descriptor_or_symbol, n_items, **_kwargs)
|
47
|
+
if nested_struct_descriptor_or_symbol.is_a?(Symbol)
|
48
|
+
n_items.times do |i|
|
49
|
+
public_send(nested_struct_descriptor_or_symbol, '%s_%d' % [field_name, i])
|
50
|
+
end
|
51
|
+
else
|
52
|
+
n_items.times do |i|
|
53
|
+
fields << ['%s_%d' % [field_name, i], nested_struct_descriptor_or_symbol]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.inner(field_name, nested_struct_descriptor, **_kwargs)
|
59
|
+
fields << [field_name, nested_struct_descriptor]
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.cleanup(v)
|
63
|
+
case v
|
64
|
+
when String
|
65
|
+
v.scrub
|
66
|
+
when Float
|
67
|
+
v.nan? ? nil : v
|
68
|
+
else
|
69
|
+
v
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.read_and_unpack(io)
|
74
|
+
fields.each_with_object({}) do |(field_name, capture), h|
|
75
|
+
maybe_value = cleanup(capture.read_and_unpack(io))
|
76
|
+
h[field_name] = maybe_value unless maybe_value.nil?
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
class FileInfo < Binstr
|
82
|
+
char :magic, 4, desc: 'Endianness (SDPX is big endian)', req: true
|
83
|
+
u32 :image_offset, desc: 'Offset to image data in bytes', req: true
|
84
|
+
char :version, 8, desc: 'Version of header format', req: true
|
85
|
+
|
86
|
+
u32 :file_size, desc: 'Total image size in bytes', req: true
|
87
|
+
u32 :ditto_key, desc: 'Whether the basic headers stay the same through the sequence (1 means they do)'
|
88
|
+
u32 :generic_size, desc: 'Generic header length'
|
89
|
+
u32 :industry_size, desc: 'Industry header length'
|
90
|
+
u32 :user_size, desc: 'User header length'
|
91
|
+
|
92
|
+
char :filename, 100, desc: 'Original filename'
|
93
|
+
char :timestamp, 24, desc: 'Creation timestamp'
|
94
|
+
char :creator, 100, desc: 'Creator application'
|
95
|
+
char :project, 200, desc: 'Project name'
|
96
|
+
char :copyright, 200, desc: 'Copyright'
|
97
|
+
|
98
|
+
u32 :encrypt_key, desc: 'Encryption key'
|
99
|
+
blanking :reserve, 104
|
100
|
+
end
|
101
|
+
|
102
|
+
class FilmInfo < Binstr
|
103
|
+
char :id, 2, desc: 'Film mfg. ID code (2 digits from film edge code)'
|
104
|
+
char :type, 2, desc: 'Film type (2 digits from film edge code)'
|
105
|
+
char :offset, 2, desc: 'Offset in perfs (2 digits from film edge code)'
|
106
|
+
char :prefix, 6, desc: 'Prefix (6 digits from film edge code'
|
107
|
+
char :count, 4, desc: 'Count (4 digits from film edge code)'
|
108
|
+
char :format, 32, desc: 'Format (e.g. Academy)'
|
109
|
+
|
110
|
+
u32 :frame_position, desc: 'Frame position in sequence'
|
111
|
+
u32 :sequence_extent, desc: 'Sequence length'
|
112
|
+
u32 :held_count, desc: 'For how many frames the frame is held'
|
113
|
+
|
114
|
+
r32 :frame_rate, desc: 'Frame rate'
|
115
|
+
r32 :shutter_angle, desc: 'Shutter angle'
|
116
|
+
|
117
|
+
char :frame_id, 32, desc: 'Frame identification (keyframe)'
|
118
|
+
char :slate, 100, desc: 'Slate information'
|
119
|
+
blanking :reserve, 56
|
120
|
+
end
|
121
|
+
|
122
|
+
class ImageElement < Binstr
|
123
|
+
u32 :data_sign, desc: 'Data sign (0=unsigned, 1=signed). Core is unsigned', req: true
|
124
|
+
|
125
|
+
u32 :low_data, desc: 'Reference low data code value'
|
126
|
+
r32 :low_quantity, desc: 'Reference low quantity represented'
|
127
|
+
u32 :high_data, desc: 'Reference high data code value (1023 for 10bit per channel)'
|
128
|
+
r32 :high_quantity, desc: 'Reference high quantity represented'
|
129
|
+
|
130
|
+
u8 :descriptor, desc: 'Descriptor for this image element (ie Video or Film), by enum', req: true
|
131
|
+
u8 :transfer, desc: 'Transfer function (ie Linear), by enum', req: true
|
132
|
+
u8 :colorimetric, desc: 'Colorimetric (ie YcbCr), by enum', req: true
|
133
|
+
u8 :bit_size, desc: 'Bit size for element (ie 10)', req: true
|
134
|
+
|
135
|
+
u16 :packing, desc: 'Packing (0=Packed into 32-bit words, 1=Filled to 32-bit words))', req: true
|
136
|
+
u16 :encoding, desc: 'Encoding (0=None, 1=RLE)', req: true
|
137
|
+
u32 :data_offset, desc: 'Offset to data for this image element', req: true
|
138
|
+
u32 :end_of_line_padding, desc: 'End-of-line padding for this image element'
|
139
|
+
u32 :end_of_image_padding, desc: 'End-of-line padding for this image element'
|
140
|
+
char :description, 32
|
141
|
+
end
|
142
|
+
|
143
|
+
class OrientationInfo < Binstr
|
144
|
+
u32 :x_offset
|
145
|
+
u32 :y_offset
|
146
|
+
|
147
|
+
r32 :x_center
|
148
|
+
r32 :y_center
|
149
|
+
|
150
|
+
u32 :x_size, desc: 'Original X size'
|
151
|
+
u32 :y_size, desc: 'Original Y size'
|
152
|
+
|
153
|
+
char :filename, 100, desc: 'Source image filename'
|
154
|
+
char :timestamp, 24, desc: 'Source image/tape timestamp'
|
155
|
+
char :device, 32, desc: 'Input device or tape'
|
156
|
+
char :serial, 32, desc: 'Input device serial number'
|
157
|
+
|
158
|
+
array :border, :u16, 4, desc: 'Border validity: XL, XR, YT, YB'
|
159
|
+
u32 :horizontal_pixel_aspect, desc: 'Aspect (H)'
|
160
|
+
u32 :vertical_pixel_aspect, desc: 'Aspect (V)'
|
161
|
+
|
162
|
+
blanking :reserve, 28
|
163
|
+
end
|
164
|
+
|
165
|
+
class TelevisionInfo < Binstr
|
166
|
+
u32 :time_code, desc: 'Timecode, formatted as HH:MM:SS:FF in the 4 higher bits of each 8bit group'
|
167
|
+
u32 :user_bits, desc: 'Timecode UBITs'
|
168
|
+
u8 :interlace, desc: 'Interlace (0 = noninterlaced; 1 = 2:1 interlace'
|
169
|
+
|
170
|
+
u8 :field_number, desc: 'Field number'
|
171
|
+
u8 :video_signal, desc: 'Video signal (by enum)'
|
172
|
+
u8 :padding, desc: 'Zero (for byte alignment)'
|
173
|
+
|
174
|
+
r32 :horizontal_sample_rate, desc: 'Horizontal sampling Hz'
|
175
|
+
r32 :vertical_sample_rate, desc: 'Vertical sampling Hz'
|
176
|
+
r32 :frame_rate, desc: 'Frame rate'
|
177
|
+
r32 :time_offset, desc: 'From sync pulse to first pixel'
|
178
|
+
r32 :gamma, desc: 'Gamma'
|
179
|
+
r32 :black_level, desc: 'Black pedestal code value'
|
180
|
+
r32 :black_gain, desc: 'Black gain code value'
|
181
|
+
r32 :break_point, desc: 'Break point (?)'
|
182
|
+
r32 :white_level, desc: 'White level'
|
183
|
+
r32 :integration_times, desc: 'Integration times (S)'
|
184
|
+
blanking :reserve, 4 # As long as a real
|
185
|
+
end
|
186
|
+
|
187
|
+
class UserInfo < Binstr
|
188
|
+
char :id, 32, desc: 'Name of the user data tag'
|
189
|
+
u32 :user_data_ptr
|
190
|
+
end
|
191
|
+
|
192
|
+
class ImageInfo < Binstr
|
193
|
+
u16 :orientation, desc: 'To which orientation descriptor this relates', req: true
|
194
|
+
u16 :number_elements, desc: 'How many elements to scan', req: true
|
195
|
+
|
196
|
+
u32 :pixels_per_line, desc: 'Pixels per horizontal line', req: true
|
197
|
+
u32 :lines_per_element, desc: 'Line count', req: true
|
198
|
+
|
199
|
+
array :image_elements, ImageElement, 8, desc: 'Image elements'
|
200
|
+
|
201
|
+
blanking :reserve, 52
|
202
|
+
|
203
|
+
# Only expose the elements present
|
204
|
+
def image_elements #:nodoc:
|
205
|
+
@image_elements[0...number_elements]
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
# This is the main structure represinting headers of one DPX file
|
210
|
+
class DPX < Binstr
|
211
|
+
inner :file, FileInfo, desc: 'File information', req: true
|
212
|
+
inner :image, ImageInfo, desc: 'Image information', req: true
|
213
|
+
inner :orientation, OrientationInfo, desc: 'Orientation', req: true
|
214
|
+
inner :film, FilmInfo, desc: 'Film industry info', req: true
|
215
|
+
inner :television, TelevisionInfo, desc: 'TV industry info', req: true
|
216
|
+
blanking :user, 32 + 4, desc: 'User info', req: true
|
217
|
+
|
218
|
+
def self.read_and_unpack(io)
|
219
|
+
super.tap do |h|
|
220
|
+
num_elems = h[:image][:number_elements]
|
221
|
+
num_elems.upto(8) do |n|
|
222
|
+
h[:image].delete("image_elements_#{n}")
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
private_constant :Binstr, :FileInfo, :FilmInfo, :ImageElement, :OrientationInfo, :TelevisionInfo, :UserInfo, :ImageInfo
|
229
|
+
end
|
data/lib/parsers/exif_parser.rb
CHANGED
@@ -1,8 +1,31 @@
|
|
1
1
|
require 'exifr/tiff'
|
2
2
|
require 'delegate'
|
3
3
|
|
4
|
-
|
5
|
-
|
4
|
+
module FormatParser::EXIFParser
|
5
|
+
ORIENTATIONS = [
|
6
|
+
:top_left,
|
7
|
+
:top_right,
|
8
|
+
:bottom_right,
|
9
|
+
:bottom_left,
|
10
|
+
:left_top,
|
11
|
+
:right_top,
|
12
|
+
:right_bottom,
|
13
|
+
:left_bottom
|
14
|
+
]
|
15
|
+
ROTATED_ORIENTATIONS = ORIENTATIONS - [
|
16
|
+
:bottom_left,
|
17
|
+
:bottom_right,
|
18
|
+
:top_left,
|
19
|
+
:top_right
|
20
|
+
]
|
21
|
+
module MethodsMethodFix
|
22
|
+
# Fix a little bug in EXIFR which breaks delegators
|
23
|
+
# https://github.com/remvee/exifr/pull/55
|
24
|
+
def methods(*)
|
25
|
+
super() # no args
|
26
|
+
end
|
27
|
+
end
|
28
|
+
EXIFR::TIFF.prepend(MethodsMethodFix)
|
6
29
|
|
7
30
|
# EXIFR kindly requests the presence of a few more methods than what our IOConstraint
|
8
31
|
# is willing to provide, but they can be derived from the available ones
|
@@ -30,47 +53,26 @@ class FormatParser::EXIFParser
|
|
30
53
|
alias_method :getbyte, :readbyte
|
31
54
|
end
|
32
55
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
attr_accessor :exif_data, :orientation, :width, :height
|
38
|
-
|
39
|
-
ORIENTATIONS = [
|
40
|
-
:top_left,
|
41
|
-
:top_right,
|
42
|
-
:bottom_right,
|
43
|
-
:bottom_left,
|
44
|
-
:left_top,
|
45
|
-
:right_top,
|
46
|
-
:right_bottom,
|
47
|
-
:left_bottom
|
48
|
-
]
|
56
|
+
class EXIFResult < SimpleDelegator
|
57
|
+
def rotated?
|
58
|
+
ROTATED_ORIENTATIONS.include?(orientation)
|
59
|
+
end
|
49
60
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
@orientation = nil
|
54
|
-
@height = nil
|
55
|
-
@width = nil
|
56
|
-
end
|
61
|
+
def to_json(*maybe_coder)
|
62
|
+
__getobj__.to_hash.to_json(*maybe_coder)
|
63
|
+
end
|
57
64
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
@exif_data = raw_exif_data
|
63
|
-
@orientation = orientation_parser(raw_exif_data)
|
64
|
-
@width = @exif_data.width
|
65
|
-
@height = @exif_data.height
|
65
|
+
def orientation
|
66
|
+
value = __getobj__.orientation.to_i
|
67
|
+
ORIENTATIONS.fetch(value - 1)
|
68
|
+
end
|
66
69
|
end
|
67
70
|
|
68
|
-
|
69
|
-
|
70
|
-
@orientation = ORIENTATIONS[value - 1] if valid_orientation?(value)
|
71
|
-
end
|
71
|
+
# Squash exifr's invalid date warning since we do not use that data.
|
72
|
+
EXIFR.logger = Logger.new(nil)
|
72
73
|
|
73
|
-
def
|
74
|
-
(
|
74
|
+
def exif_from_tiff_io(constrained_io)
|
75
|
+
raw_exif_data = EXIFR::TIFF.new(IOExt.new(constrained_io))
|
76
|
+
raw_exif_data ? EXIFResult.new(raw_exif_data) : nil
|
75
77
|
end
|
76
78
|
end
|
data/lib/parsers/jpeg_parser.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
class FormatParser::JPEGParser
|
2
2
|
include FormatParser::IOUtils
|
3
|
+
include FormatParser::EXIFParser
|
3
4
|
|
4
5
|
class InvalidStructure < StandardError
|
5
6
|
end
|
@@ -16,8 +17,6 @@ class FormatParser::JPEGParser
|
|
16
17
|
@buf = FormatParser::IOConstraint.new(io)
|
17
18
|
@width = nil
|
18
19
|
@height = nil
|
19
|
-
@orientation = nil
|
20
|
-
@intrinsics = {}
|
21
20
|
scan
|
22
21
|
end
|
23
22
|
|
@@ -66,13 +65,17 @@ class FormatParser::JPEGParser
|
|
66
65
|
|
67
66
|
# Return at the earliest possible opportunity
|
68
67
|
if @width && @height
|
69
|
-
|
68
|
+
result = FormatParser::Image.new(
|
70
69
|
format: :jpg,
|
71
70
|
width_px: @width,
|
72
71
|
height_px: @height,
|
73
|
-
|
74
|
-
|
72
|
+
display_width_px: @exif_data&.rotated? ? @height : @width,
|
73
|
+
display_height_px: @exif_data&.rotated? ? @width : @height,
|
74
|
+
orientation: @exif_data&.orientation,
|
75
|
+
intrinsics: {exif: @exif_data},
|
75
76
|
)
|
77
|
+
|
78
|
+
return result
|
76
79
|
end
|
77
80
|
|
78
81
|
nil # We could not parse anything
|
@@ -138,27 +141,8 @@ class FormatParser::JPEGParser
|
|
138
141
|
exif_data = safe_read(@buf, app1_frame_content_length - EXIF_MAGIC_STRING.bytesize)
|
139
142
|
|
140
143
|
FormatParser::Measurometer.add_distribution_value('format_parser.JPEGParser.bytes_sent_to_exif_parser', exif_data.bytesize)
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
@exif_output = scanner.exif_data
|
145
|
-
@orientation = scanner.orientation unless scanner.orientation.nil?
|
146
|
-
@intrinsics[:exif_pixel_x_dimension] = @exif_output.pixel_x_dimension
|
147
|
-
@intrinsics[:exif_pixel_y_dimension] = @exif_output.pixel_y_dimension
|
148
|
-
# Save these two for later, when we decide to provide display width /
|
149
|
-
# display height in addition to pixel buffer width / height. These two
|
150
|
-
# are _different concepts_. Imagine you have an image shot with a camera
|
151
|
-
# in portrait orientation, and the camera has an anamorphic lens. That
|
152
|
-
# anamorpohic lens is a smart lens, and therefore transmits pixel aspect
|
153
|
-
# ratio to the camera, and the camera encodes that aspect ratio into the
|
154
|
-
# image metadata. If we want to know what size our _pixel buffer_ will be,
|
155
|
-
# and how to _read_ the pixel data (stride/interleaving) - we need the
|
156
|
-
# pixel buffer dimensions. If we want to know what aspect and dimensions
|
157
|
-
# our file is going to have _once displayed_ and _once pixels have been
|
158
|
-
# brought to the right orientation_ we need to work with **display dimensions**
|
159
|
-
# which can be remarkably different from the pixel buffer dimensions.
|
160
|
-
@exif_width = scanner.width
|
161
|
-
@exif_height = scanner.height
|
144
|
+
|
145
|
+
@exif_data = exif_from_tiff_io(StringIO.new(exif_data))
|
162
146
|
rescue EXIFR::MalformedTIFF
|
163
147
|
# Not a JPEG or the Exif headers contain invalid data, or
|
164
148
|
# an APP1 marker was detected in a file that is not a JPEG
|
data/lib/parsers/tiff_parser.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
class FormatParser::TIFFParser
|
2
2
|
include FormatParser::IOUtils
|
3
|
+
include FormatParser::EXIFParser
|
3
4
|
|
4
5
|
MAGIC_LE = [0x49, 0x49, 0x2A, 0x0].pack('C4')
|
5
6
|
MAGIC_BE = [0x4D, 0x4D, 0x0, 0x2A].pack('C4')
|
@@ -14,16 +15,20 @@ class FormatParser::TIFFParser
|
|
14
15
|
# The TIFF scanner in EXIFR is plenty good enough,
|
15
16
|
# so why don't we use it? It does all the right skips
|
16
17
|
# in all the right places.
|
17
|
-
|
18
|
-
|
19
|
-
|
18
|
+
exif_data = exif_from_tiff_io(io)
|
19
|
+
return unless exif_data
|
20
|
+
|
21
|
+
w = exif_data.image_width
|
22
|
+
h = exif_data.image_length
|
20
23
|
|
21
24
|
FormatParser::Image.new(
|
22
25
|
format: :tif,
|
23
|
-
width_px:
|
24
|
-
height_px:
|
25
|
-
|
26
|
-
|
26
|
+
width_px: w,
|
27
|
+
height_px: h,
|
28
|
+
display_width_px: exif_data.rotated? ? h : w,
|
29
|
+
display_height_px: exif_data.rotated? ? w : h,
|
30
|
+
orientation: exif_data.orientation,
|
31
|
+
intrinsics: {exif: exif_data},
|
27
32
|
)
|
28
33
|
rescue EXIFR::MalformedTIFF
|
29
34
|
nil
|
data/lib/read_limits_config.rb
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
# We need to apply various limits so that parsers do not over-read, do not cause too many HTTP
|
2
|
+
# requests to be dispatched and so on. These should be _balanced_ with one another- for example,
|
3
|
+
# we cannot tell a parser that it is limited to reading 1024 bytes while at the same time
|
4
|
+
# limiting the size of the cache pages it may slurp in to less than that amount, since
|
5
|
+
# it can quickly become frustrating. ReadLimitsConfig computes these limits
|
6
|
+
# for us, in a fairly balanced way, based on one setting.
|
1
7
|
class FormatParser::ReadLimitsConfig
|
2
8
|
MAX_PAGE_FAULTS = 16
|
3
9
|
|
@@ -42,6 +42,21 @@ describe FormatParser::AttributesJSON do
|
|
42
42
|
expect(readback[:some_infinity]).to be_nil
|
43
43
|
end
|
44
44
|
|
45
|
+
it 'converts NaN to nil' do
|
46
|
+
anon_class = Class.new do
|
47
|
+
include FormatParser::AttributesJSON
|
48
|
+
attr_accessor :some_nan
|
49
|
+
def some_nan
|
50
|
+
(1.0 / 0.0).to_f
|
51
|
+
end
|
52
|
+
end
|
53
|
+
instance = anon_class.new
|
54
|
+
output = JSON.dump(instance)
|
55
|
+
readback = JSON.parse(output, symbolize_names: true)
|
56
|
+
expect(readback).to have_key(:some_nan)
|
57
|
+
expect(readback[:some_nan]).to be_nil
|
58
|
+
end
|
59
|
+
|
45
60
|
it 'provides a default implementation of to_json as well' do
|
46
61
|
anon_class = Class.new do
|
47
62
|
include FormatParser::AttributesJSON
|
data/spec/format_parser_spec.rb
CHANGED
@@ -1,17 +1,31 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe FormatParser do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
describe '.parse' do
|
5
|
+
it 'returns nil when trying to parse an empty IO' do
|
6
|
+
d = StringIO.new('')
|
7
|
+
expect(FormatParser.parse(d)).to be_nil
|
8
|
+
end
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
10
|
+
it 'returns nil when parsing an IO no parser can make sense of' do
|
11
|
+
d = StringIO.new(Random.new.bytes(1))
|
12
|
+
expect(FormatParser.parse(d)).to be_nil
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'uses the passed ReadLimitsConfig and applies limits in it' do
|
16
|
+
conf = FormatParser::ReadLimitsConfig.new(16)
|
17
|
+
d = StringIO.new(Random.new.bytes(64 * 1024))
|
18
|
+
expect(FormatParser.parse(d, limits_config: conf)).to be_nil
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'parses our fixtures without raising any errors' do
|
22
|
+
Dir.glob(fixtures_dir + '/**/*.*').sort.each do |fixture_path|
|
23
|
+
File.open(fixture_path, 'rb') do |file|
|
24
|
+
FormatParser.parse(file)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
13
28
|
|
14
|
-
describe 'with fuzzing' do
|
15
29
|
it "returns either a valid result or a nil for all fuzzed inputs at seed #{RSpec.configuration.seed}" do
|
16
30
|
r = Random.new(RSpec.configuration.seed)
|
17
31
|
1024.times do
|
@@ -19,54 +33,54 @@ describe FormatParser do
|
|
19
33
|
FormatParser.parse(random_blob) # If there is an error in one of the parsers the example will raise too
|
20
34
|
end
|
21
35
|
end
|
22
|
-
end
|
23
36
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
37
|
+
it 'fails gracefully when a parser module reads more and more causing page faults and prevents too many reads on the source' do
|
38
|
+
exploit = ->(io) {
|
39
|
+
loop {
|
40
|
+
skip = 16 * 1024
|
41
|
+
io.read(1)
|
42
|
+
io.seek(io.pos + skip)
|
43
|
+
}
|
30
44
|
}
|
31
|
-
|
32
|
-
FormatParser.register_parser exploit, natures: :document, formats: :exploit
|
45
|
+
FormatParser.register_parser exploit, natures: :document, formats: :exploit
|
33
46
|
|
34
|
-
|
35
|
-
|
47
|
+
sample_io = StringIO.new(Random.new.bytes(1024 * 1024 * 8))
|
48
|
+
allow(sample_io).to receive(:read).and_call_original
|
36
49
|
|
37
|
-
|
50
|
+
result = FormatParser.parse(sample_io, formats: [:exploit])
|
38
51
|
|
39
|
-
|
40
|
-
|
52
|
+
expect(sample_io).to have_received(:read).at_most(16).times
|
53
|
+
expect(result).to be_nil
|
41
54
|
|
42
|
-
|
43
|
-
|
55
|
+
FormatParser.deregister_parser(exploit)
|
56
|
+
end
|
44
57
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
58
|
+
describe 'when multiple results are requested' do
|
59
|
+
let(:blob) { StringIO.new(Random.new.bytes(512 * 1024)) }
|
60
|
+
let(:audio) { FormatParser::Audio.new(format: :aiff, num_audio_channels: 1) }
|
61
|
+
let(:image) { FormatParser::Image.new(format: :dpx, width_px: 1, height_px: 1) }
|
49
62
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
subject { FormatParser.parse(blob, results: :all) }
|
63
|
+
context 'with :result=> :all (multiple results)' do
|
64
|
+
before do
|
65
|
+
expect_any_instance_of(FormatParser::AIFFParser).to receive(:call).and_return(audio)
|
66
|
+
expect_any_instance_of(FormatParser::DPXParser).to receive(:call).and_return(image)
|
67
|
+
end
|
57
68
|
|
58
|
-
|
59
|
-
it { is_expected.to include(audio) }
|
60
|
-
end
|
69
|
+
subject { FormatParser.parse(blob, results: :all) }
|
61
70
|
|
62
|
-
|
63
|
-
|
64
|
-
expect_any_instance_of(FormatParser::DPXParser).to receive(:call).and_return(image)
|
71
|
+
it { is_expected.to include(image) }
|
72
|
+
it { is_expected.to include(audio) }
|
65
73
|
end
|
66
74
|
|
67
|
-
|
75
|
+
context 'without :result=> :all (first result)' do
|
76
|
+
before do
|
77
|
+
expect_any_instance_of(FormatParser::DPXParser).to receive(:call).and_return(image)
|
78
|
+
end
|
79
|
+
|
80
|
+
subject { FormatParser.parse(blob) }
|
68
81
|
|
69
|
-
|
82
|
+
it { is_expected.to eq(image) }
|
83
|
+
end
|
70
84
|
end
|
71
85
|
end
|
72
86
|
|
@@ -76,19 +90,15 @@ describe FormatParser do
|
|
76
90
|
result = FormatParser.parse_file_at(path)
|
77
91
|
expect(result.nature).to eq(:audio)
|
78
92
|
end
|
79
|
-
end
|
80
93
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
FormatParser.parse(file)
|
86
|
-
end
|
87
|
-
end
|
94
|
+
it 'passes keyword arguments to parse()' do
|
95
|
+
path = fixtures_dir + '/WAV/c_M1F1-Alaw-AFsp.wav'
|
96
|
+
expect(FormatParser).to receive(:parse).with(an_instance_of(File), foo: :bar)
|
97
|
+
FormatParser.parse_file_at(path, foo: :bar)
|
88
98
|
end
|
89
99
|
end
|
90
100
|
|
91
|
-
describe 'parsers_for' do
|
101
|
+
describe '.parsers_for' do
|
92
102
|
it 'raises on an invalid request' do
|
93
103
|
expect {
|
94
104
|
FormatParser.parsers_for([:image], [:fdx])
|
@@ -111,7 +121,7 @@ describe FormatParser do
|
|
111
121
|
end
|
112
122
|
end
|
113
123
|
|
114
|
-
describe '
|
124
|
+
describe '.register_parser and .deregister_parser' do
|
115
125
|
it 'registers a parser for a certain nature and format' do
|
116
126
|
some_parser = ->(_io) { 'I parse EXRs! Whee!' }
|
117
127
|
|
@@ -130,4 +140,10 @@ describe FormatParser do
|
|
130
140
|
}.to raise_error(/No parsers provide/)
|
131
141
|
end
|
132
142
|
end
|
143
|
+
|
144
|
+
describe '.default_limits_config' do
|
145
|
+
it 'returns a ReadLimitsConfig object' do
|
146
|
+
expect(FormatParser.default_limits_config).to be_kind_of(FormatParser::ReadLimitsConfig)
|
147
|
+
end
|
148
|
+
end
|
133
149
|
end
|
@@ -1,29 +1,40 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe FormatParser::DPXParser do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
expect(parsed.format).to eq(:dpx)
|
12
|
-
|
13
|
-
# If we have an error in the struct offsets these values are likely to become
|
14
|
-
# the maximum value of a 4-byte uint, which is way higher
|
15
|
-
expect(parsed.width_px).to be_kind_of(Integer)
|
16
|
-
expect(parsed.width_px).to be_between(0, 2048)
|
17
|
-
expect(parsed.height_px).to be_kind_of(Integer)
|
18
|
-
expect(parsed.height_px).to be_between(0, 4000)
|
19
|
-
end
|
20
|
-
end
|
4
|
+
Dir.glob(fixtures_dir + '/dpx/*.*').each do |dpx_path|
|
5
|
+
it "is able to parse #{File.basename(dpx_path)}" do
|
6
|
+
parsed = subject.call(File.open(dpx_path, 'rb'))
|
7
|
+
|
8
|
+
expect(parsed).not_to be_nil
|
9
|
+
expect(parsed.nature).to eq(:image)
|
10
|
+
expect(parsed.format).to eq(:dpx)
|
21
11
|
|
22
|
-
|
23
|
-
|
24
|
-
parsed
|
25
|
-
expect(parsed.width_px).to
|
26
|
-
expect(parsed.height_px).to
|
12
|
+
# If we have an error in the struct offsets these values are likely to become
|
13
|
+
# the maximum value of a 4-byte uint, which is way higher
|
14
|
+
expect(parsed.width_px).to be_kind_of(Integer)
|
15
|
+
expect(parsed.width_px).to be_between(0, 2048)
|
16
|
+
expect(parsed.height_px).to be_kind_of(Integer)
|
17
|
+
expect(parsed.height_px).to be_between(0, 4000)
|
27
18
|
end
|
28
19
|
end
|
20
|
+
|
21
|
+
it 'correctly reads display dimensions corrected for the pixel aspect from the DPX header' do
|
22
|
+
fi = File.open(fixtures_dir + '/dpx/aspect_237_example.dpx', 'rb')
|
23
|
+
parsed = subject.call(fi)
|
24
|
+
|
25
|
+
expect(parsed.width_px).to eq(1920)
|
26
|
+
expect(parsed.height_px).to eq(1080)
|
27
|
+
|
28
|
+
expect(parsed.display_width_px).to eq(1920)
|
29
|
+
expect(parsed.display_height_px).to eq(810)
|
30
|
+
|
31
|
+
expect(parsed.display_width_px / parsed.display_height_px.to_f).to be_within(0.01).of(2.37)
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'does not explode on invalid inputs' do
|
35
|
+
invalid = StringIO.new('SDPX' + (' ' * 64))
|
36
|
+
expect {
|
37
|
+
subject.call(invalid)
|
38
|
+
}.to raise_error(FormatParser::IOUtils::InvalidRead)
|
39
|
+
end
|
29
40
|
end
|
@@ -1,17 +1,19 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe FormatParser::EXIFParser do
|
4
|
+
let(:subject) do
|
5
|
+
Object.new.tap { |o| o.extend FormatParser::EXIFParser }
|
6
|
+
end
|
7
|
+
|
4
8
|
describe 'is able to correctly parse orientation for all the TIFF EXIF examples from FastImage' do
|
5
9
|
Dir.glob(fixtures_dir + '/exif-orientation-testimages/tiff-*/*.tif').each do |tiff_path|
|
6
10
|
filename = File.basename(tiff_path)
|
7
11
|
it "is able to parse #{filename}" do
|
8
|
-
|
9
|
-
|
10
|
-
expect(
|
11
|
-
|
12
|
-
expect(parser.orientation).to be_kind_of(Symbol)
|
12
|
+
result = subject.exif_from_tiff_io(File.open(tiff_path, 'rb'))
|
13
|
+
expect(result).not_to be_nil
|
14
|
+
expect(result.orientation).to be_kind_of(Symbol)
|
13
15
|
# Filenames in this dir correspond with the orientation of the file
|
14
|
-
expect(filename.include
|
16
|
+
expect(filename).to include(result.orientation.to_s)
|
15
17
|
end
|
16
18
|
end
|
17
19
|
end
|
@@ -26,6 +26,8 @@ describe FormatParser::JPEGParser do
|
|
26
26
|
expect(parsed.orientation).to be_kind_of(Symbol)
|
27
27
|
expect(parsed.width_px).to be > 0
|
28
28
|
expect(parsed.height_px).to be > 0
|
29
|
+
expect(parsed.display_width_px).to eq(1240)
|
30
|
+
expect(parsed.display_height_px).to eq(1754)
|
29
31
|
end
|
30
32
|
|
31
33
|
bottom_left_path = fixtures_dir + '/exif-orientation-testimages/jpg/bottom_left.jpg'
|
@@ -33,20 +35,30 @@ describe FormatParser::JPEGParser do
|
|
33
35
|
expect(parsed.orientation).to eq(:bottom_left)
|
34
36
|
expect(parsed.width_px).to eq(1240)
|
35
37
|
expect(parsed.height_px).to eq(1754)
|
38
|
+
expect(parsed.display_width_px).to eq(1240)
|
39
|
+
expect(parsed.display_height_px).to eq(1754)
|
40
|
+
expect(parsed.intrinsics[:exif]).not_to be_nil
|
36
41
|
|
37
42
|
top_right_path = fixtures_dir + '/exif-orientation-testimages/jpg/right_bottom.jpg'
|
38
43
|
parsed = subject.call(File.open(top_right_path, 'rb'))
|
39
44
|
expect(parsed.orientation).to eq(:right_bottom)
|
40
45
|
expect(parsed.width_px).to eq(1754)
|
41
46
|
expect(parsed.height_px).to eq(1240)
|
47
|
+
expect(parsed.display_width_px).to eq(1240)
|
48
|
+
expect(parsed.display_height_px).to eq(1754)
|
49
|
+
expect(parsed.intrinsics[:exif]).not_to be_nil
|
42
50
|
end
|
43
51
|
|
44
52
|
it 'gives true pixel dimensions priority over pixel dimensions in EXIF tags' do
|
45
53
|
jpeg_path = fixtures_dir + '/JPEG/divergent_pixel_dimensions_exif.jpg'
|
46
54
|
result = subject.call(File.open(jpeg_path, 'rb'))
|
55
|
+
|
47
56
|
expect(result.width_px).to eq(1920)
|
48
57
|
expect(result.height_px).to eq(1280)
|
49
|
-
|
58
|
+
|
59
|
+
exif = result.intrinsics.fetch(:exif)
|
60
|
+
expect(exif.pixel_x_dimension).to eq(8214)
|
61
|
+
expect(exif.pixel_y_dimension).to eq(5476)
|
50
62
|
end
|
51
63
|
|
52
64
|
it 'reads an example with many APP1 markers at the beginning of which none are EXIF' do
|
@@ -44,6 +44,7 @@ describe FormatParser::TIFFParser do
|
|
44
44
|
expect(parsed).not_to be_nil
|
45
45
|
expect(parsed.width_px).to eq(320)
|
46
46
|
expect(parsed.height_px).to eq(240)
|
47
|
+
expect(parsed.intrinsics[:exif]).not_to be_nil
|
47
48
|
end
|
48
49
|
|
49
50
|
describe 'correctly extracts dimensions from various TIFF flavors of the same file' do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: format_parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Noah Berman
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-04-
|
12
|
+
date: 2018-04-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: ks
|
@@ -177,6 +177,7 @@ files:
|
|
177
177
|
- lib/parsers/bmp_parser.rb
|
178
178
|
- lib/parsers/cr2_parser.rb
|
179
179
|
- lib/parsers/dpx_parser.rb
|
180
|
+
- lib/parsers/dpx_parser/dpx_structs.rb
|
180
181
|
- lib/parsers/exif_parser.rb
|
181
182
|
- lib/parsers/fdx_parser.rb
|
182
183
|
- lib/parsers/flac_parser.rb
|