marcel 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +26 -6
- data/SECURITY.md +7 -0
- data/lib/marcel/magic/definitions.rb +39 -0
- data/lib/marcel/magic/xml.rb +488 -0
- data/lib/marcel/magic/zip.rb +201 -0
- data/lib/marcel/magic.rb +204 -26
- data/lib/marcel/mime_type/definitions.rb +31 -40
- data/lib/marcel/mime_type.rb +60 -14
- data/lib/marcel/tables.rb +259 -24
- data/lib/marcel/version.rb +1 -1
- metadata +9 -19
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Marcel
|
|
4
|
+
class Magic
|
|
5
|
+
# Procedural refinement of ZIP-based container types.
|
|
6
|
+
#
|
|
7
|
+
# The declarative magic tables scan a bounded prefix of the file (matchers are capped at
|
|
8
|
+
# 64KB offsets when the tables are generated), so ZIP containers whose distinguishing
|
|
9
|
+
# members are catalogued later in the archive — such as OOXML documents with
|
|
10
|
+
# [Content_Types].xml stored past the 64KB mark — fall back to their generic container
|
|
11
|
+
# type. This module instead parses the ZIP central directory, a bounded read from the end
|
|
12
|
+
# of the file, to recover the specific document type, including macro-enabled variants,
|
|
13
|
+
# which no prefix bytes can distinguish.
|
|
14
|
+
module Zip
|
|
15
|
+
EOCD_SIGNATURE = "PK\x05\x06".b
|
|
16
|
+
ZIP64_EOCD_SIGNATURE = "PK\x06\x06".b
|
|
17
|
+
ZIP64_EOCD_LOCATOR_SIGNATURE = "PK\x06\x07".b
|
|
18
|
+
CENTRAL_DIRECTORY_SIGNATURE = "PK\x01\x02".b
|
|
19
|
+
|
|
20
|
+
EOCD_SIZE = 22
|
|
21
|
+
ZIP64_EOCD_SIZE = 56
|
|
22
|
+
ZIP64_EOCD_LOCATOR_SIZE = 20
|
|
23
|
+
MAX_COMMENT_SIZE = 0xFFFF
|
|
24
|
+
|
|
25
|
+
# The end-of-central-directory record sits at most a maximal comment from the end of
|
|
26
|
+
# the file, possibly preceded by a Zip64 locator.
|
|
27
|
+
MAX_EOCD_SEARCH = EOCD_SIZE + MAX_COMMENT_SIZE + ZIP64_EOCD_LOCATOR_SIZE
|
|
28
|
+
|
|
29
|
+
# A malformed trailer full of EOCD signature bytes could otherwise force a quadratic
|
|
30
|
+
# backward scan; real archives find the record on the first candidate.
|
|
31
|
+
MAX_EOCD_CANDIDATES = 64
|
|
32
|
+
|
|
33
|
+
CENTRAL_DIRECTORY_HEADER_SIZE = 46
|
|
34
|
+
MAX_CENTRAL_DIRECTORY_READ = 1 << 20
|
|
35
|
+
MAX_ENTRIES = 8192
|
|
36
|
+
|
|
37
|
+
# Container types a central directory listing can make more specific.
|
|
38
|
+
REFINABLE_TYPES = [
|
|
39
|
+
"application/zip",
|
|
40
|
+
"application/x-tika-ooxml",
|
|
41
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
42
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
43
|
+
"application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
|
44
|
+
].freeze
|
|
45
|
+
|
|
46
|
+
# Standardised OPC part names identifying each OOXML document family:
|
|
47
|
+
# main part, macro part, and the types each implies.
|
|
48
|
+
DOCUMENT_FAMILIES = [
|
|
49
|
+
[ "word/document.xml", "word/vbaProject.bin",
|
|
50
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
51
|
+
"application/vnd.ms-word.document.macroenabled.12" ],
|
|
52
|
+
[ "xl/workbook.xml", "xl/vbaProject.bin",
|
|
53
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
54
|
+
"application/vnd.ms-excel.sheet.macroenabled.12" ],
|
|
55
|
+
[ "ppt/presentation.xml", "ppt/vbaProject.bin",
|
|
56
|
+
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
57
|
+
"application/vnd.ms-powerpoint.presentation.macroenabled.12" ]
|
|
58
|
+
].freeze
|
|
59
|
+
|
|
60
|
+
DISCRIMINATING_PARTS = DOCUMENT_FAMILIES.flat_map { |main, macro, *| [main, macro] }.freeze
|
|
61
|
+
|
|
62
|
+
class << self
|
|
63
|
+
# Returns a more specific type than +base_type+ if the IO's ZIP central directory
|
|
64
|
+
# identifies one, or +base_type+ unchanged. Unseekable IOs, partial reads and
|
|
65
|
+
# malformed archives refine nothing: the base type stands.
|
|
66
|
+
def refine(io, base_type)
|
|
67
|
+
return base_type unless REFINABLE_TYPES.include?(base_type)
|
|
68
|
+
|
|
69
|
+
io = StringIO.new(io.to_s) unless io.respond_to?(:read)
|
|
70
|
+
return base_type unless io.respond_to?(:seek) && io.respond_to?(:size)
|
|
71
|
+
|
|
72
|
+
parts = begin
|
|
73
|
+
discriminating_parts(io)
|
|
74
|
+
rescue StandardError
|
|
75
|
+
nil
|
|
76
|
+
ensure
|
|
77
|
+
begin
|
|
78
|
+
io.rewind
|
|
79
|
+
rescue StandardError
|
|
80
|
+
nil
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
parts ? classify(parts, base_type) : base_type
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
private
|
|
88
|
+
|
|
89
|
+
def classify(parts, base_type)
|
|
90
|
+
DOCUMENT_FAMILIES.each do |main_part, macro_part, type, macro_type|
|
|
91
|
+
return parts.include?(macro_part) ? macro_type : type if parts.include?(main_part)
|
|
92
|
+
end
|
|
93
|
+
base_type
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Reads the archive's central directory and returns the OOXML part names it
|
|
97
|
+
# catalogues, or nil if no well-formed central directory is found.
|
|
98
|
+
def discriminating_parts(io)
|
|
99
|
+
size = io.size
|
|
100
|
+
return nil unless size.is_a?(Integer) && size >= EOCD_SIZE
|
|
101
|
+
|
|
102
|
+
tail_size = [size, MAX_EOCD_SEARCH].min
|
|
103
|
+
io.seek(size - tail_size)
|
|
104
|
+
tail = io.read(tail_size)
|
|
105
|
+
return nil unless tail && tail.bytesize == tail_size
|
|
106
|
+
tail = tail.dup.force_encoding(Encoding::BINARY)
|
|
107
|
+
|
|
108
|
+
eocd_pos = locate_eocd(tail)
|
|
109
|
+
return nil unless eocd_pos
|
|
110
|
+
|
|
111
|
+
entries, directory_size, directory_offset = parse_eocd(io, tail, eocd_pos)
|
|
112
|
+
return nil unless entries && directory_offset + directory_size <= size
|
|
113
|
+
|
|
114
|
+
read_parts(io, entries, directory_size, directory_offset)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Scans backward through the file's tail for the end-of-central-directory record,
|
|
118
|
+
# validating each candidate signature against its comment length.
|
|
119
|
+
def locate_eocd(tail)
|
|
120
|
+
pos = tail.bytesize - EOCD_SIZE
|
|
121
|
+
MAX_EOCD_CANDIDATES.times do
|
|
122
|
+
return nil unless pos && pos >= 0 && (pos = tail.rindex(EOCD_SIGNATURE, pos))
|
|
123
|
+
comment_size = tail[pos + 20, 2].unpack1("v")
|
|
124
|
+
return pos if pos + EOCD_SIZE + comment_size == tail.bytesize
|
|
125
|
+
pos -= 1
|
|
126
|
+
end
|
|
127
|
+
nil
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Returns [entry count, central directory size, central directory offset], following
|
|
131
|
+
# the Zip64 records when the classic fields overflow. Multi-disk archives and
|
|
132
|
+
# malformed records return nil.
|
|
133
|
+
def parse_eocd(io, tail, eocd_pos)
|
|
134
|
+
disk, directory_disk, entries, directory_size, directory_offset =
|
|
135
|
+
tail[eocd_pos + 4, 16].unpack("vvx2vVV")
|
|
136
|
+
return nil unless disk == 0 && directory_disk == 0
|
|
137
|
+
|
|
138
|
+
if entries == 0xFFFF || directory_size == 0xFFFFFFFF || directory_offset == 0xFFFFFFFF
|
|
139
|
+
parse_zip64_eocd(io, tail, eocd_pos)
|
|
140
|
+
else
|
|
141
|
+
[entries, directory_size, directory_offset]
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def parse_zip64_eocd(io, tail, eocd_pos)
|
|
146
|
+
locator_pos = eocd_pos - ZIP64_EOCD_LOCATOR_SIZE
|
|
147
|
+
return nil if locator_pos < 0 || tail[locator_pos, 4] != ZIP64_EOCD_LOCATOR_SIGNATURE
|
|
148
|
+
|
|
149
|
+
locator_disk, zip64_eocd_offset, total_disks = tail[locator_pos + 4, 16].unpack("VQ<V")
|
|
150
|
+
return nil unless locator_disk == 0 && total_disks == 1
|
|
151
|
+
|
|
152
|
+
io.seek(zip64_eocd_offset)
|
|
153
|
+
record = io.read(ZIP64_EOCD_SIZE)
|
|
154
|
+
return nil unless record && record.bytesize == ZIP64_EOCD_SIZE
|
|
155
|
+
record = record.dup.force_encoding(Encoding::BINARY)
|
|
156
|
+
return nil unless record[0, 4] == ZIP64_EOCD_SIGNATURE
|
|
157
|
+
|
|
158
|
+
disk, directory_disk, _, entries, directory_size, directory_offset =
|
|
159
|
+
record[16, 40].unpack("VVQ<Q<Q<Q<")
|
|
160
|
+
return nil unless disk == 0 && directory_disk == 0
|
|
161
|
+
|
|
162
|
+
[entries, directory_size, directory_offset]
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Walks central directory entries, collecting the OOXML part names that
|
|
166
|
+
# discriminate between document families. Reads and entry counts are bounded;
|
|
167
|
+
# the walk stops early once a family and its macro part are both seen.
|
|
168
|
+
def read_parts(io, entries, directory_size, directory_offset)
|
|
169
|
+
io.seek(directory_offset)
|
|
170
|
+
directory = io.read([directory_size, MAX_CENTRAL_DIRECTORY_READ].min)
|
|
171
|
+
return nil unless directory
|
|
172
|
+
directory = directory.dup.force_encoding(Encoding::BINARY)
|
|
173
|
+
|
|
174
|
+
parts = []
|
|
175
|
+
pos = 0
|
|
176
|
+
[entries, MAX_ENTRIES].min.times do
|
|
177
|
+
break unless pos + CENTRAL_DIRECTORY_HEADER_SIZE <= directory.bytesize &&
|
|
178
|
+
directory[pos, 4] == CENTRAL_DIRECTORY_SIGNATURE
|
|
179
|
+
|
|
180
|
+
name_size, extra_size, comment_size = directory[pos + 28, 6].unpack("vvv")
|
|
181
|
+
name = directory[pos + CENTRAL_DIRECTORY_HEADER_SIZE, name_size]
|
|
182
|
+
break unless name && name.bytesize == name_size
|
|
183
|
+
|
|
184
|
+
parts << name if DISCRIMINATING_PARTS.include?(name)
|
|
185
|
+
break if complete?(parts)
|
|
186
|
+
|
|
187
|
+
pos += CENTRAL_DIRECTORY_HEADER_SIZE + name_size + extra_size + comment_size
|
|
188
|
+
end
|
|
189
|
+
parts
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# A main part plus its macro part is as specific as classification gets.
|
|
193
|
+
def complete?(parts)
|
|
194
|
+
DOCUMENT_FAMILIES.any? do |main_part, macro_part, *|
|
|
195
|
+
parts.include?(main_part) && parts.include?(macro_part)
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
end
|
data/lib/marcel/magic.rb
CHANGED
|
@@ -25,25 +25,79 @@ module Marcel
|
|
|
25
25
|
# Option keys:
|
|
26
26
|
# * <i>:extensions</i>: String list or single string of file extensions
|
|
27
27
|
# * <i>:parents</i>: String list or single string of parent mime types
|
|
28
|
+
# * <i>:aliases</i>: String list or single string of aliased mime types
|
|
28
29
|
# * <i>:magic</i>: Mime magic specification
|
|
29
30
|
# * <i>:comment</i>: Comment string
|
|
30
31
|
def self.add(type, options)
|
|
32
|
+
# Validate the complete registration before mutating any table, so a rejected
|
|
33
|
+
# registration leaves every registry untouched.
|
|
34
|
+
#
|
|
35
|
+
# Alias keys are never registered types and alias values never alias keys, so
|
|
36
|
+
# resolution is single-hop by construction: aliasing a registered type is rejected
|
|
37
|
+
# here, and canonicalize (the sanctioned path) re-points existing aliases itself.
|
|
38
|
+
aliases = [options[:aliases]].flatten.compact.map(&:downcase) - [type.downcase]
|
|
39
|
+
aliases.each do |aliased|
|
|
40
|
+
if TYPE_EXTS.key?(aliased) || TYPE_PARENTS.key?(aliased) || MAGIC.any? { |t, _| t == aliased }
|
|
41
|
+
raise ArgumentError, "#{aliased} is a registered type; use canonicalize to alias it to #{type}"
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
31
45
|
extensions = [options[:extensions]].flatten.compact
|
|
32
46
|
TYPE_EXTS[type] = extensions
|
|
47
|
+
extensions.each {|ext| EXTENSIONS[ext] = type }
|
|
48
|
+
|
|
49
|
+
TYPE_ALIASES.delete(type)
|
|
50
|
+
aliases.each {|aliased| TYPE_ALIASES[aliased] = type }
|
|
51
|
+
|
|
33
52
|
parents = [options[:parents]].flatten.compact
|
|
34
53
|
TYPE_PARENTS[type] = parents unless parents.empty?
|
|
35
|
-
|
|
54
|
+
|
|
36
55
|
MAGIC.unshift [type, options[:magic]] if options[:magic]
|
|
37
56
|
end
|
|
38
57
|
|
|
39
|
-
#
|
|
58
|
+
# Renames a canonical type: the +instead_of+ type's extensions, magic matchers, parents,
|
|
59
|
+
# and aliases are re-registered under +type+, and the old name becomes an alias of the
|
|
60
|
+
# new. Useful when a historical or de facto type is preferable to the canonical type
|
|
61
|
+
# shipped in the generated tables, without giving up its matchers.
|
|
62
|
+
def self.canonicalize(type, instead_of:)
|
|
63
|
+
raise ArgumentError, "#{instead_of} is an alias, not canonical" if TYPE_ALIASES[instead_of]
|
|
64
|
+
|
|
65
|
+
# Displace whatever the new canonical type was registered as before.
|
|
66
|
+
remove(type)
|
|
67
|
+
|
|
68
|
+
# Re-register the old canonical type's dictionary under the new name.
|
|
69
|
+
EXTENSIONS.select { |_, existing| existing == instead_of }.each_key do |ext|
|
|
70
|
+
EXTENSIONS[ext] = type
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
if extensions = TYPE_EXTS.delete(instead_of)
|
|
74
|
+
TYPE_EXTS[type] = extensions
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
TYPE_ALIASES.select { |_, canonical| canonical == instead_of }.each_key do |aliased|
|
|
78
|
+
TYPE_ALIASES[aliased] = type
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
if parents = TYPE_PARENTS.delete(instead_of)
|
|
82
|
+
TYPE_PARENTS[type] = parents
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
MAGIC.each { |pair| pair[0] = type if pair[0] == instead_of }
|
|
86
|
+
|
|
87
|
+
# Alias the old canonical type to the new.
|
|
88
|
+
TYPE_ALIASES[instead_of] = type
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Removes a mime type from the dictionary. You might want to do this if
|
|
40
92
|
# you're seeing impossible conflicts (for instance, application/x-gmc-link).
|
|
41
|
-
# * <i>type</i>: The mime type to remove.
|
|
93
|
+
# * <i>type</i>: The mime type to remove. All associated extensions, magic,
|
|
94
|
+
# and aliases are removed too.
|
|
42
95
|
def self.remove(type)
|
|
43
96
|
EXTENSIONS.delete_if {|ext, t| t == type }
|
|
44
97
|
MAGIC.delete_if {|t, m| t == type }
|
|
45
98
|
TYPE_EXTS.delete(type)
|
|
46
99
|
TYPE_PARENTS.delete(type)
|
|
100
|
+
TYPE_ALIASES.delete_if {|aliased, canonical| aliased == type || canonical == type }
|
|
47
101
|
end
|
|
48
102
|
|
|
49
103
|
# Returns true if type is a text format
|
|
@@ -64,14 +118,31 @@ module Marcel
|
|
|
64
118
|
TYPE_EXTS[type] || []
|
|
65
119
|
end
|
|
66
120
|
|
|
121
|
+
# Resolve an aliased type to its canonical type; canonical types return themselves
|
|
122
|
+
def canonical
|
|
123
|
+
if canonical_type = TYPE_ALIASES[type]
|
|
124
|
+
self.class.new(canonical_type)
|
|
125
|
+
else
|
|
126
|
+
self
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
67
130
|
# Get mime comment
|
|
68
131
|
def comment
|
|
69
132
|
nil # deprecated
|
|
70
133
|
end
|
|
71
134
|
|
|
135
|
+
# Lookup canonical mime type by mime type string, resolving aliases
|
|
136
|
+
def self.by_type(type)
|
|
137
|
+
new(canonical(type)) if type
|
|
138
|
+
end
|
|
139
|
+
|
|
72
140
|
# Lookup mime type by file extension
|
|
73
141
|
def self.by_extension(ext)
|
|
74
|
-
ext = ext.to_s
|
|
142
|
+
ext = ext.to_s
|
|
143
|
+
return unless ext.valid_encoding?
|
|
144
|
+
|
|
145
|
+
ext = ext.downcase
|
|
75
146
|
mime = ext[0..0] == '.' ? EXTENSIONS[ext[1..-1]] : EXTENSIONS[ext]
|
|
76
147
|
mime && new(mime)
|
|
77
148
|
end
|
|
@@ -79,6 +150,8 @@ module Marcel
|
|
|
79
150
|
# Lookup mime type by filename
|
|
80
151
|
def self.by_path(path)
|
|
81
152
|
by_extension(File.extname(path))
|
|
153
|
+
rescue ArgumentError, EncodingError
|
|
154
|
+
nil
|
|
82
155
|
end
|
|
83
156
|
|
|
84
157
|
# Lookup mime type by magic content analysis.
|
|
@@ -111,46 +184,151 @@ module Marcel
|
|
|
111
184
|
alias == eql?
|
|
112
185
|
|
|
113
186
|
def self.child?(child, parent)
|
|
114
|
-
|
|
187
|
+
parent = canonical(parent)
|
|
188
|
+
pending = [child]
|
|
189
|
+
visited = {}
|
|
190
|
+
|
|
191
|
+
until pending.empty?
|
|
192
|
+
type = canonical(pending.pop)
|
|
193
|
+
return true if type == parent
|
|
194
|
+
next if visited[type]
|
|
195
|
+
|
|
196
|
+
visited[type] = true
|
|
197
|
+
pending.concat(TYPE_PARENTS[type] || [])
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
false
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# Resolve an aliased type string to its canonical type string
|
|
204
|
+
def self.canonical(type)
|
|
205
|
+
if type
|
|
206
|
+
# Allocation-free for already-lowercase input: child? resolves every node it visits.
|
|
207
|
+
type = type.downcase if /[A-Z]/.match?(type)
|
|
208
|
+
TYPE_ALIASES[type] || type
|
|
209
|
+
end
|
|
115
210
|
end
|
|
116
211
|
|
|
117
212
|
def self.magic_match(io, method)
|
|
213
|
+
if defined?(Pathname) && io.is_a?(Pathname)
|
|
214
|
+
return io.open("rb") { |file| magic_match(file, method) }
|
|
215
|
+
end
|
|
216
|
+
|
|
118
217
|
return magic_match(StringIO.new(io.to_s), method) unless io.respond_to?(:read)
|
|
119
218
|
|
|
120
219
|
buffer = "".b
|
|
121
|
-
|
|
220
|
+
read_mode = read_mode(io)
|
|
221
|
+
io.rewind
|
|
222
|
+
body_failed = true
|
|
223
|
+
begin
|
|
224
|
+
result = MAGIC.send(method) { |type, matches| magic_match_io(io, matches, buffer, read_mode) }
|
|
225
|
+
body_failed = false
|
|
226
|
+
result
|
|
227
|
+
ensure
|
|
228
|
+
begin
|
|
229
|
+
io.rewind
|
|
230
|
+
rescue Exception # Preserve an exception already raised while reading or matching.
|
|
231
|
+
raise unless body_failed
|
|
232
|
+
end
|
|
233
|
+
end
|
|
122
234
|
end
|
|
123
235
|
|
|
124
|
-
def self.magic_match_io(io, matches, buffer)
|
|
236
|
+
def self.magic_match_io(io, matches, buffer, mode = read_mode(io))
|
|
125
237
|
matches.any? do |offset, value, children|
|
|
126
|
-
match =
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
else
|
|
135
|
-
io.read(offset, buffer)
|
|
136
|
-
io.read(value.bytesize, buffer) == value
|
|
238
|
+
match = if value
|
|
239
|
+
is_range = Range === offset
|
|
240
|
+
is_regexp = Regexp === value
|
|
241
|
+
sample_size = is_regexp ? 256 : value.bytesize
|
|
242
|
+
|
|
243
|
+
x = if is_range
|
|
244
|
+
if io_seek(io, offset.begin, buffer, mode)
|
|
245
|
+
io_read(io, offset.end - offset.begin + sample_size, buffer, mode)
|
|
137
246
|
end
|
|
247
|
+
else
|
|
248
|
+
if io_seek(io, offset, buffer, mode)
|
|
249
|
+
io_read(io, sample_size, buffer, mode)
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
x.force_encoding(Encoding::BINARY) if x
|
|
253
|
+
|
|
254
|
+
if is_regexp
|
|
255
|
+
x&.match?(value)
|
|
256
|
+
elsif is_range
|
|
257
|
+
x&.include?(value)
|
|
258
|
+
else
|
|
259
|
+
x == value
|
|
138
260
|
end
|
|
261
|
+
end
|
|
139
262
|
|
|
140
263
|
io.rewind
|
|
141
|
-
match && (!children || magic_match_io(io, children, buffer))
|
|
264
|
+
match && (!children || magic_match_io(io, children, buffer, mode))
|
|
142
265
|
end
|
|
143
266
|
end
|
|
144
267
|
|
|
145
|
-
def self.
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
268
|
+
def self.io_seek(io, offset, buffer, mode)
|
|
269
|
+
return true if offset == 0
|
|
270
|
+
|
|
271
|
+
if offset < 0
|
|
272
|
+
return false unless io.respond_to?(:size)
|
|
150
273
|
|
|
151
|
-
|
|
274
|
+
offset = io.size + offset
|
|
275
|
+
return false if offset < 0
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
if io.respond_to?(:seek)
|
|
279
|
+
io.seek(offset, IO::SEEK_SET)
|
|
280
|
+
else
|
|
281
|
+
# Some IOs don't support `seek`. e.g. Rack::RewindableInput
|
|
282
|
+
skipped = io_read(io, offset, buffer, mode)
|
|
283
|
+
return false unless skipped && skipped.bytesize == offset
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
true
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
def self.io_read(io, length, buffer, mode)
|
|
290
|
+
return io.read(length, buffer) if mode == :native
|
|
291
|
+
|
|
292
|
+
buffer.clear
|
|
293
|
+
|
|
294
|
+
read_with_buffer = mode == :buffer
|
|
295
|
+
chunk = read_with_buffer ? io.read(length, buffer) : io.read(length)
|
|
296
|
+
return if chunk && chunk.bytesize > length
|
|
297
|
+
buffer.replace(chunk) if chunk && !chunk.equal?(buffer)
|
|
298
|
+
|
|
299
|
+
return if chunk.nil? || buffer.empty?
|
|
300
|
+
return buffer if buffer.bytesize == length
|
|
301
|
+
|
|
302
|
+
continuation_buffer = "".b if read_with_buffer
|
|
303
|
+
while buffer.bytesize < length
|
|
304
|
+
remaining = length - buffer.bytesize
|
|
305
|
+
chunk = if read_with_buffer
|
|
306
|
+
io.read(remaining, continuation_buffer.clear)
|
|
307
|
+
else
|
|
308
|
+
io.read(remaining)
|
|
309
|
+
end
|
|
310
|
+
return if chunk && chunk.bytesize > remaining
|
|
311
|
+
break if chunk.nil? || chunk.empty?
|
|
312
|
+
|
|
313
|
+
buffer << chunk
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
buffer unless buffer.empty?
|
|
152
317
|
end
|
|
153
318
|
|
|
154
|
-
|
|
319
|
+
def self.read_mode(io)
|
|
320
|
+
return :native if io.is_a?(IO) || io.is_a?(StringIO)
|
|
321
|
+
|
|
322
|
+
parameters = io.method(:read).parameters
|
|
323
|
+
supports_buffer = parameters.any? { |kind,| kind == :rest } ||
|
|
324
|
+
parameters.count { |kind,| kind == :req || kind == :opt } >= 2
|
|
325
|
+
supports_buffer ? :buffer : :single
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
private_class_method :magic_match, :magic_match_io, :io_seek, :io_read, :read_mode
|
|
155
329
|
end
|
|
156
330
|
end
|
|
331
|
+
|
|
332
|
+
require "marcel/magic/definitions"
|
|
333
|
+
require "marcel/magic/xml"
|
|
334
|
+
require "marcel/magic/zip"
|
|
@@ -2,29 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
Marcel::MimeType.extend "text/plain", extensions: %w( txt asc )
|
|
4
4
|
|
|
5
|
-
Marcel::Magic.remove("text/html")
|
|
6
|
-
Marcel::MimeType.extend "text/html",
|
|
7
|
-
extensions: %w( html htm ),
|
|
8
|
-
magic: [
|
|
9
|
-
[0, "<!DOCTYPE html"],
|
|
10
|
-
[0, "<!DOCTYPE HTML"],
|
|
11
|
-
[0, "<!doctype html"],
|
|
12
|
-
[0, "<!doctype HTML"],
|
|
13
|
-
[0, "<html"],
|
|
14
|
-
[0, "<HTML"],
|
|
15
|
-
[0, " <!DOCTYPE html"],
|
|
16
|
-
[0, "\n<!DOCTYPE html"],
|
|
17
|
-
[0, "\r<!DOCTYPE html"],
|
|
18
|
-
[0, "\r\n<!DOCTYPE html"],
|
|
19
|
-
[0, "\t<!DOCTYPE html"],
|
|
20
|
-
[0, " <html"],
|
|
21
|
-
[0, "\n<html"],
|
|
22
|
-
[0, "\r<html"],
|
|
23
|
-
[0, "\r\n<html"],
|
|
24
|
-
[0, "\t<html"]
|
|
25
|
-
]
|
|
26
|
-
|
|
27
|
-
Marcel::MimeType.extend "application/illustrator", parents: "application/pdf"
|
|
28
5
|
Marcel::MimeType.extend "image/vnd.adobe.photoshop", magic: [[0, "8BPS"]], extensions: %w( psd psb )
|
|
29
6
|
|
|
30
7
|
Marcel::MimeType.extend "application/vnd.ms-excel", parents: "application/x-ole-storage"
|
|
@@ -33,47 +10,61 @@ Marcel::MimeType.extend "application/vnd.ms-powerpoint", parents: "application/x
|
|
|
33
10
|
Marcel::MimeType.extend "application/vnd.openxmlformats-officedocument.wordprocessingml.document", parents: "application/zip"
|
|
34
11
|
Marcel::MimeType.extend "application/vnd.openxmlformats-officedocument.wordprocessingml.template", parents: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
|
35
12
|
Marcel::MimeType.extend "application/vnd.ms-word.document.macroenabled.12", parents: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
|
36
|
-
Marcel::MimeType.extend "application/vnd.ms-word.template.macroenabled.12", parents:
|
|
13
|
+
Marcel::MimeType.extend "application/vnd.ms-word.template.macroenabled.12", parents: %w( application/vnd.openxmlformats-officedocument.wordprocessingml.document application/vnd.ms-word.document.macroenabled.12 )
|
|
37
14
|
|
|
38
15
|
Marcel::MimeType.extend "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", parents: "application/zip"
|
|
39
16
|
Marcel::MimeType.extend "application/vnd.openxmlformats-officedocument.spreadsheetml.template", parents: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
40
17
|
Marcel::MimeType.extend "application/vnd.ms-excel.sheet.macroenabled.12", parents: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
41
|
-
Marcel::MimeType.extend "application/vnd.ms-excel.template.macroenabled.12", parents:
|
|
42
|
-
Marcel::MimeType.extend "application/vnd.ms-excel.addin.macroenabled.12", parents:
|
|
18
|
+
Marcel::MimeType.extend "application/vnd.ms-excel.template.macroenabled.12", parents: %w( application/vnd.openxmlformats-officedocument.spreadsheetml.sheet application/vnd.ms-excel.sheet.macroenabled.12 )
|
|
19
|
+
Marcel::MimeType.extend "application/vnd.ms-excel.addin.macroenabled.12", parents: %w( application/vnd.openxmlformats-officedocument.spreadsheetml.sheet application/vnd.ms-excel.sheet.macroenabled.12 )
|
|
43
20
|
Marcel::MimeType.extend "application/vnd.ms-excel.sheet.binary.macroenabled.12", parents: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
44
21
|
|
|
45
22
|
Marcel::MimeType.extend "application/vnd.openxmlformats-officedocument.presentationml.presentation", parents: "application/zip"
|
|
46
23
|
Marcel::MimeType.extend "application/vnd.openxmlformats-officedocument.presentationml.template", parents: "application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
|
47
24
|
Marcel::MimeType.extend "application/vnd.openxmlformats-officedocument.presentationml.slideshow", parents: "application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
|
48
|
-
Marcel::MimeType.extend "application/vnd.ms-powerpoint.addin.macroenabled.12", parents:
|
|
25
|
+
Marcel::MimeType.extend "application/vnd.ms-powerpoint.addin.macroenabled.12", parents: %w( application/vnd.openxmlformats-officedocument.presentationml.presentation application/vnd.ms-powerpoint.presentation.macroenabled.12 )
|
|
49
26
|
Marcel::MimeType.extend "application/vnd.ms-powerpoint.presentation.macroenabled.12", parents: "application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
|
50
|
-
Marcel::MimeType.extend "application/vnd.ms-powerpoint.template.macroenabled.12", parents:
|
|
51
|
-
Marcel::MimeType.extend "application/vnd.ms-powerpoint.slideshow.macroenabled.12", parents:
|
|
27
|
+
Marcel::MimeType.extend "application/vnd.ms-powerpoint.template.macroenabled.12", parents: %w( application/vnd.openxmlformats-officedocument.presentationml.presentation application/vnd.ms-powerpoint.presentation.macroenabled.12 )
|
|
28
|
+
Marcel::MimeType.extend "application/vnd.ms-powerpoint.slideshow.macroenabled.12", parents: %w( application/vnd.openxmlformats-officedocument.presentationml.presentation application/vnd.ms-powerpoint.presentation.macroenabled.12 )
|
|
52
29
|
|
|
53
|
-
Marcel::MimeType.extend "application/vnd.apple.pages",
|
|
54
|
-
Marcel::MimeType.extend "application/vnd.apple.numbers",
|
|
55
|
-
Marcel::MimeType.extend "application/vnd.apple.keynote",
|
|
30
|
+
Marcel::MimeType.extend "application/vnd.apple.pages", parents: "application/zip"
|
|
31
|
+
Marcel::MimeType.extend "application/vnd.apple.numbers", parents: "application/zip"
|
|
32
|
+
Marcel::MimeType.extend "application/vnd.apple.keynote", parents: "application/zip"
|
|
56
33
|
|
|
57
|
-
Marcel::MimeType.extend "audio/aac", extensions: %w( aac ), parents: "audio/x-aac"
|
|
58
34
|
Marcel::MimeType.extend("audio/ogg", extensions: %w( opus ), magic: [[0, 'OggS', [[28, 'OpusHead']]]])
|
|
59
35
|
Marcel::MimeType.extend("audio/ogg", extensions: %w( ogg oga ), magic: [[0, 'OggS', [[29, 'vorbis']]]])
|
|
60
36
|
|
|
37
|
+
# Prefer the types browsers actually use over Tika's historical canonical types,
|
|
38
|
+
# keeping their file extensions and magic byte matchers.
|
|
39
|
+
Marcel::MimeType.canonicalize "audio/aac", instead_of: "audio/x-aac"
|
|
40
|
+
Marcel::MimeType.canonicalize "audio/flac", instead_of: "audio/x-flac"
|
|
41
|
+
Marcel::MimeType.canonicalize "audio/x-wav", instead_of: "audio/vnd.wave"
|
|
42
|
+
|
|
43
|
+
# Prefer IANA-registered types where Tika canonicalizes on a deprecated or private-tree name.
|
|
44
|
+
Marcel::MimeType.canonicalize "application/yaml", instead_of: "text/x-yaml" # RFC 9512
|
|
45
|
+
Marcel::MimeType.canonicalize "application/vnd.debian.binary-package", instead_of: "application/x-debian-package"
|
|
46
|
+
Marcel::MimeType.canonicalize "application/xliff+xml", instead_of: "application/x-xliff+xml"
|
|
47
|
+
|
|
61
48
|
Marcel::MimeType.extend "image/vnd.dwg", magic: [[0, "AC10"]]
|
|
62
49
|
Marcel::MimeType.extend "application/pkcs8", magic: [[0, '-----BEGIN PRIVATE KEY-----']], extensions: %w( p8 )
|
|
63
50
|
|
|
51
|
+
# Tika aliases this to application/x-x509-cert; the generated tables deliberately skip that
|
|
52
|
+
# alias so Marcel can reuse the name for PEM-format certificates via a ;format=pem subtype.
|
|
64
53
|
Marcel::MimeType.extend "application/x-x509-ca-cert", magic: [[0, '-----BEGIN CERTIFICATE-----']], extensions: %w( pem ), parents: "application/x-x509-cert;format=pem"
|
|
65
54
|
|
|
66
|
-
|
|
67
|
-
Marcel::MimeType.extend "image/
|
|
68
|
-
Marcel::MimeType.extend "image/
|
|
55
|
+
# Re-registering these matchers promotes them ahead of video/quicktime's broad ftyp match.
|
|
56
|
+
Marcel::MimeType.extend "image/avif", magic: [[4, "ftypavif"]]
|
|
57
|
+
Marcel::MimeType.extend "image/avif", magic: [[4, "ftypavis"]]
|
|
58
|
+
Marcel::MimeType.extend "image/heif", magic: [[4, "ftypmif1"]]
|
|
59
|
+
Marcel::MimeType.extend "image/heic", magic: [[4, "ftypheic"]]
|
|
69
60
|
|
|
70
61
|
Marcel::MimeType.extend "image/x-raw-sony", magic: [[0, "II*\000", [[0..4096, 'SONY']]], [0, "MM\000*", [[0..4096, 'SONY']]]], extensions: %w( arw ), parents: "image/tiff"
|
|
71
|
-
|
|
62
|
+
# CRW only: .cr2 belongs to image/x-canon-cr2, which Tika matches by magic (TIFF header
|
|
63
|
+
# plus CR marker at offset 8) and already subclasses image/tiff.
|
|
64
|
+
Marcel::MimeType.extend "image/x-raw-canon", parents: "image/tiff"
|
|
72
65
|
|
|
73
66
|
Marcel::MimeType.extend "video/mp4", magic: [[4, "ftypisom"], [4, "ftypM4V "]], extensions: %w( mp4 m4v )
|
|
74
67
|
|
|
75
|
-
Marcel::MimeType.extend "audio/flac", magic: [[0, 'fLaC']], extensions: %w( flac ), parents: "audio/x-flac"
|
|
76
|
-
Marcel::MimeType.extend "audio/x-wav", magic: [[0, 'RIFF', [[8, 'WAVE']]]], extensions: %w( wav ), parents: "audio/vnd.wav"
|
|
77
68
|
Marcel::MimeType.extend "audio/mpc", magic: [[0, "MPCKSH"]], extensions: %w( mpc )
|
|
78
69
|
|
|
79
70
|
Marcel::MimeType.extend "font/ttf", magic: [[0, "\x00\x01\x00\x00"]], extensions: %w( ttf ttc )
|
|
@@ -92,4 +83,4 @@ Marcel::MimeType.extend(
|
|
|
92
83
|
parents: "application/x-msaccess"
|
|
93
84
|
)
|
|
94
85
|
|
|
95
|
-
Marcel::MimeType.extend "text/markdown",
|
|
86
|
+
Marcel::MimeType.extend "text/markdown", parents: "text/x-web-markdown"
|