upload_inspector 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +6 -0
- data/LICENSE.txt +21 -0
- data/README.md +98 -0
- data/lib/upload_inspector/inspector.rb +71 -0
- data/lib/upload_inspector/ooxml_inspector.rb +126 -0
- data/lib/upload_inspector/pdf_inspector.rb +134 -0
- data/lib/upload_inspector/policy.rb +46 -0
- data/lib/upload_inspector/result.rb +36 -0
- data/lib/upload_inspector/sandbox.rb +99 -0
- data/lib/upload_inspector/version.rb +5 -0
- data/lib/upload_inspector/zip_inspector.rb +134 -0
- data/lib/upload_inspector.rb +34 -0
- metadata +93 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 2f372382728dc5813c7186575d5fd9ddb9586b27fac0dd4719690892ab1d087a
|
|
4
|
+
data.tar.gz: 160855eb88dac78e04a30454b325359a5094caa653a9b05ca8d5c8e229be0c25
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 0160d3f7ce788421176da0b9761e9e044f3207549ae409b887915cff1586c1961b17469496b244f5ec29ce08c2333c8fd14281360403e57a3ee236e18ac61d25
|
|
7
|
+
data.tar.gz: ea6d033d6ea7842113800683075320ab514a0ae1e83bce871ba59db189656ba72fb84be85f4455ea30c4b76eae167879a4aee5fd83017d272a3894a0fa556024
|
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Minh Doan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# upload_inspector
|
|
2
|
+
|
|
3
|
+
Fail-closed structural inspection of untrusted uploaded files for any Ruby or Rails app.
|
|
4
|
+
|
|
5
|
+
A file is accepted only when its structure is proven to match a known-safe shape. Anything unknown,
|
|
6
|
+
unparsable, encrypted, oversized, or erroring is rejected. Pure Ruby (`marcel`, `rubyzip`), no AV engine,
|
|
7
|
+
no native parsers, nothing is rendered or executed.
|
|
8
|
+
|
|
9
|
+
> **Not an antivirus.** It blocks structural attack vectors (disguised executables, macros, PDF
|
|
10
|
+
> JavaScript, OLE objects, remote templates, DDE, zip bombs). It does not detect malware signatures or
|
|
11
|
+
> exploits hidden inside otherwise valid content. Pair it with an AV scanner (e.g. Amazon GuardDuty
|
|
12
|
+
> Malware Protection for S3, endpoint AV) when you need that.
|
|
13
|
+
|
|
14
|
+
## What it checks
|
|
15
|
+
|
|
16
|
+
| Format | Checks |
|
|
17
|
+
|---|---|
|
|
18
|
+
| All | real type by magic bytes (Marcel) must be allowlisted; file extension must agree with content; size limit |
|
|
19
|
+
| PDF (and PDF-based `.ai`) | rejects `/JavaScript /JS /Launch /EmbeddedFile(s) /RichMedia /XFA /SubmitForm /ImportData /GoToR /GoToE /Rendition`, hex-escaped spellings included, also inside compressed object streams; rejects encrypted PDFs and unsupported object-stream filters |
|
|
20
|
+
| docx / pptx / xlsx | every part name must match an allowlist; rejects VBA, `embeddings/`, `activeX/`, `externalLinks/`, macro sheets, macro content types, DTDs, external relationships other than http(s)/mailto hyperlinks, Word DDE/INCLUDE/IMPORT/LINK/QUOTE fields, PowerPoint program/macro actions, Excel DDE formulas, non-UTF-8 XML |
|
|
21
|
+
| zip | path traversal, symlinks, encrypted entries, duplicates, entry/total size and compression-ratio limits, nested archives; every member extracted (bounded) and inspected with the same policy |
|
|
22
|
+
| Others | rejected unless listed in `allow_unverifiable` (type + extension check only) |
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
```ruby
|
|
27
|
+
gem 'upload_inspector'
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Requires Ruby >= 3.2.
|
|
31
|
+
|
|
32
|
+
## Usage
|
|
33
|
+
|
|
34
|
+
```ruby
|
|
35
|
+
result = UploadInspector.check(
|
|
36
|
+
'/tmp/upload.pdf',
|
|
37
|
+
filename: 'design.pdf', # client-supplied name
|
|
38
|
+
allowed_types: %w[application/pdf application/zip],
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
result.safe? # => true / false
|
|
42
|
+
result.reason # => :pdf_active_content, :type_not_allowed, ... (nil when safe)
|
|
43
|
+
result.detail # => human-readable hint
|
|
44
|
+
result.detected_type # => "application/pdf"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### Rails / Active Storage
|
|
48
|
+
|
|
49
|
+
```ruby
|
|
50
|
+
result = blob.open do |file|
|
|
51
|
+
UploadInspector.check(file.path, filename: blob.filename.to_s, allowed_types: ALLOWED_TYPES)
|
|
52
|
+
end
|
|
53
|
+
blob.purge_later unless result.safe?
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Run it in a background job for large files.
|
|
57
|
+
|
|
58
|
+
### Policy options
|
|
59
|
+
|
|
60
|
+
| Option | Default |
|
|
61
|
+
|---|---|
|
|
62
|
+
| `allowed_types` | `['application/pdf']` |
|
|
63
|
+
| `allow_unverifiable` | `[]` |
|
|
64
|
+
| `max_file_size` | 500 MB |
|
|
65
|
+
| `max_zip_entries` / `max_zip_entry_size` / `max_zip_total_size` | 5 000 / 200 MB / 1 GB |
|
|
66
|
+
| `max_compression_ratio` | 100 (entries ≥ 1 MB) |
|
|
67
|
+
| `max_inflate_size` | 50 MB per PDF object stream / OOXML XML part |
|
|
68
|
+
| `timeout` / `cpu_limit` | 60 s / 60 s |
|
|
69
|
+
| `memory_limit` | 1 GB (`RLIMIT_DATA`, Linux only) |
|
|
70
|
+
|
|
71
|
+
## Sandbox
|
|
72
|
+
|
|
73
|
+
By default (`sandbox: true`) inspection runs in a child Ruby process with CPU, memory (Linux) and
|
|
74
|
+
wall-clock limits. A crash, hang or memory blow-up kills only the child and yields a rejection
|
|
75
|
+
(`:timeout`, `:resource_limit`, `:inspection_failed`), so web/job workers are unaffected.
|
|
76
|
+
It is resource isolation, not a security boundary (same OS user).
|
|
77
|
+
|
|
78
|
+
## Development
|
|
79
|
+
|
|
80
|
+
Tests build hostile-shaped samples in memory (no real malware, no AV test signatures).
|
|
81
|
+
Run them in a container or CI rather than on a workstation with endpoint AV:
|
|
82
|
+
|
|
83
|
+
```sh
|
|
84
|
+
docker run --rm -v "$PWD":/src:ro ruby:3.3-slim \
|
|
85
|
+
bash -c 'cp -r /src /app && cd /app && bundle install && bundle exec rake test'
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
CI runs the suite on Ruby 3.2 / 3.3 / 3.4 against rubyzip 2.3 and 3.x.
|
|
89
|
+
|
|
90
|
+
### Releasing
|
|
91
|
+
|
|
92
|
+
1. Bump `lib/upload_inspector/version.rb`, update `CHANGELOG.md`, merge to `main`.
|
|
93
|
+
2. Actions → **Release** → *Run workflow*. It runs CI, tags `vX.Y.Z` and publishes via
|
|
94
|
+
rubygems.org Trusted Publishing.
|
|
95
|
+
|
|
96
|
+
## License
|
|
97
|
+
|
|
98
|
+
MIT
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module UploadInspector
|
|
4
|
+
# Detects the real type of a file and routes it to the structural inspector for that format.
|
|
5
|
+
class Inspector
|
|
6
|
+
include Rejectable
|
|
7
|
+
|
|
8
|
+
PDF_MAGIC = '%PDF-'.b
|
|
9
|
+
ZIP_MAGIC = "PK\x03\x04".b
|
|
10
|
+
|
|
11
|
+
attr_reader :policy, :depth
|
|
12
|
+
|
|
13
|
+
def initialize(policy, depth: 0)
|
|
14
|
+
@policy = policy
|
|
15
|
+
@depth = depth
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Boundary method: always returns a Result, rejecting on any error (fail closed).
|
|
19
|
+
def call(path, filename:)
|
|
20
|
+
Result.accept(verify!(path, filename))
|
|
21
|
+
rescue Rejection => e
|
|
22
|
+
Result.reject(e.reason, e.detail, detected_type: @detected_type)
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
Result.reject(:inspection_failed, "#{e.class}: #{e.message}", detected_type: @detected_type)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Raises Rejection when the file is not proven safe; returns the detected MIME type.
|
|
28
|
+
def verify!(path, filename)
|
|
29
|
+
check_size!(path)
|
|
30
|
+
@detected_type = detect_type(path, filename)
|
|
31
|
+
reject!(:type_not_allowed, @detected_type) unless policy.permitted_type?(@detected_type)
|
|
32
|
+
check_extension!(filename, @detected_type)
|
|
33
|
+
inspect_structure!(path, @detected_type)
|
|
34
|
+
@detected_type
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def check_size!(path)
|
|
40
|
+
size = File.size(path)
|
|
41
|
+
reject!(:empty_file) if size.zero?
|
|
42
|
+
reject!(:too_large, "#{size} bytes") if size > policy.max_file_size
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Magic bytes win over the name; the name only refines a magic match (e.g. zip -> docx).
|
|
46
|
+
def detect_type(path, filename)
|
|
47
|
+
File.open(path, 'rb') { |io| Marcel::MimeType.for(io, name: filename.to_s) }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# The client-supplied extension must describe the same format, so a file cannot be
|
|
51
|
+
# stored as one type and opened downstream as another (e.g. a PDF named x.zip).
|
|
52
|
+
def check_extension!(filename, detected)
|
|
53
|
+
by_name = Marcel::MimeType.for(name: filename.to_s)
|
|
54
|
+
return if by_name == detected
|
|
55
|
+
return if Marcel::Magic.child?(detected, by_name) || Marcel::Magic.child?(by_name, detected)
|
|
56
|
+
|
|
57
|
+
reject!(:extension_mismatch, "#{File.extname(filename.to_s)} is #{by_name}, content is #{detected}")
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def inspect_structure!(path, detected)
|
|
61
|
+
head = File.binread(path, 8)
|
|
62
|
+
if head.start_with?(PDF_MAGIC)
|
|
63
|
+
PdfInspector.new(policy).verify!(path)
|
|
64
|
+
elsif head.start_with?(ZIP_MAGIC) && policy.allowed_types.include?(detected)
|
|
65
|
+
ZipInspector.new(policy, depth:).verify!(path, detected)
|
|
66
|
+
elsif !policy.allow_unverifiable.include?(detected)
|
|
67
|
+
reject!(:unverifiable_type, detected)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module UploadInspector
|
|
4
|
+
# Allowlist inspection of Office Open XML (docx / pptx / xlsx) parts.
|
|
5
|
+
#
|
|
6
|
+
# Every part name must match a known-safe pattern. XML parts are read (bounded) and
|
|
7
|
+
# rejected for macro content types, DTDs, external relationships (remote template /
|
|
8
|
+
# Follina-style OLE links), Word field codes that fetch or run things (DDE, INCLUDETEXT...),
|
|
9
|
+
# PowerPoint program/macro actions and Excel DDE formulas.
|
|
10
|
+
class OoxmlInspector
|
|
11
|
+
include Rejectable
|
|
12
|
+
|
|
13
|
+
SEGMENT = '[A-Za-z0-9_\-.]'
|
|
14
|
+
ALLOWED_PARTS = [
|
|
15
|
+
%r{\A\[Content_Types\]\.xml\z},
|
|
16
|
+
%r{\A(?:#{SEGMENT}+/)*_rels/#{SEGMENT}*\.rels\z},
|
|
17
|
+
%r{\A(?:word|ppt|xl|customXml|docProps)/(?:#{SEGMENT}+/)*#{SEGMENT}+\.xml\z},
|
|
18
|
+
%r{\A(?:word|ppt|xl)/media/#{SEGMENT}+\.(?:png|jpe?g|gif|bmp|tiff?|emf|wmf|svg|wdp|jxr)\z}i,
|
|
19
|
+
%r{\AdocProps/thumbnail\.(?:jpe?g|png|emf|wmf)\z}i,
|
|
20
|
+
%r{\A(?:word|ppt|xl)/fonts/#{SEGMENT}+\.(?:odttf|fntdata)\z}i,
|
|
21
|
+
%r{\A(?:word|ppt|xl)/printerSettings/printerSettings\d+\.bin\z},
|
|
22
|
+
].freeze
|
|
23
|
+
FORBIDDEN_PARTS = %r{vbaProject|vbaData|/embeddings/|/activeX/|/externalLinks/|oleObject|/macrosheets/|/dialogsheets/|attachedToolbars|customUI}i
|
|
24
|
+
|
|
25
|
+
MACRO_CONTENT_TYPE = /macroEnabled|vbaProject|ms-office\.vba|activeX|oleObject/i
|
|
26
|
+
DTD = /<!DOCTYPE|<!ENTITY/i
|
|
27
|
+
PREFIX = '(?:[\w.\-]+:)?'
|
|
28
|
+
RELATIONSHIP = /<#{PREFIX}Relationship\b([^>]*)>/n
|
|
29
|
+
ATTRIBUTE = /([\w:.\-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/n
|
|
30
|
+
INSTR_TEXT = %r{<#{PREFIX}instrText\b[^>]*>(.*?)</#{PREFIX}instrText>}mn
|
|
31
|
+
FLD_SIMPLE = /<#{PREFIX}fldSimple\b[^>]*?\b#{PREFIX}instr\s*=\s*(?:"([^"]*)"|'([^']*)')/mn
|
|
32
|
+
DANGEROUS_FIELD = /(?<![A-Z])(?:DDE|DDEAUTO|INCLUDETEXT|INCLUDEPICTURE|IMPORT|LINK|QUOTE)(?![A-Z])/
|
|
33
|
+
PPT_ACTION = %r{ppaction://(?:program|macro|ole)}i
|
|
34
|
+
DDE_FORMULA = /<#{PREFIX}f\b[^>]*>[^<]*\|/n
|
|
35
|
+
SAFE_EXTERNAL_TARGET = /\A(?:https?|mailto):/i
|
|
36
|
+
XML_ENTITIES = { 'amp' => '&', 'lt' => '<', 'gt' => '>', 'quot' => '"', 'apos' => "'" }.freeze
|
|
37
|
+
|
|
38
|
+
def initialize(policy, zip_inspector)
|
|
39
|
+
@policy = policy
|
|
40
|
+
@zip_inspector = zip_inspector
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def verify!(entries)
|
|
44
|
+
reject!(:ooxml_missing_content_types) unless entries.any? { |e| e.name == '[Content_Types].xml' }
|
|
45
|
+
|
|
46
|
+
entries.each do |entry|
|
|
47
|
+
next if entry.directory?
|
|
48
|
+
|
|
49
|
+
check_part_name!(entry.name)
|
|
50
|
+
inspect_xml!(entry) if entry.name.end_with?('.xml', '.rels')
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
attr_reader :policy, :zip_inspector
|
|
57
|
+
|
|
58
|
+
def check_part_name!(name)
|
|
59
|
+
reject!(:ooxml_forbidden_part, name) if name.match?(FORBIDDEN_PARTS)
|
|
60
|
+
reject!(:ooxml_unknown_part, name) unless ALLOWED_PARTS.any? { |pattern| name.match?(pattern) }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def inspect_xml!(entry)
|
|
64
|
+
name = entry.name
|
|
65
|
+
xml = zip_inspector.read_bounded(entry, policy.max_inflate_size)
|
|
66
|
+
# Office writes UTF-8; other encodings would let patterns below be bypassed.
|
|
67
|
+
reject!(:ooxml_unsupported_encoding, name) if xml.include?("\0") || xml.start_with?("\xFF\xFE".b, "\xFE\xFF".b)
|
|
68
|
+
reject!(:ooxml_dtd, name) if xml.match?(DTD)
|
|
69
|
+
reject!(:ooxml_program_action, name) if xml.match?(PPT_ACTION)
|
|
70
|
+
|
|
71
|
+
if name == '[Content_Types].xml'
|
|
72
|
+
reject!(:ooxml_macro_content_type, name) if xml.match?(MACRO_CONTENT_TYPE)
|
|
73
|
+
elsif name.end_with?('.rels')
|
|
74
|
+
check_relationships!(xml, name)
|
|
75
|
+
elsif name.start_with?('word/')
|
|
76
|
+
check_word_fields!(xml, name)
|
|
77
|
+
elsif name.start_with?('xl/')
|
|
78
|
+
reject!(:ooxml_dde_formula, name) if xml.match?(DDE_FORMULA)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# External targets are only allowed for plain web/mail hyperlinks.
|
|
83
|
+
def check_relationships!(xml, name)
|
|
84
|
+
xml.scan(RELATIONSHIP) do |(raw_attributes)|
|
|
85
|
+
attributes = parse_attributes(raw_attributes)
|
|
86
|
+
next unless attributes['TargetMode'].to_s.casecmp?('External')
|
|
87
|
+
|
|
88
|
+
type = attributes['Type'].to_s
|
|
89
|
+
target = attributes['Target'].to_s
|
|
90
|
+
next if type.end_with?('/hyperlink') && target.match?(SAFE_EXTERNAL_TARGET)
|
|
91
|
+
|
|
92
|
+
reject!(:ooxml_external_relationship, "#{name}: #{type.split('/').last} -> #{target}")
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Field instructions may be split across runs, so all pieces are joined before matching.
|
|
97
|
+
def check_word_fields!(xml, name)
|
|
98
|
+
pieces = xml.scan(INSTR_TEXT).flatten + xml.scan(FLD_SIMPLE).flatten.compact
|
|
99
|
+
return if pieces.empty?
|
|
100
|
+
|
|
101
|
+
instructions = pieces.map { |piece| decode_entities(piece) }.join.upcase
|
|
102
|
+
match = instructions.match(DANGEROUS_FIELD)
|
|
103
|
+
reject!(:ooxml_dangerous_field, "#{name}: #{match[0]}") if match
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def parse_attributes(raw)
|
|
107
|
+
raw.scan(ATTRIBUTE).to_h do |key, double_quoted, single_quoted|
|
|
108
|
+
[key.split(':').last, decode_entities(double_quoted || single_quoted)]
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def decode_entities(text)
|
|
113
|
+
text.to_s.b
|
|
114
|
+
.gsub(/&#x(\h+);/n) { codepoint_bytes(Regexp.last_match(1).hex) }
|
|
115
|
+
.gsub(/&#(\d+);/n) { codepoint_bytes(Regexp.last_match(1).to_i) }
|
|
116
|
+
.gsub(/&(amp|lt|gt|quot|apos);/n) { XML_ENTITIES[Regexp.last_match(1)] }
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Invalid code points become a marker that never matches an allowed value (fail closed).
|
|
120
|
+
def codepoint_bytes(codepoint)
|
|
121
|
+
[codepoint].pack('U').b
|
|
122
|
+
rescue RangeError
|
|
123
|
+
"�".b
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module UploadInspector
|
|
4
|
+
# Rejects PDFs (and PDF-based formats such as .ai) that carry active content.
|
|
5
|
+
#
|
|
6
|
+
# Streams the raw bytes looking for dangerous name tokens (hex-escaped spellings
|
|
7
|
+
# like /J#61vaScript included), then inflates every compressed object stream
|
|
8
|
+
# (/Type /ObjStm), where those keys could otherwise hide. Nothing is rendered.
|
|
9
|
+
class PdfInspector
|
|
10
|
+
include Rejectable
|
|
11
|
+
|
|
12
|
+
CHUNK_SIZE = 4 * MEGABYTE
|
|
13
|
+
INFLATE_CHUNK_SIZE = 64 * 1024
|
|
14
|
+
OVERLAP = 256 # longer than the longest fully hex-escaped token
|
|
15
|
+
OBJECT_WINDOW = 8 * 1024
|
|
16
|
+
|
|
17
|
+
ACTIVE_NAMES = %w[
|
|
18
|
+
JavaScript JS Launch EmbeddedFile EmbeddedFiles RichMedia XFA
|
|
19
|
+
SubmitForm ImportData GoToR GoToE Rendition
|
|
20
|
+
].freeze
|
|
21
|
+
PDF_DELIMITER = '(?=[\s\x00\/<>\[\]()%{}]|\z)'
|
|
22
|
+
|
|
23
|
+
# PDF name token matching both literal and #xx-escaped characters.
|
|
24
|
+
def self.name_pattern(name)
|
|
25
|
+
chars = name.chars.map do |char|
|
|
26
|
+
hex = format('%02x', char.ord).chars.map { |h| h.match?(/[a-f]/) ? "[#{h}#{h.upcase}]" : h }.join
|
|
27
|
+
"(?:#{Regexp.escape(char)}|##{hex})"
|
|
28
|
+
end
|
|
29
|
+
"/#{chars.join}#{PDF_DELIMITER}"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
ACTIVE_PATTERN = Regexp.new(ACTIVE_NAMES.map { |n| name_pattern(n) }.join('|'), Regexp::NOENCODING)
|
|
33
|
+
ENCRYPT_PATTERN = Regexp.new(name_pattern('Encrypt'), Regexp::NOENCODING)
|
|
34
|
+
OBJSTM_PATTERN = Regexp.new(name_pattern('ObjStm'), Regexp::NOENCODING)
|
|
35
|
+
OBJECT_HEADER = /\d+\s+\d+\s+obj\b/n
|
|
36
|
+
STREAM_KEYWORD = /stream(?:\r\n|\n|\r)/n
|
|
37
|
+
|
|
38
|
+
def initialize(policy)
|
|
39
|
+
@policy = policy
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def verify!(path)
|
|
43
|
+
object_stream_offsets = scan_raw!(path)
|
|
44
|
+
File.open(path, 'rb') do |file|
|
|
45
|
+
object_stream_offsets.each { |offset| inspect_object_stream!(file, offset) }
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
attr_reader :policy
|
|
52
|
+
|
|
53
|
+
# Returns byte offsets of every /ObjStm token.
|
|
54
|
+
def scan_raw!(path)
|
|
55
|
+
offsets = []
|
|
56
|
+
carry = ''.b
|
|
57
|
+
consumed = 0
|
|
58
|
+
File.open(path, 'rb') do |file|
|
|
59
|
+
while (chunk = file.read(CHUNK_SIZE))
|
|
60
|
+
buffer = carry + chunk
|
|
61
|
+
base = consumed - carry.bytesize
|
|
62
|
+
scan_tokens!(buffer)
|
|
63
|
+
buffer.scan(OBJSTM_PATTERN) { offsets << base + Regexp.last_match.begin(0) }
|
|
64
|
+
carry = tail(buffer)
|
|
65
|
+
consumed += chunk.bytesize
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
offsets.uniq
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def scan_tokens!(buffer)
|
|
72
|
+
reject!(:pdf_encrypted, 'encrypted PDF cannot be inspected') if buffer.match?(ENCRYPT_PATTERN)
|
|
73
|
+
match = buffer.match(ACTIVE_PATTERN)
|
|
74
|
+
reject!(:pdf_active_content, decode_name(match[0])) if match
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def inspect_object_stream!(file, offset)
|
|
78
|
+
start = [offset - OBJECT_WINDOW, 0].max
|
|
79
|
+
file.seek(start)
|
|
80
|
+
before = file.read(offset - start) || ''.b
|
|
81
|
+
header_index = before.rindex(OBJECT_HEADER) || reject!(:pdf_unparsable, 'object stream header not found')
|
|
82
|
+
file.seek(offset)
|
|
83
|
+
after = file.read(OBJECT_WINDOW) || ''.b
|
|
84
|
+
stream_match = after.match(STREAM_KEYWORD) || reject!(:pdf_unparsable, 'object stream data not found')
|
|
85
|
+
|
|
86
|
+
dictionary = before.byteslice(header_index..) + after.byteslice(0, stream_match.begin(0))
|
|
87
|
+
return unless compressed_object_stream?(dictionary)
|
|
88
|
+
|
|
89
|
+
inflate_and_scan!(file, offset + stream_match.end(0))
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Only a single plain FlateDecode filter is accepted; anything else cannot be verified.
|
|
93
|
+
# Returns false for an unfiltered stream (its bytes were already covered by the raw scan).
|
|
94
|
+
def compressed_object_stream?(dictionary)
|
|
95
|
+
reject!(:pdf_unparsable, 'object stream spans objects') if dictionary.match?(/endobj|endstream/n)
|
|
96
|
+
reject!(:pdf_unparsable, 'escaped names in object stream dictionary') if dictionary.include?('#')
|
|
97
|
+
reject!(:pdf_unparsable, 'object stream decode parameters') if dictionary.include?('/DecodeParms')
|
|
98
|
+
|
|
99
|
+
filters = dictionary.scan(%r{/Filter\s*(\[[^\]]*\]|/[A-Za-z0-9]+)}n).flatten
|
|
100
|
+
return false if filters.empty?
|
|
101
|
+
return true if filters.size == 1 && filters.first.gsub(/[\[\]\s]/n, '') == '/FlateDecode'
|
|
102
|
+
|
|
103
|
+
reject!(:pdf_unparsable, "unsupported object stream filter #{filters.join(' ')}")
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def inflate_and_scan!(file, data_offset)
|
|
107
|
+
file.seek(data_offset)
|
|
108
|
+
inflater = Zlib::Inflate.new
|
|
109
|
+
inflated = 0
|
|
110
|
+
carry = ''.b
|
|
111
|
+
until inflater.finished?
|
|
112
|
+
chunk = file.read(INFLATE_CHUNK_SIZE) || reject!(:pdf_unparsable, 'truncated object stream')
|
|
113
|
+
output = inflater.inflate(chunk)
|
|
114
|
+
inflated += output.bytesize
|
|
115
|
+
reject!(:pdf_inflate_limit, "object stream exceeds #{policy.max_inflate_size} bytes") if inflated > policy.max_inflate_size
|
|
116
|
+
buffer = carry + output
|
|
117
|
+
scan_tokens!(buffer)
|
|
118
|
+
carry = tail(buffer)
|
|
119
|
+
end
|
|
120
|
+
rescue Zlib::Error => e
|
|
121
|
+
reject!(:pdf_unparsable, "object stream: #{e.message}")
|
|
122
|
+
ensure
|
|
123
|
+
inflater&.close
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def tail(buffer)
|
|
127
|
+
buffer.bytesize > OVERLAP ? buffer.byteslice(-OVERLAP, OVERLAP) : buffer
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def decode_name(token)
|
|
131
|
+
token.gsub(/#(\h\h)/n) { Regexp.last_match(1).hex.chr }
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module UploadInspector
|
|
4
|
+
MEGABYTE = 1024 * 1024
|
|
5
|
+
|
|
6
|
+
# Inspection limits and allowlists. Defaults are deliberately strict (PDF only).
|
|
7
|
+
#
|
|
8
|
+
# allowed_types - MIME types whose structure this gem can verify (PDF family, OOXML, zip).
|
|
9
|
+
# allow_unverifiable - MIME types accepted on real-type + extension match only, because their
|
|
10
|
+
# content cannot be inspected (e.g. image/vnd.adobe.photoshop). Opt-in risk.
|
|
11
|
+
Policy = Data.define(
|
|
12
|
+
:allowed_types, :allow_unverifiable, :max_file_size,
|
|
13
|
+
:max_zip_entries, :max_zip_entry_size, :max_zip_total_size, :max_compression_ratio,
|
|
14
|
+
:max_inflate_size, :timeout, :memory_limit, :cpu_limit
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
class Policy
|
|
18
|
+
DEFAULTS = {
|
|
19
|
+
allowed_types: ['application/pdf'],
|
|
20
|
+
allow_unverifiable: [],
|
|
21
|
+
max_file_size: 500 * MEGABYTE,
|
|
22
|
+
max_zip_entries: 5_000,
|
|
23
|
+
max_zip_entry_size: 200 * MEGABYTE,
|
|
24
|
+
max_zip_total_size: 1024 * MEGABYTE,
|
|
25
|
+
max_compression_ratio: 100,
|
|
26
|
+
max_inflate_size: 50 * MEGABYTE, # per PDF object stream / per OOXML XML part
|
|
27
|
+
timeout: 60, # seconds, wall clock (sandbox only)
|
|
28
|
+
memory_limit: 1024 * MEGABYTE, # RLIMIT_DATA for the sandbox child; nil disables
|
|
29
|
+
cpu_limit: 60, # seconds, RLIMIT_CPU for the sandbox child; nil disables
|
|
30
|
+
}.freeze
|
|
31
|
+
|
|
32
|
+
def initialize(**options)
|
|
33
|
+
unknown = options.keys - DEFAULTS.keys
|
|
34
|
+
raise ArgumentError, "unknown policy option(s): #{unknown.join(', ')}" if unknown.any?
|
|
35
|
+
|
|
36
|
+
merged = DEFAULTS.merge(options)
|
|
37
|
+
merged[:allowed_types] = Array(merged[:allowed_types]).map(&:to_s).freeze
|
|
38
|
+
merged[:allow_unverifiable] = Array(merged[:allow_unverifiable]).map(&:to_s).freeze
|
|
39
|
+
super(**merged)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def permitted_type?(type)
|
|
43
|
+
allowed_types.include?(type) || allow_unverifiable.include?(type)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module UploadInspector
|
|
4
|
+
# Outcome of an inspection. +reason+ is a Symbol (nil when safe), +detail+ a human-readable hint.
|
|
5
|
+
Result = Data.define(:safe, :reason, :detail, :detected_type) do
|
|
6
|
+
def self.accept(detected_type)
|
|
7
|
+
new(safe: true, reason: nil, detail: nil, detected_type:)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def self.reject(reason, detail = nil, detected_type: nil)
|
|
11
|
+
new(safe: false, reason: reason.to_sym, detail:, detected_type:)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def safe? = safe
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Raised internally by inspectors; converted to a rejected Result at the boundary.
|
|
18
|
+
class Rejection < StandardError
|
|
19
|
+
attr_reader :reason, :detail
|
|
20
|
+
|
|
21
|
+
def initialize(reason, detail = nil)
|
|
22
|
+
@reason = reason
|
|
23
|
+
@detail = detail
|
|
24
|
+
super([reason, detail].compact.join(': '))
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Shared helper for inspectors.
|
|
29
|
+
module Rejectable
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def reject!(reason, detail = nil)
|
|
33
|
+
raise Rejection.new(reason, detail)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'io/wait'
|
|
4
|
+
require 'rbconfig'
|
|
5
|
+
|
|
6
|
+
module UploadInspector
|
|
7
|
+
# Runs the inspection in a separate Ruby process with CPU, memory and wall-clock limits.
|
|
8
|
+
# A crash, hang or memory blow-up kills only the child and is reported as a rejection,
|
|
9
|
+
# so the host process (web server / job worker) is never affected.
|
|
10
|
+
#
|
|
11
|
+
# This is resource isolation, not a security boundary: the child runs as the same user.
|
|
12
|
+
# Parsing is pure Ruby and never executes file content, which keeps that risk low.
|
|
13
|
+
module Sandbox
|
|
14
|
+
LIB_DIR = File.expand_path('..', __dir__)
|
|
15
|
+
CHILD_SCRIPT = "require 'upload_inspector'; UploadInspector::Sandbox.child_main(ARGV.fetch(0))"
|
|
16
|
+
READ_CHUNK_SIZE = 16 * 1024
|
|
17
|
+
MEMORY_LIMIT_SUPPORTED = RUBY_PLATFORM.include?('linux')
|
|
18
|
+
|
|
19
|
+
module_function
|
|
20
|
+
|
|
21
|
+
def run(path, filename:, policy:)
|
|
22
|
+
reader, writer = IO.pipe
|
|
23
|
+
payload = JSON.generate(path: File.expand_path(path), filename: filename.to_s, policy: policy.to_h)
|
|
24
|
+
pid = Process.spawn(RbConfig.ruby, '-I', LIB_DIR, '-e', CHILD_SCRIPT, payload,
|
|
25
|
+
spawn_options(policy, writer))
|
|
26
|
+
writer.close
|
|
27
|
+
collect(pid, reader, policy.timeout)
|
|
28
|
+
rescue SystemCallError => e
|
|
29
|
+
Result.reject(:sandbox_failed, e.message)
|
|
30
|
+
ensure
|
|
31
|
+
writer&.close unless writer&.closed?
|
|
32
|
+
reader&.close unless reader&.closed?
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Entry point inside the child process.
|
|
36
|
+
def child_main(payload)
|
|
37
|
+
input = JSON.parse(payload, symbolize_names: true)
|
|
38
|
+
policy = Policy.new(**input.fetch(:policy))
|
|
39
|
+
result = Inspector.new(policy).call(input.fetch(:path), filename: input.fetch(:filename))
|
|
40
|
+
$stdout.write(JSON.generate(result.to_h))
|
|
41
|
+
rescue NoMemoryError
|
|
42
|
+
$stdout.write(JSON.generate(Result.reject(:resource_limit, 'memory limit').to_h))
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def spawn_options(policy, writer)
|
|
46
|
+
options = { out: writer, err: File::NULL, in: File::NULL, pgroup: true, chdir: Dir.tmpdir }
|
|
47
|
+
# RLIMIT_DATA is enforced on Linux (production containers); macOS rejects lowering it.
|
|
48
|
+
options[:rlimit_data] = policy.memory_limit if policy.memory_limit && MEMORY_LIMIT_SUPPORTED
|
|
49
|
+
options[:rlimit_cpu] = policy.cpu_limit if policy.cpu_limit
|
|
50
|
+
options
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def collect(pid, reader, timeout)
|
|
54
|
+
output = read_until_eof(reader, timeout)
|
|
55
|
+
if output.nil?
|
|
56
|
+
kill(pid)
|
|
57
|
+
return Result.reject(:timeout, "exceeded #{timeout}s")
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
_, status = Process.wait2(pid)
|
|
61
|
+
return Result.reject(:resource_limit, "child killed by signal #{status.termsig}") if status.signaled?
|
|
62
|
+
return Result.reject(:inspection_failed, "child exited with status #{status.exitstatus}") unless status.success?
|
|
63
|
+
|
|
64
|
+
parse(output)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Returns nil when the deadline passes before the child closes its output.
|
|
68
|
+
def read_until_eof(reader, timeout)
|
|
69
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
|
|
70
|
+
output = +''
|
|
71
|
+
loop do
|
|
72
|
+
remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
73
|
+
return nil if remaining <= 0
|
|
74
|
+
next unless reader.wait_readable(remaining)
|
|
75
|
+
|
|
76
|
+
chunk = reader.read_nonblock(READ_CHUNK_SIZE, exception: false)
|
|
77
|
+
return output if chunk.nil?
|
|
78
|
+
|
|
79
|
+
output << chunk unless chunk == :wait_readable
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def kill(pid)
|
|
84
|
+
Process.kill(:KILL, -pid)
|
|
85
|
+
rescue Errno::ESRCH
|
|
86
|
+
nil
|
|
87
|
+
ensure
|
|
88
|
+
Process.wait(pid) rescue Errno::ECHILD # rubocop:disable Style/RescueModifier
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def parse(output)
|
|
92
|
+
data = JSON.parse(output, symbolize_names: true)
|
|
93
|
+
Result.new(safe: data.fetch(:safe) == true, reason: data[:reason]&.to_sym,
|
|
94
|
+
detail: data[:detail], detected_type: data[:detected_type])
|
|
95
|
+
rescue JSON::ParserError, KeyError
|
|
96
|
+
Result.reject(:inspection_failed, 'unreadable sandbox output')
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'tmpdir'
|
|
4
|
+
|
|
5
|
+
module UploadInspector
|
|
6
|
+
# Validates zip containers. OOXML documents are checked part by part; plain archives
|
|
7
|
+
# have every member extracted (size-bounded, to a temp dir) and inspected with the same
|
|
8
|
+
# policy. Archives inside archives are rejected.
|
|
9
|
+
class ZipInspector
|
|
10
|
+
include Rejectable
|
|
11
|
+
|
|
12
|
+
OOXML_TYPES = %w[
|
|
13
|
+
application/vnd.openxmlformats-officedocument.wordprocessingml.document
|
|
14
|
+
application/vnd.openxmlformats-officedocument.presentationml.presentation
|
|
15
|
+
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
|
16
|
+
].freeze
|
|
17
|
+
NESTED_ARCHIVE = /\.(?:zip|7z|rar|gz|tgz|bz2|xz|zst|tar|jar|war|ear|apk|ipa|cab|iso|img|dmg|vhdx?|arj|lzh|lha|z)\z/i
|
|
18
|
+
READ_CHUNK_SIZE = 64 * 1024
|
|
19
|
+
RATIO_CHECK_MIN_SIZE = MEGABYTE # tiny files compress extremely well legitimately
|
|
20
|
+
|
|
21
|
+
def initialize(policy, depth:)
|
|
22
|
+
@policy = policy
|
|
23
|
+
@depth = depth
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def verify!(path, detected_type)
|
|
27
|
+
Zip::File.open(path) do |zip|
|
|
28
|
+
entries = zip.entries
|
|
29
|
+
check_entries!(entries)
|
|
30
|
+
if OOXML_TYPES.include?(detected_type)
|
|
31
|
+
OoxmlInspector.new(policy, self).verify!(entries)
|
|
32
|
+
else
|
|
33
|
+
verify_archive_members!(entries)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
rescue Zip::Error => e
|
|
37
|
+
reject!(:zip_unparsable, e.message)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Reads a member fully into memory, never beyond +limit+ or its declared size.
|
|
41
|
+
def read_bounded(entry, limit)
|
|
42
|
+
reject!(:zip_inflate_limit, entry.name) if entry.size > limit
|
|
43
|
+
data = +''.b
|
|
44
|
+
each_chunk(entry) do |chunk|
|
|
45
|
+
data << chunk
|
|
46
|
+
reject!(:zip_inflate_limit, entry.name) if data.bytesize > limit
|
|
47
|
+
end
|
|
48
|
+
data
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
attr_reader :policy, :depth
|
|
54
|
+
|
|
55
|
+
def check_entries!(entries)
|
|
56
|
+
reject!(:zip_empty) if entries.empty?
|
|
57
|
+
reject!(:zip_too_many_entries, entries.size.to_s) if entries.size > policy.max_zip_entries
|
|
58
|
+
names = entries.map { |e| e.name.to_s.downcase }
|
|
59
|
+
reject!(:zip_duplicate_entry) if names.uniq.size != names.size
|
|
60
|
+
|
|
61
|
+
total = 0
|
|
62
|
+
entries.each do |entry|
|
|
63
|
+
check_entry!(entry)
|
|
64
|
+
total += entry.size.to_i
|
|
65
|
+
reject!(:zip_too_large, 'total uncompressed size') if total > policy.max_zip_total_size
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def check_entry!(entry)
|
|
70
|
+
check_path!(entry.name.to_s)
|
|
71
|
+
reject!(:zip_encrypted_entry, entry.name) if entry.gp_flags.to_i.anybits?(1)
|
|
72
|
+
reject!(:zip_unsafe_path, "symlink #{entry.name}") if entry.symlink?
|
|
73
|
+
reject!(:zip_too_large, entry.name) if entry.size.to_i > policy.max_zip_entry_size
|
|
74
|
+
check_ratio!(entry)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def check_path!(name)
|
|
78
|
+
unsafe = name.empty? || name.include?("\0") || name.include?('\\') || name.start_with?('/') ||
|
|
79
|
+
name.match?(/\A[A-Za-z]:/) || name.split('/').include?('..')
|
|
80
|
+
reject!(:zip_unsafe_path, name.inspect) if unsafe
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def check_ratio!(entry)
|
|
84
|
+
size = entry.size.to_i
|
|
85
|
+
return if size < RATIO_CHECK_MIN_SIZE
|
|
86
|
+
|
|
87
|
+
compressed = entry.compressed_size.to_i
|
|
88
|
+
return if compressed.positive? && size / compressed <= policy.max_compression_ratio
|
|
89
|
+
|
|
90
|
+
reject!(:zip_bomb, "#{entry.name} ratio exceeds #{policy.max_compression_ratio}")
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def verify_archive_members!(entries)
|
|
94
|
+
reject!(:nested_archive, 'archive inside archive') if depth.positive?
|
|
95
|
+
|
|
96
|
+
Dir.mktmpdir('upload_inspector') do |dir|
|
|
97
|
+
entries.reject(&:directory?).each_with_index do |entry, index|
|
|
98
|
+
reject!(:nested_archive, entry.name) if entry.name.match?(NESTED_ARCHIVE)
|
|
99
|
+
member_path = File.join(dir, "member-#{index}")
|
|
100
|
+
extract_bounded!(entry, member_path)
|
|
101
|
+
verify_member!(member_path, entry.name)
|
|
102
|
+
File.delete(member_path)
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def verify_member!(member_path, name)
|
|
108
|
+
Inspector.new(policy, depth: depth + 1).verify!(member_path, File.basename(name))
|
|
109
|
+
rescue Rejection => e
|
|
110
|
+
raise Rejection.new(e.reason, [name, e.detail].compact.join(': '))
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Actual inflated bytes must match the declared size (no lying headers).
|
|
114
|
+
def extract_bounded!(entry, destination)
|
|
115
|
+
written = 0
|
|
116
|
+
File.open(destination, 'wb') do |out|
|
|
117
|
+
each_chunk(entry) do |chunk|
|
|
118
|
+
written += chunk.bytesize
|
|
119
|
+
reject!(:zip_size_mismatch, entry.name) if written > entry.size.to_i
|
|
120
|
+
out.write(chunk)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
reject!(:zip_size_mismatch, entry.name) if written != entry.size.to_i
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def each_chunk(entry)
|
|
127
|
+
entry.get_input_stream do |io|
|
|
128
|
+
while (chunk = io.read(READ_CHUNK_SIZE)) && !chunk.empty?
|
|
129
|
+
yield chunk
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'marcel'
|
|
5
|
+
require 'zip'
|
|
6
|
+
require 'zlib'
|
|
7
|
+
|
|
8
|
+
require_relative 'upload_inspector/version'
|
|
9
|
+
require_relative 'upload_inspector/result'
|
|
10
|
+
require_relative 'upload_inspector/policy'
|
|
11
|
+
require_relative 'upload_inspector/pdf_inspector'
|
|
12
|
+
require_relative 'upload_inspector/ooxml_inspector'
|
|
13
|
+
require_relative 'upload_inspector/zip_inspector'
|
|
14
|
+
require_relative 'upload_inspector/inspector'
|
|
15
|
+
require_relative 'upload_inspector/sandbox'
|
|
16
|
+
|
|
17
|
+
# Fail-closed structural inspection of untrusted uploaded files.
|
|
18
|
+
# Only content proven to match a known-safe structure is accepted; anything
|
|
19
|
+
# unknown, unparsable, or erroring is rejected.
|
|
20
|
+
module UploadInspector
|
|
21
|
+
# Inspects the file at +path+. Never raises on hostile input: always returns a Result.
|
|
22
|
+
#
|
|
23
|
+
# UploadInspector.check('/tmp/x.pdf', filename: 'design.pdf', allowed_types: ['application/pdf'])
|
|
24
|
+
#
|
|
25
|
+
# +filename+ is the client-supplied name (its extension must agree with the real content).
|
|
26
|
+
# +sandbox+ runs the inspection in a resource-limited child process (default true).
|
|
27
|
+
# Other keywords are Policy options.
|
|
28
|
+
def self.check(path, filename:, sandbox: true, **policy_options)
|
|
29
|
+
policy = Policy.new(**policy_options)
|
|
30
|
+
return Sandbox.run(path, filename:, policy:) if sandbox
|
|
31
|
+
|
|
32
|
+
Inspector.new(policy).call(path, filename:)
|
|
33
|
+
end
|
|
34
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: upload_inspector
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Minh Doan
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: marcel
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '1.0'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '1.0'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: rubyzip
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '2.3'
|
|
33
|
+
- - "<"
|
|
34
|
+
- !ruby/object:Gem::Version
|
|
35
|
+
version: '4'
|
|
36
|
+
type: :runtime
|
|
37
|
+
prerelease: false
|
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
39
|
+
requirements:
|
|
40
|
+
- - ">="
|
|
41
|
+
- !ruby/object:Gem::Version
|
|
42
|
+
version: '2.3'
|
|
43
|
+
- - "<"
|
|
44
|
+
- !ruby/object:Gem::Version
|
|
45
|
+
version: '4'
|
|
46
|
+
description: |
|
|
47
|
+
Rejects uploaded files whose real type does not match the allowlist or the file extension,
|
|
48
|
+
PDFs with active content (JavaScript, launch actions, embedded files, also inside compressed
|
|
49
|
+
object streams), Office documents with macros, OLE objects, external relationships, DDE fields,
|
|
50
|
+
and dangerous zip archives. Runs in a resource-limited child process. Pure Ruby, no AV engine.
|
|
51
|
+
email:
|
|
52
|
+
- minhdpc@gmail.com
|
|
53
|
+
executables: []
|
|
54
|
+
extensions: []
|
|
55
|
+
extra_rdoc_files: []
|
|
56
|
+
files:
|
|
57
|
+
- CHANGELOG.md
|
|
58
|
+
- LICENSE.txt
|
|
59
|
+
- README.md
|
|
60
|
+
- lib/upload_inspector.rb
|
|
61
|
+
- lib/upload_inspector/inspector.rb
|
|
62
|
+
- lib/upload_inspector/ooxml_inspector.rb
|
|
63
|
+
- lib/upload_inspector/pdf_inspector.rb
|
|
64
|
+
- lib/upload_inspector/policy.rb
|
|
65
|
+
- lib/upload_inspector/result.rb
|
|
66
|
+
- lib/upload_inspector/sandbox.rb
|
|
67
|
+
- lib/upload_inspector/version.rb
|
|
68
|
+
- lib/upload_inspector/zip_inspector.rb
|
|
69
|
+
homepage: https://github.com/akarui2004/upload_inspector
|
|
70
|
+
licenses:
|
|
71
|
+
- MIT
|
|
72
|
+
metadata:
|
|
73
|
+
rubygems_mfa_required: 'true'
|
|
74
|
+
changelog_uri: https://github.com/akarui2004/upload_inspector/blob/main/CHANGELOG.md
|
|
75
|
+
source_code_uri: https://github.com/akarui2004/upload_inspector
|
|
76
|
+
rdoc_options: []
|
|
77
|
+
require_paths:
|
|
78
|
+
- lib
|
|
79
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
80
|
+
requirements:
|
|
81
|
+
- - ">="
|
|
82
|
+
- !ruby/object:Gem::Version
|
|
83
|
+
version: '3.2'
|
|
84
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - ">="
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: '0'
|
|
89
|
+
requirements: []
|
|
90
|
+
rubygems_version: 3.6.9
|
|
91
|
+
specification_version: 4
|
|
92
|
+
summary: Fail-closed structural inspection of untrusted uploads (PDF, Office, zip).
|
|
93
|
+
test_files: []
|