file_hutch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +37 -0
- data/LICENSE.txt +21 -0
- data/README.md +288 -0
- data/app/assets/javascripts/file_hutch/direct_upload_controller.js +129 -0
- data/app/controllers/file_hutch/direct_uploads_controller.rb +60 -0
- data/config/routes.rb +7 -0
- data/exe/file_hutch +6 -0
- data/lib/file_hutch/cli.rb +232 -0
- data/lib/file_hutch/client.rb +255 -0
- data/lib/file_hutch/configuration.rb +33 -0
- data/lib/file_hutch/errors.rb +96 -0
- data/lib/file_hutch/file.rb +57 -0
- data/lib/file_hutch/project.rb +49 -0
- data/lib/file_hutch/rails/attachable.rb +184 -0
- data/lib/file_hutch/rails/engine.rb +28 -0
- data/lib/file_hutch/rails.rb +5 -0
- data/lib/file_hutch/resource.rb +44 -0
- data/lib/file_hutch/upload.rb +28 -0
- data/lib/file_hutch/version.rb +5 -0
- data/lib/file_hutch/webhook.rb +52 -0
- data/lib/file_hutch.rb +63 -0
- data/lib/generators/file_hutch/attachment/attachment_generator.rb +30 -0
- data/lib/generators/file_hutch/attachment/templates/migration.rb.tt +6 -0
- data/lib/generators/file_hutch/install/install_generator.rb +34 -0
- data/lib/generators/file_hutch/install/templates/initializer.rb +18 -0
- metadata +73 -0
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "yaml"
|
|
4
|
+
require "optparse"
|
|
5
|
+
|
|
6
|
+
module FileHutch
|
|
7
|
+
# `file_hutch` on the command line. Plan/apply a project's config file,
|
|
8
|
+
# export it, inspect the project, upload a file, dump the manifest.
|
|
9
|
+
# Reads FILE_HUTCH_API_KEY and FILE_HUTCH_URL like the library does.
|
|
10
|
+
class CLI
|
|
11
|
+
DEFAULT_FILE = "file_hutch.yml"
|
|
12
|
+
MARK = { "create" => "+", "update" => "~", "delete" => "-", "noop" => "=" }.freeze
|
|
13
|
+
|
|
14
|
+
USAGE = <<~TEXT
|
|
15
|
+
Usage: file_hutch <command> [options]
|
|
16
|
+
|
|
17
|
+
plan [FILE] [--prune] show what apply would change (read-only keys allowed)
|
|
18
|
+
apply [FILE] [--prune] [--yes] make the project match FILE
|
|
19
|
+
export print the project as a config file
|
|
20
|
+
inspect project, environment, storage, plan and usage
|
|
21
|
+
upload PATH --policy NAME direct upload, prints the file id
|
|
22
|
+
manifest every ready file with its object key, as JSON lines
|
|
23
|
+
version
|
|
24
|
+
|
|
25
|
+
FILE defaults to #{DEFAULT_FILE}. Nothing is deleted without --prune; with --prune,
|
|
26
|
+
apply asks before deleting unless --yes is given.
|
|
27
|
+
Environment: FILE_HUTCH_API_KEY (required), FILE_HUTCH_URL.
|
|
28
|
+
TEXT
|
|
29
|
+
|
|
30
|
+
def initialize(argv, out: $stdout, err: $stderr, input: $stdin, client: nil)
|
|
31
|
+
@argv, @out, @err, @input, @client = argv.dup, out, err, input, client
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Returns the exit status.
|
|
35
|
+
def run
|
|
36
|
+
command = @argv.shift
|
|
37
|
+
case command
|
|
38
|
+
when "plan" then plan
|
|
39
|
+
when "apply" then apply
|
|
40
|
+
when "export" then export
|
|
41
|
+
when "inspect" then inspect_project
|
|
42
|
+
when "upload" then upload
|
|
43
|
+
when "manifest" then manifest
|
|
44
|
+
when "version", "--version", "-v" then print_and_succeed("file_hutch #{VERSION}")
|
|
45
|
+
when nil, "help", "--help", "-h" then print_and_succeed(USAGE)
|
|
46
|
+
else
|
|
47
|
+
@err.puts("Unknown command #{command.inspect}\n\n#{USAGE}")
|
|
48
|
+
2
|
|
49
|
+
end
|
|
50
|
+
rescue ConfigurationError => e
|
|
51
|
+
@err.puts(e.message)
|
|
52
|
+
2
|
|
53
|
+
rescue ApiError => e
|
|
54
|
+
@err.puts("FileHutch said no (#{e.code}): #{e.message}")
|
|
55
|
+
1
|
|
56
|
+
rescue ConnectionError => e
|
|
57
|
+
@err.puts("Could not reach FileHutch: #{e.message}")
|
|
58
|
+
1
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def client
|
|
64
|
+
@client ||= Client.new
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def print_and_succeed(text)
|
|
68
|
+
@out.puts(text)
|
|
69
|
+
0
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# -- plan / apply -------------------------------------------------------
|
|
73
|
+
|
|
74
|
+
def plan
|
|
75
|
+
options = parse_options
|
|
76
|
+
config = read_config(options[:file]) or return 2
|
|
77
|
+
plan = client.plan_config(config, prune: options[:prune])
|
|
78
|
+
print_changes(plan["changes"])
|
|
79
|
+
summary = plan["summary"]
|
|
80
|
+
@out.puts "Plan: #{summary['create']} to create, #{summary['update']} to update, #{summary['delete']} to delete."
|
|
81
|
+
0
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def apply
|
|
85
|
+
options = parse_options
|
|
86
|
+
config = read_config(options[:file]) or return 2
|
|
87
|
+
|
|
88
|
+
if options[:prune] && !options[:yes]
|
|
89
|
+
plan = client.plan_config(config, prune: true)
|
|
90
|
+
deletes = plan["changes"].select { |c| c["action"] == "delete" }
|
|
91
|
+
if deletes.any?
|
|
92
|
+
print_changes(deletes)
|
|
93
|
+
@out.print "Delete #{deletes.size} #{deletes.size == 1 ? 'resource' : 'resources'}? [y/N] "
|
|
94
|
+
@out.flush
|
|
95
|
+
unless @input.gets.to_s.strip.downcase.start_with?("y")
|
|
96
|
+
@out.puts "Nothing applied."
|
|
97
|
+
return 1
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
result = client.apply_config(config, prune: options[:prune])
|
|
103
|
+
result["results"].each do |r|
|
|
104
|
+
line = "#{MARK.fetch(r['action'], '?')} #{r['resource']} #{r['name']}"
|
|
105
|
+
line += " #{describe_diff(r['diff'])}" if r["diff"]
|
|
106
|
+
line += r["status"] == "applied" ? " ok" : " FAILED: #{r['error']}"
|
|
107
|
+
@out.puts line
|
|
108
|
+
end
|
|
109
|
+
summary = result["summary"]
|
|
110
|
+
@out.puts "Applied #{summary['applied']}, failed #{summary['failed']}, unchanged #{summary['noop']}."
|
|
111
|
+
summary["failed"].to_i.zero? ? 0 : 1
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def export
|
|
115
|
+
@out.puts YAML.dump(client.project_config).sub(/\A---\n/, "")
|
|
116
|
+
0
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# -- inspect / upload / manifest ----------------------------------------
|
|
120
|
+
|
|
121
|
+
def inspect_project
|
|
122
|
+
project = client.project
|
|
123
|
+
env = project["environment"]
|
|
124
|
+
storage = project.storage_connection
|
|
125
|
+
plan = project["plan"] || {}
|
|
126
|
+
usage = project["usage"] || {}
|
|
127
|
+
@out.puts "#{project['name']} (#{project.id})"
|
|
128
|
+
@out.puts " environment: #{env ? env['name'] : 'production'}#{" of #{project['environments'].join(', ')}" if project['environments']}"
|
|
129
|
+
if storage
|
|
130
|
+
@out.puts " storage: #{storage['provider']} (#{storage['mode']})#{storage['status'] ? ", #{storage['status']}" : ''}"
|
|
131
|
+
else
|
|
132
|
+
@out.puts " storage: none connected"
|
|
133
|
+
end
|
|
134
|
+
if plan["name"]
|
|
135
|
+
used = usage["storage_bytes_used"].to_i
|
|
136
|
+
@out.puts " plan: #{plan['name']}, #{human_size(used)} of #{human_size(plan['storage_bytes'].to_i)} used, " \
|
|
137
|
+
"#{usage['projects_used']} of #{plan['project_limit'] || 'unlimited'} projects"
|
|
138
|
+
end
|
|
139
|
+
@out.puts " policies: #{project.upload_policies.map { |p| "#{p.name} (#{p.visibility}, #{human_size(p.maximum_size.to_i)})" }.join(', ').then { _1.empty? ? 'none' : _1 }}"
|
|
140
|
+
@out.puts " transforms: #{project.transforms.map(&:name).join(', ').then { _1.empty? ? 'none' : _1 }}"
|
|
141
|
+
0
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def upload
|
|
145
|
+
options = { policy: nil }
|
|
146
|
+
parser = OptionParser.new do |o|
|
|
147
|
+
o.on("--policy NAME") { |v| options[:policy] = v }
|
|
148
|
+
o.on("--content-type TYPE") { |v| options[:content_type] = v }
|
|
149
|
+
end
|
|
150
|
+
paths = parser.parse(@argv)
|
|
151
|
+
path = paths.first
|
|
152
|
+
if path.nil? || options[:policy].nil?
|
|
153
|
+
@err.puts "Usage: file_hutch upload PATH --policy NAME"
|
|
154
|
+
return 2
|
|
155
|
+
end
|
|
156
|
+
unless ::File.file?(path)
|
|
157
|
+
@err.puts "No such file: #{path}"
|
|
158
|
+
return 2
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
file = client.upload(path, policy: options[:policy], content_type: options[:content_type])
|
|
162
|
+
@out.puts file.id
|
|
163
|
+
0
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def manifest
|
|
167
|
+
after = nil
|
|
168
|
+
loop do
|
|
169
|
+
page = client.manifest(after: after)
|
|
170
|
+
page.fetch("files").each { |entry| @out.puts JSON.generate(entry) }
|
|
171
|
+
break unless page["has_more"]
|
|
172
|
+
after = page["next_after"]
|
|
173
|
+
end
|
|
174
|
+
0
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# -- helpers ------------------------------------------------------------
|
|
178
|
+
|
|
179
|
+
def parse_options
|
|
180
|
+
options = { file: DEFAULT_FILE, prune: false, yes: false }
|
|
181
|
+
parser = OptionParser.new do |o|
|
|
182
|
+
o.on("--prune") { options[:prune] = true }
|
|
183
|
+
o.on("--yes", "-y") { options[:yes] = true }
|
|
184
|
+
end
|
|
185
|
+
rest = parser.parse(@argv)
|
|
186
|
+
options[:file] = rest.first if rest.first
|
|
187
|
+
options
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def read_config(path)
|
|
191
|
+
unless ::File.file?(path)
|
|
192
|
+
@err.puts "No config file at #{path}. Run `file_hutch export > #{DEFAULT_FILE}` to start from what you have."
|
|
193
|
+
return nil
|
|
194
|
+
end
|
|
195
|
+
data = YAML.safe_load(::File.read(path), aliases: true) || {}
|
|
196
|
+
unless data.is_a?(Hash)
|
|
197
|
+
@err.puts "#{path} must be a YAML mapping (uploads:, transforms:, environments:)."
|
|
198
|
+
return nil
|
|
199
|
+
end
|
|
200
|
+
data
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def print_changes(changes)
|
|
204
|
+
changes.each do |c|
|
|
205
|
+
next if c["action"] == "noop"
|
|
206
|
+
|
|
207
|
+
line = "#{MARK.fetch(c['action'], '?')} #{c['resource']} #{c['name']}"
|
|
208
|
+
line += " #{describe_diff(c['diff'])}" if c["diff"]
|
|
209
|
+
@out.puts line
|
|
210
|
+
end
|
|
211
|
+
unchanged = changes.count { |c| c["action"] == "noop" }
|
|
212
|
+
@out.puts "= #{unchanged} unchanged" if unchanged.positive?
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def describe_diff(diff)
|
|
216
|
+
diff.map { |key, (from, to)| "#{key}: #{from.inspect} → #{to.inspect}" }.join(", ")
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def human_size(bytes)
|
|
220
|
+
return "#{bytes} B" if bytes < 1024
|
|
221
|
+
units = %w[KB MB GB TB]
|
|
222
|
+
value = bytes.to_f
|
|
223
|
+
unit = nil
|
|
224
|
+
units.each do |u|
|
|
225
|
+
value /= 1024
|
|
226
|
+
unit = u
|
|
227
|
+
break if value < 1024
|
|
228
|
+
end
|
|
229
|
+
format(value >= 10 ? "%.0f %s" : "%.1f %s", value, unit)
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FileHutch
|
|
4
|
+
# HTTP client for the FileHutch v1 API. Stdlib only.
|
|
5
|
+
#
|
|
6
|
+
# client = FileHutch::Client.new(api_key: "fh_…", url: "https://api.filehutch.com")
|
|
7
|
+
# client.upload("report.pdf", policy: "documents") # 3-step direct upload, returns the ready file
|
|
8
|
+
# client.file("file_…").signed_url(expires_in: 600)
|
|
9
|
+
class Client
|
|
10
|
+
JSON_TYPE = "application/json"
|
|
11
|
+
NET_ERRORS = [ Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::ENETUNREACH,
|
|
12
|
+
Errno::EPIPE, SocketError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout, EOFError, IOError ].freeze
|
|
13
|
+
|
|
14
|
+
attr_reader :config
|
|
15
|
+
|
|
16
|
+
# Accepts a Configuration or keyword overrides on top of the global one.
|
|
17
|
+
def initialize(config = nil, **overrides)
|
|
18
|
+
@config = (config || FileHutch.configuration).dup
|
|
19
|
+
overrides.each { |k, v| @config.public_send(:"#{k}=", v) }
|
|
20
|
+
@config.validate!
|
|
21
|
+
@base = URI(@config.url.to_s.sub(%r{/+\z}, ""))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# -- Resources ---------------------------------------------------------
|
|
25
|
+
|
|
26
|
+
def project
|
|
27
|
+
Project.new(request(:get, "/api/v1/project").fetch("project"), client: self)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def file(id)
|
|
31
|
+
File.new(request(:get, "/api/v1/files/#{path_id(id)}").fetch("file"), client: self)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def transforms
|
|
35
|
+
request(:get, "/api/v1/transforms").fetch("transforms").map { |t| Transform.new(t, client: self) }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# -- Declarative configuration (see `file_hutch plan|apply`) -----------
|
|
39
|
+
|
|
40
|
+
# The project as a config hash: {"uploads" => {...}, "transforms" => {...}, "environments" => [...]}.
|
|
41
|
+
# (`config` is the client's own settings.)
|
|
42
|
+
def project_config
|
|
43
|
+
request(:get, "/api/v1/config").fetch("config")
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# What apply would change. Read-only keys may call this.
|
|
47
|
+
def plan_config(config, prune: false)
|
|
48
|
+
request(:post, "/api/v1/config/plan", { config: config, prune: prune }).fetch("plan")
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def apply_config(config, prune: false)
|
|
52
|
+
request(:post, "/api/v1/config/apply", { config: config, prune: prune }).fetch("apply")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# One page of the export: every ready file with its object key.
|
|
56
|
+
def manifest(after: nil, limit: nil)
|
|
57
|
+
query = { after: after, limit: limit }.compact
|
|
58
|
+
path = "/api/v1/manifest"
|
|
59
|
+
path += "?#{URI.encode_www_form(query)}" unless query.empty?
|
|
60
|
+
request(:get, path)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# URL for one named transform. `expires_at` is nil for public files, which
|
|
64
|
+
# are delivered from a stable URL and never expire.
|
|
65
|
+
def transform_url(id, transform:, expires_in: nil)
|
|
66
|
+
body = { transform: transform.to_s, expires_in: expires_in }.compact
|
|
67
|
+
data = request(:post, "/api/v1/files/#{path_id(id)}/transform_url", body)
|
|
68
|
+
expires_at = data["expires_at"]
|
|
69
|
+
SignedUrl.new(url: data.fetch("url"), expires_at: expires_at && Time.iso8601(expires_at))
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def create_upload(policy:, filename:, content_type:, byte_size:, metadata: nil, checksum: nil)
|
|
73
|
+
body = { policy: policy, filename: filename, content_type: content_type, byte_size: byte_size }
|
|
74
|
+
body[:checksum] = checksum if checksum
|
|
75
|
+
body[:metadata] = metadata if metadata && !metadata.empty?
|
|
76
|
+
data = request(:post, "/api/v1/uploads", body)
|
|
77
|
+
Upload.new(data.fetch("upload"), file: File.new(data.fetch("file"), client: self), client: self)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def complete_upload(id)
|
|
81
|
+
File.new(request(:post, "/api/v1/uploads/#{path_id(id)}/complete").fetch("file"), client: self)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def signed_url(id, expires_in: nil, disposition: nil)
|
|
85
|
+
body = { expires_in: expires_in, disposition: disposition }.compact
|
|
86
|
+
data = request(:post, "/api/v1/files/#{path_id(id)}/signed_url", body)
|
|
87
|
+
SignedUrl.new(url: data.fetch("url"), expires_at: Time.iso8601(data.fetch("expires_at")))
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def delete_file(id)
|
|
91
|
+
request(:delete, "/api/v1/files/#{path_id(id)}")
|
|
92
|
+
true
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# -- The whole upload flow, server side --------------------------------
|
|
96
|
+
#
|
|
97
|
+
# source: a path, Pathname, File, Tempfile, StringIO, ActionDispatch::Http::UploadedFile,
|
|
98
|
+
# or a String of bytes (pass filename: then).
|
|
99
|
+
# Returns the ready FileHutch::File. Bytes go straight to storage.
|
|
100
|
+
# verify: sends an MD5 of the bytes so FileHutch refuses the upload if what
|
|
101
|
+
# arrives is not what left. Costs one pass over the file; on by default
|
|
102
|
+
# because a corrupted upload that completes is worse than a slow one.
|
|
103
|
+
def upload(source, policy:, filename: nil, content_type: nil, metadata: nil, verify: true)
|
|
104
|
+
io, name, type, size = Source.open(source, filename: filename, content_type: content_type)
|
|
105
|
+
upload = create_upload(policy: policy, filename: name, content_type: type, byte_size: size,
|
|
106
|
+
metadata: metadata, checksum: (Source.md5(io) if verify))
|
|
107
|
+
put_to_storage(upload, io)
|
|
108
|
+
complete_upload(upload.id)
|
|
109
|
+
ensure
|
|
110
|
+
io&.close if io && Source.owned?(io, source)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# PUT bytes to the storage URL in an upload authorization. Never hits an FileHutch endpoint.
|
|
114
|
+
def put_to_storage(upload, source)
|
|
115
|
+
# The name is irrelevant to a PUT — the object's key is already fixed by the
|
|
116
|
+
# authorization — but Source needs one to normalize an in-memory source, and
|
|
117
|
+
# an upload rebuilt from JSON (a browser flow proxied through your app) does
|
|
118
|
+
# not carry its file.
|
|
119
|
+
io, _name, _type, size = Source.open(
|
|
120
|
+
source, filename: upload.file&.filename || "upload", content_type: upload.headers["Content-Type"]
|
|
121
|
+
)
|
|
122
|
+
uri = URI(upload.url)
|
|
123
|
+
req = Net::HTTP.const_get(upload.method.to_s.capitalize).new(uri)
|
|
124
|
+
upload.headers.each { |k, v| req[k] = v }
|
|
125
|
+
req["Content-Length"] = size.to_s
|
|
126
|
+
req.body_stream = io
|
|
127
|
+
response = http(uri) { |h| h.request(req) }
|
|
128
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
129
|
+
raise UploadError.new("Storage rejected the upload: HTTP #{response.code} #{response.body.to_s[0, 500]}".strip,
|
|
130
|
+
code: "storage_rejected", status: response.code.to_i)
|
|
131
|
+
end
|
|
132
|
+
true
|
|
133
|
+
ensure
|
|
134
|
+
io&.close if io && Source.owned?(io, source)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# -- Transport ---------------------------------------------------------
|
|
138
|
+
|
|
139
|
+
def request(method, path, body = nil)
|
|
140
|
+
uri = URI.join("#{@base}/", path.sub(%r{\A/}, ""))
|
|
141
|
+
req = Net::HTTP.const_get(method.to_s.capitalize).new(uri)
|
|
142
|
+
req["Authorization"] = "Bearer #{config.api_key}"
|
|
143
|
+
req["Accept"] = JSON_TYPE
|
|
144
|
+
req["User-Agent"] = config.user_agent
|
|
145
|
+
if body
|
|
146
|
+
req["Content-Type"] = JSON_TYPE
|
|
147
|
+
req.body = JSON.generate(body)
|
|
148
|
+
end
|
|
149
|
+
response = http(uri) { |h| h.request(req) }
|
|
150
|
+
log(method, uri, response)
|
|
151
|
+
parse(response)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
private
|
|
155
|
+
|
|
156
|
+
def http(uri)
|
|
157
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: config.open_timeout,
|
|
158
|
+
read_timeout: config.read_timeout, write_timeout: config.write_timeout) { |h| yield h }
|
|
159
|
+
rescue *NET_ERRORS => e
|
|
160
|
+
raise ConnectionError.new("#{e.class}: #{e.message} (#{uri.host})", e)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def parse(response)
|
|
164
|
+
status = response.code.to_i
|
|
165
|
+
return nil if status == 204 || response.body.to_s.empty? && response.is_a?(Net::HTTPSuccess)
|
|
166
|
+
|
|
167
|
+
data = begin
|
|
168
|
+
JSON.parse(response.body)
|
|
169
|
+
rescue JSON::ParserError
|
|
170
|
+
nil
|
|
171
|
+
end
|
|
172
|
+
return data if response.is_a?(Net::HTTPSuccess)
|
|
173
|
+
|
|
174
|
+
raise ApiError.build(status, data)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def path_id(id)
|
|
178
|
+
value = id.respond_to?(:id) ? id.id : id.to_s
|
|
179
|
+
raise ArgumentError, "expected an FileHutch id, got #{id.inspect}" if value.to_s.empty? || value.to_s.include?("/")
|
|
180
|
+
URI.encode_www_form_component(value)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def log(method, uri, response)
|
|
184
|
+
config.logger&.debug { "[file_hutch] #{method.to_s.upcase} #{uri.path} -> #{response.code}" }
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Normalizes the many things Ruby calls "a file" into [io, filename, content_type, byte_size].
|
|
188
|
+
module Source
|
|
189
|
+
TYPES = {
|
|
190
|
+
".pdf" => "application/pdf", ".png" => "image/png", ".jpg" => "image/jpeg", ".jpeg" => "image/jpeg",
|
|
191
|
+
".gif" => "image/gif", ".webp" => "image/webp", ".avif" => "image/avif", ".svg" => "image/svg+xml",
|
|
192
|
+
".heic" => "image/heic", ".txt" => "text/plain", ".csv" => "text/csv", ".json" => "application/json",
|
|
193
|
+
".zip" => "application/zip", ".mp4" => "video/mp4", ".mp3" => "audio/mpeg", ".webm" => "video/webm",
|
|
194
|
+
".doc" => "application/msword", ".xls" => "application/vnd.ms-excel",
|
|
195
|
+
".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
196
|
+
".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
197
|
+
".pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
|
198
|
+
}.freeze
|
|
199
|
+
FALLBACK = "application/octet-stream"
|
|
200
|
+
|
|
201
|
+
CHUNK = 1_048_576 # 1 MiB; this gem is stdlib only, so no 1.megabyte
|
|
202
|
+
|
|
203
|
+
module_function
|
|
204
|
+
|
|
205
|
+
def open(source, filename: nil, content_type: nil)
|
|
206
|
+
io = to_io(source, filename)
|
|
207
|
+
name = filename || guess_filename(source, io)
|
|
208
|
+
raise ArgumentError, "filename is required for #{source.class} sources" if name.nil? || name.to_s.empty?
|
|
209
|
+
type = content_type || guess_content_type(source, name)
|
|
210
|
+
io.rewind if io.respond_to?(:rewind)
|
|
211
|
+
size = io.respond_to?(:size) ? io.size : io.stat.size
|
|
212
|
+
[ io, ::File.basename(name.to_s), type, size ]
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Streamed rather than read whole: an upload can be far larger than the
|
|
216
|
+
# memory the process has, and rewinding afterwards leaves the IO exactly
|
|
217
|
+
# as it was found so the PUT still starts at the beginning.
|
|
218
|
+
def md5(io)
|
|
219
|
+
digest = Digest::MD5.new
|
|
220
|
+
io.rewind if io.respond_to?(:rewind)
|
|
221
|
+
while (chunk = io.read(CHUNK))
|
|
222
|
+
digest << chunk
|
|
223
|
+
end
|
|
224
|
+
io.rewind if io.respond_to?(:rewind)
|
|
225
|
+
digest.hexdigest
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# We close IOs we opened ourselves (paths), never the caller's.
|
|
229
|
+
def owned?(io, source) = path_like?(source) && io.is_a?(::File)
|
|
230
|
+
|
|
231
|
+
def path_like?(source) = source.is_a?(Pathname) || (source.is_a?(String) && source.encoding != Encoding::BINARY && ::File.file?(source))
|
|
232
|
+
|
|
233
|
+
def to_io(source, filename)
|
|
234
|
+
return ::File.open(source.to_s, "rb") if path_like?(source)
|
|
235
|
+
return source.tempfile if source.respond_to?(:tempfile) && source.tempfile
|
|
236
|
+
return source if source.respond_to?(:read)
|
|
237
|
+
return StringIO.new(source.b) if source.is_a?(String) && filename
|
|
238
|
+
raise ArgumentError, "cannot upload #{source.class}; pass a path, IO, or bytes with filename:"
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def guess_filename(source, io)
|
|
242
|
+
return source.original_filename if source.respond_to?(:original_filename)
|
|
243
|
+
return ::File.basename(source.to_s) if path_like?(source)
|
|
244
|
+
return ::File.basename(io.path) if io.respond_to?(:path) && io.path
|
|
245
|
+
nil
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def guess_content_type(source, name)
|
|
249
|
+
return source.content_type if source.respond_to?(:content_type) && !source.content_type.to_s.empty?
|
|
250
|
+
return Marcel::MimeType.for(name: name.to_s) if defined?(Marcel::MimeType)
|
|
251
|
+
TYPES.fetch(::File.extname(name.to_s).downcase, FALLBACK)
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FileHutch
|
|
4
|
+
# Global settings. Every value can come from the environment so a deploy
|
|
5
|
+
# (or an AI agent) can configure the gem without touching Ruby.
|
|
6
|
+
class Configuration
|
|
7
|
+
DEFAULT_URL = "https://api.filehutch.com"
|
|
8
|
+
|
|
9
|
+
attr_accessor :api_key, :url, :open_timeout, :read_timeout, :write_timeout, :logger, :user_agent
|
|
10
|
+
|
|
11
|
+
# Rails direct uploads (see FileHutch::Engine).
|
|
12
|
+
# authorize_direct_upload = ->(controller, policy) { controller.current_user.present? }
|
|
13
|
+
attr_accessor :authorize_direct_upload, :direct_upload_parent_controller
|
|
14
|
+
|
|
15
|
+
def initialize
|
|
16
|
+
@api_key = ENV["FILE_HUTCH_API_KEY"]
|
|
17
|
+
@url = ENV.fetch("FILE_HUTCH_URL", DEFAULT_URL)
|
|
18
|
+
@open_timeout = 5
|
|
19
|
+
@read_timeout = 30
|
|
20
|
+
@write_timeout = 120
|
|
21
|
+
@logger = nil
|
|
22
|
+
@user_agent = "file_hutch-ruby/#{VERSION} ruby/#{RUBY_VERSION}"
|
|
23
|
+
@authorize_direct_upload = nil
|
|
24
|
+
@direct_upload_parent_controller = "ApplicationController"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def validate!
|
|
28
|
+
raise ConfigurationError, "FileHutch.config.api_key is missing (set FILE_HUTCH_API_KEY)" if api_key.nil? || api_key.to_s.strip.empty?
|
|
29
|
+
raise ConfigurationError, "FileHutch.config.url is missing (set FILE_HUTCH_URL)" if url.nil? || url.to_s.strip.empty?
|
|
30
|
+
self
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FileHutch
|
|
4
|
+
class Error < StandardError; end
|
|
5
|
+
|
|
6
|
+
# Client-side problems: missing API key, bad arguments.
|
|
7
|
+
class ConfigurationError < Error; end
|
|
8
|
+
|
|
9
|
+
# A webhook body was not signed by FileHutch with your endpoint's secret.
|
|
10
|
+
class SignatureVerificationError < Error; end
|
|
11
|
+
|
|
12
|
+
# Could not reach FileHutch or storage (DNS, timeout, TLS, reset).
|
|
13
|
+
class ConnectionError < Error
|
|
14
|
+
attr_reader :cause_error
|
|
15
|
+
|
|
16
|
+
def initialize(message, cause_error = nil)
|
|
17
|
+
super(message)
|
|
18
|
+
@cause_error = cause_error
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# FileHutch answered with {"error": {"code", "message", "details"}}.
|
|
23
|
+
class ApiError < Error
|
|
24
|
+
attr_reader :code, :status, :details
|
|
25
|
+
|
|
26
|
+
def initialize(message, code: nil, status: nil, details: nil)
|
|
27
|
+
super(message)
|
|
28
|
+
@code = code&.to_s
|
|
29
|
+
@status = status
|
|
30
|
+
@details = details
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
CODE_CLASSES = {
|
|
34
|
+
"unauthorized" => :AuthenticationError,
|
|
35
|
+
"not_found" => :NotFoundError,
|
|
36
|
+
"invalid" => :InvalidRequestError,
|
|
37
|
+
"policy_violation" => :PolicyError,
|
|
38
|
+
"policy_not_found" => :PolicyError,
|
|
39
|
+
"storage_not_ready" => :StorageNotReadyError,
|
|
40
|
+
"invalid_state" => :InvalidStateError,
|
|
41
|
+
"not_ready" => :InvalidStateError,
|
|
42
|
+
"already_deleted" => :InvalidStateError,
|
|
43
|
+
"not_public" => :InvalidStateError,
|
|
44
|
+
"no_public_base_url" => :InvalidStateError,
|
|
45
|
+
"upload_expired" => :UploadError,
|
|
46
|
+
"upload_incomplete" => :UploadError,
|
|
47
|
+
"size_mismatch" => :UploadError,
|
|
48
|
+
"transform_not_found" => :TransformError,
|
|
49
|
+
"not_transformable" => :TransformError,
|
|
50
|
+
"transforms_unsupported" => :TransformsUnsupportedError,
|
|
51
|
+
"plan_limit" => :PlanLimitError,
|
|
52
|
+
"read_only_key" => :PermissionError,
|
|
53
|
+
"invalid_config" => :ConfigError,
|
|
54
|
+
"environment_not_found" => :InvalidRequestError,
|
|
55
|
+
"not_verified" => :InvalidStateError,
|
|
56
|
+
"storage_error" => :StorageError,
|
|
57
|
+
"verification_failed" => :StorageError
|
|
58
|
+
}.freeze
|
|
59
|
+
|
|
60
|
+
STATUS_CLASSES = { 401 => :AuthenticationError, 403 => :PermissionError, 404 => :NotFoundError,
|
|
61
|
+
402 => :PlanLimitError, 409 => :InvalidStateError, 410 => :InvalidStateError, 422 => :InvalidRequestError,
|
|
62
|
+
429 => :RateLimitError, 502 => :StorageError }.freeze
|
|
63
|
+
|
|
64
|
+
# Picks the most specific subclass for an error payload.
|
|
65
|
+
def self.build(status, body)
|
|
66
|
+
error = body.is_a?(Hash) && body["error"].is_a?(Hash) ? body["error"] : {}
|
|
67
|
+
code = error["code"]&.to_s
|
|
68
|
+
message = error["message"] || (status >= 500 ? "FileHutch returned HTTP #{status}" : "Request failed with HTTP #{status}")
|
|
69
|
+
klass_name = CODE_CLASSES[code] || STATUS_CLASSES[status] || (status >= 500 ? :ServerError : :ApiError)
|
|
70
|
+
FileHutch.const_get(klass_name).new(message, code: code, status: status, details: error["details"])
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
class AuthenticationError < ApiError; end
|
|
75
|
+
# The key is valid but may not do this: a read-only key calling a write endpoint.
|
|
76
|
+
class PermissionError < AuthenticationError; end
|
|
77
|
+
class NotFoundError < ApiError; end
|
|
78
|
+
class InvalidRequestError < ApiError; end
|
|
79
|
+
class PolicyError < InvalidRequestError; end
|
|
80
|
+
# A declarative config the API refused: unknown key, bad size, bad name.
|
|
81
|
+
class ConfigError < InvalidRequestError; end
|
|
82
|
+
class StorageNotReadyError < ApiError; end
|
|
83
|
+
class InvalidStateError < ApiError; end
|
|
84
|
+
# A named transform is unknown, or the file is not an image.
|
|
85
|
+
class TransformError < InvalidRequestError; end
|
|
86
|
+
# The project's storage cannot render transforms. The message says what to set up.
|
|
87
|
+
class TransformsUnsupportedError < TransformError; end
|
|
88
|
+
# The team's plan is out of storage or projects. The message names the plan and what to do.
|
|
89
|
+
class PlanLimitError < ApiError; end
|
|
90
|
+
class RateLimitError < ApiError; end
|
|
91
|
+
class ServerError < ApiError; end
|
|
92
|
+
class StorageError < ApiError; end
|
|
93
|
+
|
|
94
|
+
# The direct PUT to storage failed, or FileHutch could not verify it.
|
|
95
|
+
class UploadError < ApiError; end
|
|
96
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FileHutch
|
|
4
|
+
# A stored file. `id` ("file_…") is the only thing your app should persist.
|
|
5
|
+
class File < Resource
|
|
6
|
+
ID_PATTERN = /\Afile_[0-9A-Za-z]{20}\z/
|
|
7
|
+
STATUSES = %w[pending ready failed deleted].freeze
|
|
8
|
+
|
|
9
|
+
attribute :filename, :content_type, :byte_size, :checksum, :visibility, :status, :metadata,
|
|
10
|
+
:policy, :storage_connection_id, :url
|
|
11
|
+
time_attribute :created_at, :updated_at
|
|
12
|
+
|
|
13
|
+
def self.id?(value) = value.is_a?(String) && value.match?(ID_PATTERN)
|
|
14
|
+
|
|
15
|
+
def self.find(id, client: FileHutch.client) = client.file(id)
|
|
16
|
+
|
|
17
|
+
STATUSES.each { |s| define_method(:"#{s}?") { status == s } }
|
|
18
|
+
|
|
19
|
+
def public? = visibility == "public"
|
|
20
|
+
def private? = visibility == "private"
|
|
21
|
+
def image? = content_type.to_s.start_with?("image/")
|
|
22
|
+
def pdf? = content_type == "application/pdf"
|
|
23
|
+
def metadata = self["metadata"] || {}
|
|
24
|
+
|
|
25
|
+
# Named transform URLs, keyed by name. Filled for ready public images on
|
|
26
|
+
# storage that can render them; empty otherwise.
|
|
27
|
+
def transforms = self["transforms"] || {}
|
|
28
|
+
|
|
29
|
+
# URL for one named transform. Free for public images (the URL is already
|
|
30
|
+
# on the payload); one request for private ones, which get a signed source.
|
|
31
|
+
def transform_url(name, expires_in: nil)
|
|
32
|
+
transforms[name.to_s] || client!.transform_url(id, transform: name, expires_in: expires_in).url
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Short-lived URL that works for private and public files alike.
|
|
36
|
+
def signed_url(expires_in: nil, disposition: nil)
|
|
37
|
+
client!.signed_url(id, expires_in: expires_in, disposition: disposition).url
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Public URL when the file is public and ready, otherwise a signed URL.
|
|
41
|
+
def url_or_signed_url(expires_in: nil, disposition: nil)
|
|
42
|
+
url || signed_url(expires_in: expires_in, disposition: disposition)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def reload = client!.file(id)
|
|
46
|
+
|
|
47
|
+
# Deletes the bytes; the id keeps resolving with status "deleted".
|
|
48
|
+
def delete
|
|
49
|
+
client!.delete_file(id)
|
|
50
|
+
reload
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
SignedUrl = Struct.new(:url, :expires_at, keyword_init: true) do
|
|
55
|
+
def to_s = url
|
|
56
|
+
end
|
|
57
|
+
end
|