graphile-settings 6.13.8 → 6.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/presigned-url-resolver.d.ts +4 -3
- package/esm/presigned-url-resolver.js +4 -3
- package/esm/upload-resolver.d.ts +16 -7
- package/esm/upload-resolver.js +172 -64
- package/package.json +5 -5
- package/presigned-url-resolver.d.ts +4 -3
- package/presigned-url-resolver.js +4 -3
- package/upload-resolver.d.ts +16 -7
- package/upload-resolver.js +171 -63
|
@@ -19,9 +19,10 @@ import type { BucketNameResolver, EnsureBucketProvisioned, S3Config } from 'grap
|
|
|
19
19
|
* pgpmDefaults → config file → env vars), creates an S3Client, and caches
|
|
20
20
|
* the result. Same CDN config as upload-resolver.ts.
|
|
21
21
|
*
|
|
22
|
-
* NOTE: The `bucket` field here is the
|
|
23
|
-
*
|
|
24
|
-
*
|
|
22
|
+
* NOTE: The `bucket` field here is only the connection's default and is never
|
|
23
|
+
* uploaded to. Every managed upload names its bucket explicitly, resolved from
|
|
24
|
+
* the tenant's logical bucket row via `resolveBucketName`; there is no
|
|
25
|
+
* environment-global upload bucket.
|
|
25
26
|
*/
|
|
26
27
|
export declare function getPresignedUrlS3Config(): S3Config;
|
|
27
28
|
/**
|
|
@@ -24,9 +24,10 @@ let s3Config = null;
|
|
|
24
24
|
* pgpmDefaults → config file → env vars), creates an S3Client, and caches
|
|
25
25
|
* the result. Same CDN config as upload-resolver.ts.
|
|
26
26
|
*
|
|
27
|
-
* NOTE: The `bucket` field here is the
|
|
28
|
-
*
|
|
29
|
-
*
|
|
27
|
+
* NOTE: The `bucket` field here is only the connection's default and is never
|
|
28
|
+
* uploaded to. Every managed upload names its bucket explicitly, resolved from
|
|
29
|
+
* the tenant's logical bucket row via `resolveBucketName`; there is no
|
|
30
|
+
* environment-global upload bucket.
|
|
30
31
|
*/
|
|
31
32
|
export function getPresignedUrlS3Config() {
|
|
32
33
|
if (s3Config)
|
package/esm/upload-resolver.d.ts
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Upload resolver for the Constructive upload plugin.
|
|
2
|
+
* Upload resolver for the Constructive upload plugin (multipart `Upload` scalar).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* and
|
|
4
|
+
* This is the streaming transport into the *managed* storage lane: bytes arrive
|
|
5
|
+
* on the mutation, and the file they carry gets the same treatment a presigned
|
|
6
|
+
* upload gets — a bucket resolved inside the tenant, a content-addressed key, a
|
|
7
|
+
* files row, and a projection document naming that row.
|
|
6
8
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
+
* It used to be a second storage model: stream to `BUCKET_NAME` under a random
|
|
10
|
+
* key, hand back a URL, record nothing. Objects written that way belonged to no
|
|
11
|
+
* database, could not be deduplicated, listed, or access-controlled, and storage
|
|
12
|
+
* GC could not see that a document still pointed at them. There is no
|
|
13
|
+
* environment bucket in this path any more; `cdn.*` supplies S3 credentials and
|
|
14
|
+
* an endpoint only.
|
|
9
15
|
*
|
|
10
|
-
*
|
|
16
|
+
* Compatibility: `image`/`upload` columns still receive `url` alongside the new
|
|
17
|
+
* `id`/`key`/`bucket_id`/`size` fields, so existing readers of `photo.url` keep
|
|
18
|
+
* working while they migrate to `id` + the files row's late-bound `downloadUrl`.
|
|
19
|
+
*
|
|
20
|
+
* ENV VARS (S3 connection only):
|
|
11
21
|
* BUCKET_PROVIDER - 'minio' | 's3' (default: 'minio')
|
|
12
|
-
* BUCKET_NAME - bucket name (default: 'test-bucket')
|
|
13
22
|
* AWS_REGION - AWS region (default: 'us-east-1')
|
|
14
23
|
* AWS_ACCESS_KEY - access key (default: 'minioadmin')
|
|
15
24
|
* AWS_SECRET_KEY - secret key (default: 'minioadmin')
|
package/esm/upload-resolver.js
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Upload resolver for the Constructive upload plugin.
|
|
2
|
+
* Upload resolver for the Constructive upload plugin (multipart `Upload` scalar).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* and
|
|
4
|
+
* This is the streaming transport into the *managed* storage lane: bytes arrive
|
|
5
|
+
* on the mutation, and the file they carry gets the same treatment a presigned
|
|
6
|
+
* upload gets — a bucket resolved inside the tenant, a content-addressed key, a
|
|
7
|
+
* files row, and a projection document naming that row.
|
|
6
8
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
+
* It used to be a second storage model: stream to `BUCKET_NAME` under a random
|
|
10
|
+
* key, hand back a URL, record nothing. Objects written that way belonged to no
|
|
11
|
+
* database, could not be deduplicated, listed, or access-controlled, and storage
|
|
12
|
+
* GC could not see that a document still pointed at them. There is no
|
|
13
|
+
* environment bucket in this path any more; `cdn.*` supplies S3 credentials and
|
|
14
|
+
* an endpoint only.
|
|
9
15
|
*
|
|
10
|
-
*
|
|
16
|
+
* Compatibility: `image`/`upload` columns still receive `url` alongside the new
|
|
17
|
+
* `id`/`key`/`bucket_id`/`size` fields, so existing readers of `photo.url` keep
|
|
18
|
+
* working while they migrate to `id` + the files row's late-bound `downloadUrl`.
|
|
19
|
+
*
|
|
20
|
+
* ENV VARS (S3 connection only):
|
|
11
21
|
* BUCKET_PROVIDER - 'minio' | 's3' (default: 'minio')
|
|
12
|
-
* BUCKET_NAME - bucket name (default: 'test-bucket')
|
|
13
22
|
* AWS_REGION - AWS region (default: 'us-east-1')
|
|
14
23
|
* AWS_ACCESS_KEY - access key (default: 'minioadmin')
|
|
15
24
|
* AWS_SECRET_KEY - secret key (default: 'minioadmin')
|
|
@@ -17,100 +26,199 @@
|
|
|
17
26
|
*/
|
|
18
27
|
import { getEnvOptions } from '@constructive-io/graphql-env';
|
|
19
28
|
import Streamer from '@constructive-io/s3-streamer';
|
|
20
|
-
import uploadNames from '@constructive-io/upload-names';
|
|
21
29
|
import { Logger } from '@pgpmjs/logger';
|
|
22
|
-
import {
|
|
30
|
+
import { createHash, randomUUID } from 'crypto';
|
|
31
|
+
import { finalizeStagedUpload, resolveManagedUploadTarget, withRequestPgClient, } from 'graphile-presigned-url-plugin';
|
|
32
|
+
import { Transform } from 'stream';
|
|
33
|
+
import { createBucketNameResolver, createEnsureBucketProvisioned, getPresignedUrlS3Config, } from './presigned-url-resolver';
|
|
23
34
|
const log = new Logger('upload-resolver');
|
|
24
35
|
const DEFAULT_IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/svg+xml'];
|
|
25
36
|
let streamer = null;
|
|
26
|
-
|
|
37
|
+
/**
|
|
38
|
+
* The S3 streamer, built from the CDN connection settings.
|
|
39
|
+
*
|
|
40
|
+
* Deliberately constructed with no `defaultBucket`: every upload names the
|
|
41
|
+
* bucket it resolved, and a default here would be an environment-owned bucket
|
|
42
|
+
* standing in for a tenant's.
|
|
43
|
+
*/
|
|
27
44
|
function getStreamer() {
|
|
28
45
|
if (streamer)
|
|
29
46
|
return streamer;
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
bucketName = cdn.bucketName || 'test-bucket';
|
|
34
|
-
const awsRegion = cdn.awsRegion || 'us-east-1';
|
|
35
|
-
const awsAccessKey = cdn.awsAccessKey || 'minioadmin';
|
|
36
|
-
const awsSecretKey = cdn.awsSecretKey || 'minioadmin';
|
|
37
|
-
const endpoint = cdn.endpoint || 'http://localhost:9000';
|
|
38
|
-
if (process.env.NODE_ENV === 'production') {
|
|
39
|
-
if (!cdn.awsAccessKey || !cdn.awsSecretKey) {
|
|
40
|
-
log.warn('[upload-resolver] WARNING: Using default credentials in production.');
|
|
41
|
-
}
|
|
47
|
+
const { cdn = {} } = getEnvOptions();
|
|
48
|
+
if (process.env.NODE_ENV === 'production' && (!cdn.awsAccessKey || !cdn.awsSecretKey)) {
|
|
49
|
+
log.warn('[upload-resolver] WARNING: Using default credentials in production.');
|
|
42
50
|
}
|
|
43
|
-
|
|
51
|
+
const provider = cdn.provider || 'minio';
|
|
52
|
+
log.info(`[upload-resolver] Initializing: provider=${provider}`);
|
|
44
53
|
streamer = new Streamer({
|
|
45
|
-
defaultBucket: bucketName,
|
|
46
|
-
awsRegion,
|
|
47
|
-
awsSecretKey,
|
|
48
|
-
awsAccessKey,
|
|
49
|
-
endpoint,
|
|
50
54
|
provider,
|
|
55
|
+
awsRegion: cdn.awsRegion || 'us-east-1',
|
|
56
|
+
awsAccessKey: cdn.awsAccessKey || 'minioadmin',
|
|
57
|
+
awsSecretKey: cdn.awsSecretKey || 'minioadmin',
|
|
58
|
+
endpoint: cdn.endpoint || 'http://localhost:9000',
|
|
51
59
|
});
|
|
52
60
|
return streamer;
|
|
53
61
|
}
|
|
54
62
|
/**
|
|
55
|
-
*
|
|
56
|
-
*
|
|
63
|
+
* The upload lane's view of the presigned plugin's options: the same S3
|
|
64
|
+
* connection, physical-name policy, and provisioning hook the presigned lane
|
|
65
|
+
* uses, so both transports resolve identical coordinates for a bucket.
|
|
66
|
+
*
|
|
67
|
+
* Built on first upload rather than at import time — `createBucketNameResolver`
|
|
68
|
+
* throws on a missing name prefix, and that must surface as a failed upload, not
|
|
69
|
+
* as a server that will not boot.
|
|
57
70
|
*/
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
71
|
+
let managedOptions = null;
|
|
72
|
+
function getManagedOptions() {
|
|
73
|
+
if (!managedOptions) {
|
|
74
|
+
managedOptions = {
|
|
75
|
+
s3: getPresignedUrlS3Config,
|
|
76
|
+
resolveBucketName: createBucketNameResolver(),
|
|
77
|
+
ensureBucketProvisioned: createEnsureBucketProvisioned(),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
return managedOptions;
|
|
81
|
+
}
|
|
82
|
+
/** A staging key: transient, and never what the object ends up under. */
|
|
83
|
+
function stagingKey() {
|
|
84
|
+
return `.staging/${randomUUID()}`;
|
|
85
|
+
}
|
|
86
|
+
async function resolveDatabaseId(pgClient) {
|
|
87
|
+
const result = await pgClient.query({ text: `SELECT jwt_private.current_database_id() AS id` });
|
|
88
|
+
return result.rows[0]?.id ?? null;
|
|
61
89
|
}
|
|
62
90
|
/**
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
* Returns different shapes based on the column's type hint:
|
|
66
|
-
* - 'image' / 'upload' → { filename, mime, url } (for jsonb domain columns)
|
|
67
|
-
* - 'attachment' / default → url string (for text domain columns)
|
|
91
|
+
* Which default bucket an *unregistered* column resolves to.
|
|
68
92
|
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
93
|
+
* Columns written by the pre-managed resolver held a directly-embedded URL, so
|
|
94
|
+
* their readers assume a publicly addressable object; resolving them to the
|
|
95
|
+
* private default would break every page rendering one. A registered column
|
|
96
|
+
* states its own intent and this is not consulted.
|
|
71
97
|
*/
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const { filename } = upload;
|
|
76
|
-
const key = generateKey(filename);
|
|
77
|
-
// MIME type validation from smart tags
|
|
78
|
-
const typ = type || tags?.type;
|
|
98
|
+
const LEGACY_DEFAULT_PUBLIC_ACCESS = true;
|
|
99
|
+
/** The mime allowlist for a column, from its smart tags or its type. */
|
|
100
|
+
function allowedMimeTypes(tags, typ) {
|
|
79
101
|
const VALID_MIME = /^[a-z]+\/[a-z0-9][a-z0-9!#$&\-.^_+]*$/i;
|
|
80
|
-
|
|
81
|
-
|
|
102
|
+
if (tags?.mime) {
|
|
103
|
+
return String(tags.mime)
|
|
82
104
|
.trim()
|
|
83
105
|
.split(',')
|
|
84
106
|
.map((a) => a.trim())
|
|
85
|
-
.filter((m) => VALID_MIME.test(m))
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
107
|
+
.filter((m) => VALID_MIME.test(m));
|
|
108
|
+
}
|
|
109
|
+
return typ === 'image' ? DEFAULT_IMAGE_MIME_TYPES : [];
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Hash and measure bytes as they stream past, without buffering them.
|
|
113
|
+
*
|
|
114
|
+
* The final key is the content hash, which is only known once the last byte has
|
|
115
|
+
* gone by — so the object is staged first and promoted after. Nothing is held in
|
|
116
|
+
* memory: a 2GB upload streams through this the same as a 2KB one.
|
|
117
|
+
*/
|
|
118
|
+
function hashingPassThrough() {
|
|
119
|
+
const hash = createHash('sha256');
|
|
120
|
+
let bytes = 0;
|
|
121
|
+
const stream = new Transform({
|
|
122
|
+
transform(chunk, _encoding, callback) {
|
|
123
|
+
hash.update(chunk);
|
|
124
|
+
bytes += chunk.length;
|
|
125
|
+
callback(null, chunk);
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
return Object.assign(stream, {
|
|
129
|
+
digest: () => hash.digest('hex'),
|
|
130
|
+
bytes: () => bytes,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Stream an upload into managed storage and return the value the column stores.
|
|
135
|
+
*
|
|
136
|
+
* Shape by column type hint:
|
|
137
|
+
* * `image` / `upload` (jsonb domains) → the projection document, including a
|
|
138
|
+
* compatibility `url` for public buckets.
|
|
139
|
+
* * `attachment` (text domain) → the object's public URL. A text column cannot
|
|
140
|
+
* hold a projection, so the files row is still authoritative but the column
|
|
141
|
+
* itself carries no id; the row keeps the object alive. A private bucket
|
|
142
|
+
* raises rather than storing an expiring presigned URL in a column.
|
|
143
|
+
*/
|
|
144
|
+
async function uploadResolver(upload, _args, context, info) {
|
|
145
|
+
const { tags, type, field } = info.uploadPlugin;
|
|
146
|
+
const typ = type || tags?.type;
|
|
147
|
+
const withPgClient = context?.withPgClient;
|
|
148
|
+
const pgSettings = context?.pgSettings ?? null;
|
|
149
|
+
if (!withPgClient) {
|
|
150
|
+
throw new Error('UPLOAD_NO_PG_CLIENT: a managed upload resolves its bucket in the database, so the ' +
|
|
151
|
+
'GraphQL context must carry withPgClient');
|
|
152
|
+
}
|
|
153
|
+
if (!field) {
|
|
154
|
+
throw new Error('UPLOAD_FIELD_UNKNOWN: the upload plugin did not report which column is being written, ' +
|
|
155
|
+
'so the storage module and bucket backing it cannot be resolved');
|
|
156
|
+
}
|
|
157
|
+
const databaseId = await withRequestPgClient(withPgClient, pgSettings, (pgClient) => resolveDatabaseId(pgClient));
|
|
158
|
+
if (!databaseId)
|
|
159
|
+
throw new Error('DATABASE_NOT_FOUND');
|
|
160
|
+
const target = await resolveManagedUploadTarget({
|
|
161
|
+
options: getManagedOptions(),
|
|
162
|
+
withPgClient,
|
|
163
|
+
pgSettings,
|
|
164
|
+
databaseId,
|
|
165
|
+
field: field,
|
|
166
|
+
defaultPublicAccess: LEGACY_DEFAULT_PUBLIC_ACCESS,
|
|
167
|
+
});
|
|
168
|
+
if (typ === 'attachment' && !target.bucket.is_public) {
|
|
169
|
+
throw new Error('ATTACHMENT_BUCKET_NOT_PUBLIC: an attachment column stores a plain URL, and the resolved ' +
|
|
170
|
+
`bucket "${target.bucket.key}" is private, whose only URLs expire. Use an upload column, ` +
|
|
171
|
+
'which stores the file id and resolves a fresh download URL on read.');
|
|
172
|
+
}
|
|
173
|
+
const s3 = getStreamer();
|
|
174
|
+
const { filename } = upload;
|
|
175
|
+
// Validate before persisting: content type comes from the leading bytes, not
|
|
176
|
+
// from the client's claim about them.
|
|
89
177
|
const detected = await s3.detectContentType({
|
|
90
178
|
readStream: upload.createReadStream(),
|
|
91
179
|
filename,
|
|
92
180
|
});
|
|
93
|
-
const
|
|
94
|
-
if (
|
|
181
|
+
const allowed = allowedMimeTypes(tags, typ);
|
|
182
|
+
if (allowed.length && !allowed.includes(detected.contentType)) {
|
|
95
183
|
detected.stream.destroy();
|
|
96
184
|
throw new Error('UPLOAD_MIMETYPE');
|
|
97
185
|
}
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
|
|
186
|
+
const staged = stagingKey();
|
|
187
|
+
const hashing = hashingPassThrough();
|
|
188
|
+
const uploadResult = await s3.uploadWithContentType({
|
|
189
|
+
readStream: detected.stream.pipe(hashing),
|
|
190
|
+
contentType: detected.contentType,
|
|
101
191
|
magic: detected.magic,
|
|
102
|
-
key,
|
|
103
|
-
bucket:
|
|
192
|
+
key: staged,
|
|
193
|
+
bucket: target.physicalName,
|
|
194
|
+
});
|
|
195
|
+
// Owns the staged key from here: it either promotes it into a files row or
|
|
196
|
+
// removes it, so a failed upload leaves nothing behind in S3.
|
|
197
|
+
const { projection } = await finalizeStagedUpload({
|
|
198
|
+
target,
|
|
199
|
+
withPgClient,
|
|
200
|
+
pgSettings,
|
|
201
|
+
staged: {
|
|
202
|
+
stagingKey: staged,
|
|
203
|
+
contentHash: hashing.digest(),
|
|
204
|
+
contentType: uploadResult.contentType,
|
|
205
|
+
size: hashing.bytes(),
|
|
206
|
+
filename,
|
|
207
|
+
},
|
|
104
208
|
});
|
|
105
|
-
const url = result.upload.Location;
|
|
106
|
-
const { contentType } = result;
|
|
107
209
|
switch (typ) {
|
|
108
210
|
case 'image':
|
|
109
211
|
case 'upload':
|
|
110
|
-
|
|
212
|
+
// `filename` and `mime` were in the pre-managed shape and stay in it;
|
|
213
|
+
// `url` is populated for public buckets and deprecated in favour of `id`.
|
|
214
|
+
return { ...projection, filename, mime: uploadResult.contentType };
|
|
111
215
|
case 'attachment':
|
|
112
216
|
default:
|
|
113
|
-
|
|
217
|
+
if (!projection.url) {
|
|
218
|
+
throw new Error(`ATTACHMENT_NO_PUBLIC_URL: bucket "${target.bucket.key}" has no public URL prefix ` +
|
|
219
|
+
'configured, so there is no durable URL to store in a text column');
|
|
220
|
+
}
|
|
221
|
+
return projection.url;
|
|
114
222
|
}
|
|
115
223
|
}
|
|
116
224
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "graphile-settings",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.14.0",
|
|
4
4
|
"author": "Constructive <developers@constructive.io>",
|
|
5
5
|
"description": "graphile settings",
|
|
6
6
|
"main": "index.js",
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"@constructive-io/bucket-provisioner": "^0.24.0",
|
|
34
34
|
"@constructive-io/graphql-env": "^3.29.3",
|
|
35
35
|
"@constructive-io/graphql-types": "^3.28.3",
|
|
36
|
-
"@constructive-io/s3-streamer": "^2.
|
|
36
|
+
"@constructive-io/s3-streamer": "^2.40.0",
|
|
37
37
|
"@constructive-io/s3-utils": "^2.30.0",
|
|
38
38
|
"@constructive-io/upload-names": "^2.29.0",
|
|
39
39
|
"@dataplan/json": "1.0.1",
|
|
@@ -59,10 +59,10 @@
|
|
|
59
59
|
"graphile-meta": "^1.7.1",
|
|
60
60
|
"graphile-pg-aggregates": "^2.11.7",
|
|
61
61
|
"graphile-postgis": "^3.11.7",
|
|
62
|
-
"graphile-presigned-url-plugin": "^1.
|
|
62
|
+
"graphile-presigned-url-plugin": "^1.12.0",
|
|
63
63
|
"graphile-realtime-subscriptions": "^1.10.5",
|
|
64
64
|
"graphile-search": "^2.11.7",
|
|
65
|
-
"graphile-upload-plugin": "^2.
|
|
65
|
+
"graphile-upload-plugin": "^2.25.0",
|
|
66
66
|
"graphile-utils": "5.0.3",
|
|
67
67
|
"graphql": "16.13.0",
|
|
68
68
|
"inflekt": "^0.8.1",
|
|
@@ -91,5 +91,5 @@
|
|
|
91
91
|
"constructive",
|
|
92
92
|
"graphql"
|
|
93
93
|
],
|
|
94
|
-
"gitHead": "
|
|
94
|
+
"gitHead": "44f56ee80a348f649a22e5bd32b35f2822726509"
|
|
95
95
|
}
|
|
@@ -19,9 +19,10 @@ import type { BucketNameResolver, EnsureBucketProvisioned, S3Config } from 'grap
|
|
|
19
19
|
* pgpmDefaults → config file → env vars), creates an S3Client, and caches
|
|
20
20
|
* the result. Same CDN config as upload-resolver.ts.
|
|
21
21
|
*
|
|
22
|
-
* NOTE: The `bucket` field here is the
|
|
23
|
-
*
|
|
24
|
-
*
|
|
22
|
+
* NOTE: The `bucket` field here is only the connection's default and is never
|
|
23
|
+
* uploaded to. Every managed upload names its bucket explicitly, resolved from
|
|
24
|
+
* the tenant's logical bucket row via `resolveBucketName`; there is no
|
|
25
|
+
* environment-global upload bucket.
|
|
25
26
|
*/
|
|
26
27
|
export declare function getPresignedUrlS3Config(): S3Config;
|
|
27
28
|
/**
|
|
@@ -31,9 +31,10 @@ let s3Config = null;
|
|
|
31
31
|
* pgpmDefaults → config file → env vars), creates an S3Client, and caches
|
|
32
32
|
* the result. Same CDN config as upload-resolver.ts.
|
|
33
33
|
*
|
|
34
|
-
* NOTE: The `bucket` field here is the
|
|
35
|
-
*
|
|
36
|
-
*
|
|
34
|
+
* NOTE: The `bucket` field here is only the connection's default and is never
|
|
35
|
+
* uploaded to. Every managed upload names its bucket explicitly, resolved from
|
|
36
|
+
* the tenant's logical bucket row via `resolveBucketName`; there is no
|
|
37
|
+
* environment-global upload bucket.
|
|
37
38
|
*/
|
|
38
39
|
function getPresignedUrlS3Config() {
|
|
39
40
|
if (s3Config)
|
package/upload-resolver.d.ts
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Upload resolver for the Constructive upload plugin.
|
|
2
|
+
* Upload resolver for the Constructive upload plugin (multipart `Upload` scalar).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* and
|
|
4
|
+
* This is the streaming transport into the *managed* storage lane: bytes arrive
|
|
5
|
+
* on the mutation, and the file they carry gets the same treatment a presigned
|
|
6
|
+
* upload gets — a bucket resolved inside the tenant, a content-addressed key, a
|
|
7
|
+
* files row, and a projection document naming that row.
|
|
6
8
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
+
* It used to be a second storage model: stream to `BUCKET_NAME` under a random
|
|
10
|
+
* key, hand back a URL, record nothing. Objects written that way belonged to no
|
|
11
|
+
* database, could not be deduplicated, listed, or access-controlled, and storage
|
|
12
|
+
* GC could not see that a document still pointed at them. There is no
|
|
13
|
+
* environment bucket in this path any more; `cdn.*` supplies S3 credentials and
|
|
14
|
+
* an endpoint only.
|
|
9
15
|
*
|
|
10
|
-
*
|
|
16
|
+
* Compatibility: `image`/`upload` columns still receive `url` alongside the new
|
|
17
|
+
* `id`/`key`/`bucket_id`/`size` fields, so existing readers of `photo.url` keep
|
|
18
|
+
* working while they migrate to `id` + the files row's late-bound `downloadUrl`.
|
|
19
|
+
*
|
|
20
|
+
* ENV VARS (S3 connection only):
|
|
11
21
|
* BUCKET_PROVIDER - 'minio' | 's3' (default: 'minio')
|
|
12
|
-
* BUCKET_NAME - bucket name (default: 'test-bucket')
|
|
13
22
|
* AWS_REGION - AWS region (default: 'us-east-1')
|
|
14
23
|
* AWS_ACCESS_KEY - access key (default: 'minioadmin')
|
|
15
24
|
* AWS_SECRET_KEY - secret key (default: 'minioadmin')
|
package/upload-resolver.js
CHANGED
|
@@ -1,16 +1,25 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* Upload resolver for the Constructive upload plugin.
|
|
3
|
+
* Upload resolver for the Constructive upload plugin (multipart `Upload` scalar).
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
* and
|
|
5
|
+
* This is the streaming transport into the *managed* storage lane: bytes arrive
|
|
6
|
+
* on the mutation, and the file they carry gets the same treatment a presigned
|
|
7
|
+
* upload gets — a bucket resolved inside the tenant, a content-addressed key, a
|
|
8
|
+
* files row, and a projection document naming that row.
|
|
7
9
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
+
* It used to be a second storage model: stream to `BUCKET_NAME` under a random
|
|
11
|
+
* key, hand back a URL, record nothing. Objects written that way belonged to no
|
|
12
|
+
* database, could not be deduplicated, listed, or access-controlled, and storage
|
|
13
|
+
* GC could not see that a document still pointed at them. There is no
|
|
14
|
+
* environment bucket in this path any more; `cdn.*` supplies S3 credentials and
|
|
15
|
+
* an endpoint only.
|
|
10
16
|
*
|
|
11
|
-
*
|
|
17
|
+
* Compatibility: `image`/`upload` columns still receive `url` alongside the new
|
|
18
|
+
* `id`/`key`/`bucket_id`/`size` fields, so existing readers of `photo.url` keep
|
|
19
|
+
* working while they migrate to `id` + the files row's late-bound `downloadUrl`.
|
|
20
|
+
*
|
|
21
|
+
* ENV VARS (S3 connection only):
|
|
12
22
|
* BUCKET_PROVIDER - 'minio' | 's3' (default: 'minio')
|
|
13
|
-
* BUCKET_NAME - bucket name (default: 'test-bucket')
|
|
14
23
|
* AWS_REGION - AWS region (default: 'us-east-1')
|
|
15
24
|
* AWS_ACCESS_KEY - access key (default: 'minioadmin')
|
|
16
25
|
* AWS_SECRET_KEY - secret key (default: 'minioadmin')
|
|
@@ -23,100 +32,199 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
23
32
|
exports.constructiveUploadFieldDefinitions = void 0;
|
|
24
33
|
const graphql_env_1 = require("@constructive-io/graphql-env");
|
|
25
34
|
const s3_streamer_1 = __importDefault(require("@constructive-io/s3-streamer"));
|
|
26
|
-
const upload_names_1 = __importDefault(require("@constructive-io/upload-names"));
|
|
27
35
|
const logger_1 = require("@pgpmjs/logger");
|
|
28
36
|
const crypto_1 = require("crypto");
|
|
37
|
+
const graphile_presigned_url_plugin_1 = require("graphile-presigned-url-plugin");
|
|
38
|
+
const stream_1 = require("stream");
|
|
39
|
+
const presigned_url_resolver_1 = require("./presigned-url-resolver");
|
|
29
40
|
const log = new logger_1.Logger('upload-resolver');
|
|
30
41
|
const DEFAULT_IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/svg+xml'];
|
|
31
42
|
let streamer = null;
|
|
32
|
-
|
|
43
|
+
/**
|
|
44
|
+
* The S3 streamer, built from the CDN connection settings.
|
|
45
|
+
*
|
|
46
|
+
* Deliberately constructed with no `defaultBucket`: every upload names the
|
|
47
|
+
* bucket it resolved, and a default here would be an environment-owned bucket
|
|
48
|
+
* standing in for a tenant's.
|
|
49
|
+
*/
|
|
33
50
|
function getStreamer() {
|
|
34
51
|
if (streamer)
|
|
35
52
|
return streamer;
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
bucketName = cdn.bucketName || 'test-bucket';
|
|
40
|
-
const awsRegion = cdn.awsRegion || 'us-east-1';
|
|
41
|
-
const awsAccessKey = cdn.awsAccessKey || 'minioadmin';
|
|
42
|
-
const awsSecretKey = cdn.awsSecretKey || 'minioadmin';
|
|
43
|
-
const endpoint = cdn.endpoint || 'http://localhost:9000';
|
|
44
|
-
if (process.env.NODE_ENV === 'production') {
|
|
45
|
-
if (!cdn.awsAccessKey || !cdn.awsSecretKey) {
|
|
46
|
-
log.warn('[upload-resolver] WARNING: Using default credentials in production.');
|
|
47
|
-
}
|
|
53
|
+
const { cdn = {} } = (0, graphql_env_1.getEnvOptions)();
|
|
54
|
+
if (process.env.NODE_ENV === 'production' && (!cdn.awsAccessKey || !cdn.awsSecretKey)) {
|
|
55
|
+
log.warn('[upload-resolver] WARNING: Using default credentials in production.');
|
|
48
56
|
}
|
|
49
|
-
|
|
57
|
+
const provider = cdn.provider || 'minio';
|
|
58
|
+
log.info(`[upload-resolver] Initializing: provider=${provider}`);
|
|
50
59
|
streamer = new s3_streamer_1.default({
|
|
51
|
-
defaultBucket: bucketName,
|
|
52
|
-
awsRegion,
|
|
53
|
-
awsSecretKey,
|
|
54
|
-
awsAccessKey,
|
|
55
|
-
endpoint,
|
|
56
60
|
provider,
|
|
61
|
+
awsRegion: cdn.awsRegion || 'us-east-1',
|
|
62
|
+
awsAccessKey: cdn.awsAccessKey || 'minioadmin',
|
|
63
|
+
awsSecretKey: cdn.awsSecretKey || 'minioadmin',
|
|
64
|
+
endpoint: cdn.endpoint || 'http://localhost:9000',
|
|
57
65
|
});
|
|
58
66
|
return streamer;
|
|
59
67
|
}
|
|
60
68
|
/**
|
|
61
|
-
*
|
|
62
|
-
*
|
|
69
|
+
* The upload lane's view of the presigned plugin's options: the same S3
|
|
70
|
+
* connection, physical-name policy, and provisioning hook the presigned lane
|
|
71
|
+
* uses, so both transports resolve identical coordinates for a bucket.
|
|
72
|
+
*
|
|
73
|
+
* Built on first upload rather than at import time — `createBucketNameResolver`
|
|
74
|
+
* throws on a missing name prefix, and that must surface as a failed upload, not
|
|
75
|
+
* as a server that will not boot.
|
|
63
76
|
*/
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
77
|
+
let managedOptions = null;
|
|
78
|
+
function getManagedOptions() {
|
|
79
|
+
if (!managedOptions) {
|
|
80
|
+
managedOptions = {
|
|
81
|
+
s3: presigned_url_resolver_1.getPresignedUrlS3Config,
|
|
82
|
+
resolveBucketName: (0, presigned_url_resolver_1.createBucketNameResolver)(),
|
|
83
|
+
ensureBucketProvisioned: (0, presigned_url_resolver_1.createEnsureBucketProvisioned)(),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
return managedOptions;
|
|
87
|
+
}
|
|
88
|
+
/** A staging key: transient, and never what the object ends up under. */
|
|
89
|
+
function stagingKey() {
|
|
90
|
+
return `.staging/${(0, crypto_1.randomUUID)()}`;
|
|
91
|
+
}
|
|
92
|
+
async function resolveDatabaseId(pgClient) {
|
|
93
|
+
const result = await pgClient.query({ text: `SELECT jwt_private.current_database_id() AS id` });
|
|
94
|
+
return result.rows[0]?.id ?? null;
|
|
67
95
|
}
|
|
68
96
|
/**
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* Returns different shapes based on the column's type hint:
|
|
72
|
-
* - 'image' / 'upload' → { filename, mime, url } (for jsonb domain columns)
|
|
73
|
-
* - 'attachment' / default → url string (for text domain columns)
|
|
97
|
+
* Which default bucket an *unregistered* column resolves to.
|
|
74
98
|
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
99
|
+
* Columns written by the pre-managed resolver held a directly-embedded URL, so
|
|
100
|
+
* their readers assume a publicly addressable object; resolving them to the
|
|
101
|
+
* private default would break every page rendering one. A registered column
|
|
102
|
+
* states its own intent and this is not consulted.
|
|
77
103
|
*/
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
const { filename } = upload;
|
|
82
|
-
const key = generateKey(filename);
|
|
83
|
-
// MIME type validation from smart tags
|
|
84
|
-
const typ = type || tags?.type;
|
|
104
|
+
const LEGACY_DEFAULT_PUBLIC_ACCESS = true;
|
|
105
|
+
/** The mime allowlist for a column, from its smart tags or its type. */
|
|
106
|
+
function allowedMimeTypes(tags, typ) {
|
|
85
107
|
const VALID_MIME = /^[a-z]+\/[a-z0-9][a-z0-9!#$&\-.^_+]*$/i;
|
|
86
|
-
|
|
87
|
-
|
|
108
|
+
if (tags?.mime) {
|
|
109
|
+
return String(tags.mime)
|
|
88
110
|
.trim()
|
|
89
111
|
.split(',')
|
|
90
112
|
.map((a) => a.trim())
|
|
91
|
-
.filter((m) => VALID_MIME.test(m))
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
113
|
+
.filter((m) => VALID_MIME.test(m));
|
|
114
|
+
}
|
|
115
|
+
return typ === 'image' ? DEFAULT_IMAGE_MIME_TYPES : [];
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Hash and measure bytes as they stream past, without buffering them.
|
|
119
|
+
*
|
|
120
|
+
* The final key is the content hash, which is only known once the last byte has
|
|
121
|
+
* gone by — so the object is staged first and promoted after. Nothing is held in
|
|
122
|
+
* memory: a 2GB upload streams through this the same as a 2KB one.
|
|
123
|
+
*/
|
|
124
|
+
function hashingPassThrough() {
|
|
125
|
+
const hash = (0, crypto_1.createHash)('sha256');
|
|
126
|
+
let bytes = 0;
|
|
127
|
+
const stream = new stream_1.Transform({
|
|
128
|
+
transform(chunk, _encoding, callback) {
|
|
129
|
+
hash.update(chunk);
|
|
130
|
+
bytes += chunk.length;
|
|
131
|
+
callback(null, chunk);
|
|
132
|
+
},
|
|
133
|
+
});
|
|
134
|
+
return Object.assign(stream, {
|
|
135
|
+
digest: () => hash.digest('hex'),
|
|
136
|
+
bytes: () => bytes,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Stream an upload into managed storage and return the value the column stores.
|
|
141
|
+
*
|
|
142
|
+
* Shape by column type hint:
|
|
143
|
+
* * `image` / `upload` (jsonb domains) → the projection document, including a
|
|
144
|
+
* compatibility `url` for public buckets.
|
|
145
|
+
* * `attachment` (text domain) → the object's public URL. A text column cannot
|
|
146
|
+
* hold a projection, so the files row is still authoritative but the column
|
|
147
|
+
* itself carries no id; the row keeps the object alive. A private bucket
|
|
148
|
+
* raises rather than storing an expiring presigned URL in a column.
|
|
149
|
+
*/
|
|
150
|
+
async function uploadResolver(upload, _args, context, info) {
|
|
151
|
+
const { tags, type, field } = info.uploadPlugin;
|
|
152
|
+
const typ = type || tags?.type;
|
|
153
|
+
const withPgClient = context?.withPgClient;
|
|
154
|
+
const pgSettings = context?.pgSettings ?? null;
|
|
155
|
+
if (!withPgClient) {
|
|
156
|
+
throw new Error('UPLOAD_NO_PG_CLIENT: a managed upload resolves its bucket in the database, so the ' +
|
|
157
|
+
'GraphQL context must carry withPgClient');
|
|
158
|
+
}
|
|
159
|
+
if (!field) {
|
|
160
|
+
throw new Error('UPLOAD_FIELD_UNKNOWN: the upload plugin did not report which column is being written, ' +
|
|
161
|
+
'so the storage module and bucket backing it cannot be resolved');
|
|
162
|
+
}
|
|
163
|
+
const databaseId = await (0, graphile_presigned_url_plugin_1.withRequestPgClient)(withPgClient, pgSettings, (pgClient) => resolveDatabaseId(pgClient));
|
|
164
|
+
if (!databaseId)
|
|
165
|
+
throw new Error('DATABASE_NOT_FOUND');
|
|
166
|
+
const target = await (0, graphile_presigned_url_plugin_1.resolveManagedUploadTarget)({
|
|
167
|
+
options: getManagedOptions(),
|
|
168
|
+
withPgClient,
|
|
169
|
+
pgSettings,
|
|
170
|
+
databaseId,
|
|
171
|
+
field: field,
|
|
172
|
+
defaultPublicAccess: LEGACY_DEFAULT_PUBLIC_ACCESS,
|
|
173
|
+
});
|
|
174
|
+
if (typ === 'attachment' && !target.bucket.is_public) {
|
|
175
|
+
throw new Error('ATTACHMENT_BUCKET_NOT_PUBLIC: an attachment column stores a plain URL, and the resolved ' +
|
|
176
|
+
`bucket "${target.bucket.key}" is private, whose only URLs expire. Use an upload column, ` +
|
|
177
|
+
'which stores the file id and resolves a fresh download URL on read.');
|
|
178
|
+
}
|
|
179
|
+
const s3 = getStreamer();
|
|
180
|
+
const { filename } = upload;
|
|
181
|
+
// Validate before persisting: content type comes from the leading bytes, not
|
|
182
|
+
// from the client's claim about them.
|
|
95
183
|
const detected = await s3.detectContentType({
|
|
96
184
|
readStream: upload.createReadStream(),
|
|
97
185
|
filename,
|
|
98
186
|
});
|
|
99
|
-
const
|
|
100
|
-
if (
|
|
187
|
+
const allowed = allowedMimeTypes(tags, typ);
|
|
188
|
+
if (allowed.length && !allowed.includes(detected.contentType)) {
|
|
101
189
|
detected.stream.destroy();
|
|
102
190
|
throw new Error('UPLOAD_MIMETYPE');
|
|
103
191
|
}
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
192
|
+
const staged = stagingKey();
|
|
193
|
+
const hashing = hashingPassThrough();
|
|
194
|
+
const uploadResult = await s3.uploadWithContentType({
|
|
195
|
+
readStream: detected.stream.pipe(hashing),
|
|
196
|
+
contentType: detected.contentType,
|
|
107
197
|
magic: detected.magic,
|
|
108
|
-
key,
|
|
109
|
-
bucket:
|
|
198
|
+
key: staged,
|
|
199
|
+
bucket: target.physicalName,
|
|
200
|
+
});
|
|
201
|
+
// Owns the staged key from here: it either promotes it into a files row or
|
|
202
|
+
// removes it, so a failed upload leaves nothing behind in S3.
|
|
203
|
+
const { projection } = await (0, graphile_presigned_url_plugin_1.finalizeStagedUpload)({
|
|
204
|
+
target,
|
|
205
|
+
withPgClient,
|
|
206
|
+
pgSettings,
|
|
207
|
+
staged: {
|
|
208
|
+
stagingKey: staged,
|
|
209
|
+
contentHash: hashing.digest(),
|
|
210
|
+
contentType: uploadResult.contentType,
|
|
211
|
+
size: hashing.bytes(),
|
|
212
|
+
filename,
|
|
213
|
+
},
|
|
110
214
|
});
|
|
111
|
-
const url = result.upload.Location;
|
|
112
|
-
const { contentType } = result;
|
|
113
215
|
switch (typ) {
|
|
114
216
|
case 'image':
|
|
115
217
|
case 'upload':
|
|
116
|
-
|
|
218
|
+
// `filename` and `mime` were in the pre-managed shape and stay in it;
|
|
219
|
+
// `url` is populated for public buckets and deprecated in favour of `id`.
|
|
220
|
+
return { ...projection, filename, mime: uploadResult.contentType };
|
|
117
221
|
case 'attachment':
|
|
118
222
|
default:
|
|
119
|
-
|
|
223
|
+
if (!projection.url) {
|
|
224
|
+
throw new Error(`ATTACHMENT_NO_PUBLIC_URL: bucket "${target.bucket.key}" has no public URL prefix ` +
|
|
225
|
+
'configured, so there is no durable URL to store in a text column');
|
|
226
|
+
}
|
|
227
|
+
return projection.url;
|
|
120
228
|
}
|
|
121
229
|
}
|
|
122
230
|
/**
|