graphile-presigned-url-plugin 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.d.ts +3 -3
- package/esm/index.js +2 -2
- package/esm/managed-upload.js +2 -4
- package/esm/physical-bucket.d.ts +18 -40
- package/esm/physical-bucket.js +26 -76
- package/esm/plugin.js +7 -11
- package/esm/storage-module-cache.d.ts +0 -8
- package/esm/storage-module-cache.js +0 -29
- package/esm/types.d.ts +3 -45
- package/index.d.ts +3 -3
- package/index.js +3 -5
- package/managed-upload.js +1 -3
- package/package.json +2 -2
- package/physical-bucket.d.ts +18 -40
- package/physical-bucket.js +29 -78
- package/plugin.js +6 -10
- package/storage-module-cache.d.ts +0 -8
- package/storage-module-cache.js +0 -31
- package/types.d.ts +3 -45
package/esm/index.d.ts
CHANGED
|
@@ -34,11 +34,11 @@ export { createDownloadUrlPlugin } from './download-url-field';
|
|
|
34
34
|
export type { FileRefFieldBinding } from './file-ref-registry';
|
|
35
35
|
export { clearFileRefFieldCache, FileRefFieldNotRegisteredError, getFileRefFieldBinding } from './file-ref-registry';
|
|
36
36
|
export { assertUploadAllowedByBucket, buildFileProjection, type FileProjection, finalizeStagedUpload, type ManagedUploadTarget, resolveManagedUploadTarget, } from './managed-upload';
|
|
37
|
-
export {
|
|
37
|
+
export { assertBucketReconciled, resolveS3, resolveS3ForDatabase, StorageBucketNotReconciledError, } from './physical-bucket';
|
|
38
38
|
export { createPresignedUrlPlugin, PresignedUrlPlugin } from './plugin';
|
|
39
39
|
export { PresignedUrlPreset } from './preset';
|
|
40
40
|
export { type WithPgClient, withRequestPgClient } from './request-pg-client';
|
|
41
41
|
export { describeS3Failure, s3FailureError } from './s3-failure';
|
|
42
42
|
export { copyS3Object, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject, readObjectPrefix } from './s3-signer';
|
|
43
|
-
export { clearBucketCache, clearStorageModuleCache, getBucketConfig,
|
|
44
|
-
export type { BucketConfig,
|
|
43
|
+
export { clearBucketCache, clearStorageModuleCache, getBucketConfig, loadAllStorageModules, resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache';
|
|
44
|
+
export type { BucketConfig, PresignedUrlPluginOptions, RequestUploadUrlInput, RequestUploadUrlPayload, S3Config, S3ConfigOrGetter, StorageModuleConfig, } from './types';
|
package/esm/index.js
CHANGED
|
@@ -32,10 +32,10 @@ export { resolveDefaultBucket } from './default-bucket';
|
|
|
32
32
|
export { createDownloadUrlPlugin } from './download-url-field';
|
|
33
33
|
export { clearFileRefFieldCache, FileRefFieldNotRegisteredError, getFileRefFieldBinding } from './file-ref-registry';
|
|
34
34
|
export { assertUploadAllowedByBucket, buildFileProjection, finalizeStagedUpload, resolveManagedUploadTarget, } from './managed-upload';
|
|
35
|
-
export {
|
|
35
|
+
export { assertBucketReconciled, resolveS3, resolveS3ForDatabase, StorageBucketNotReconciledError, } from './physical-bucket';
|
|
36
36
|
export { createPresignedUrlPlugin, PresignedUrlPlugin } from './plugin';
|
|
37
37
|
export { PresignedUrlPreset } from './preset';
|
|
38
38
|
export { withRequestPgClient } from './request-pg-client';
|
|
39
39
|
export { describeS3Failure, s3FailureError } from './s3-failure';
|
|
40
40
|
export { copyS3Object, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject, readObjectPrefix } from './s3-signer';
|
|
41
|
-
export { clearBucketCache, clearStorageModuleCache, getBucketConfig,
|
|
41
|
+
export { clearBucketCache, clearStorageModuleCache, getBucketConfig, loadAllStorageModules, resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache';
|
package/esm/managed-upload.js
CHANGED
|
@@ -21,7 +21,7 @@ import { Logger } from '@pgpmjs/logger';
|
|
|
21
21
|
import { resolveDefaultBucket } from './default-bucket';
|
|
22
22
|
import { isLiveFileRow, statusSelectFragment } from './file-lifecycle';
|
|
23
23
|
import { getFileRefFieldBinding } from './file-ref-registry';
|
|
24
|
-
import {
|
|
24
|
+
import { assertBucketReconciled, resolveS3ForDatabase } from './physical-bucket';
|
|
25
25
|
import { withRequestPgClient } from './request-pg-client';
|
|
26
26
|
import { copyS3Object, deleteS3Object } from './s3-signer';
|
|
27
27
|
import { recordManagedFile } from './storage-file-recorder';
|
|
@@ -134,9 +134,7 @@ export async function resolveManagedUploadTarget(args) {
|
|
|
134
134
|
'the multipart upload lane only writes content-addressed keys. Use the presigned upload ' +
|
|
135
135
|
'mutation with an explicit key.');
|
|
136
136
|
}
|
|
137
|
-
const physicalName = bucket
|
|
138
|
-
? await provisionAndRecordPhysicalBucket(options, withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins)
|
|
139
|
-
: bucket.physical_name;
|
|
137
|
+
const physicalName = assertBucketReconciled(bucket, databaseId);
|
|
140
138
|
return {
|
|
141
139
|
databaseId,
|
|
142
140
|
storageConfig,
|
package/esm/physical-bucket.d.ts
CHANGED
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Physical bucket coordinates:
|
|
3
|
-
* an S3 config against
|
|
2
|
+
* Physical bucket coordinates: reading the reconciler's recorded name and
|
|
3
|
+
* building an S3 config against that known name.
|
|
4
4
|
*
|
|
5
5
|
* A logical bucket belongs to a tenant; a physical bucket is an S3 name. The
|
|
6
|
-
* mapping is recorded on the bucket row the
|
|
7
|
-
*
|
|
8
|
-
* name is ever recomputed from a prefix convention, and there is no
|
|
9
|
-
* environment-level bucket standing in for a tenant's.
|
|
6
|
+
* mapping is recorded on the bucket row by the storage reconciler, and that
|
|
7
|
+
* value is the only coordinate anything reads — no name is ever recomputed.
|
|
10
8
|
*/
|
|
11
|
-
import { type WithPgClient } from './request-pg-client';
|
|
12
9
|
import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types';
|
|
10
|
+
export declare class StorageBucketNotReconciledError extends Error {
|
|
11
|
+
readonly code = "STORAGE_BUCKET_NOT_RECONCILED";
|
|
12
|
+
readonly retryable = true;
|
|
13
|
+
readonly extensions: {
|
|
14
|
+
code: string;
|
|
15
|
+
retryable: boolean;
|
|
16
|
+
};
|
|
17
|
+
constructor(bucket: BucketConfig, databaseId: string);
|
|
18
|
+
}
|
|
13
19
|
/**
|
|
14
20
|
* Resolve the plugin's S3 connection (credentials, endpoint, region), memoizing
|
|
15
21
|
* a lazy getter on first use.
|
|
@@ -19,42 +25,14 @@ import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleCo
|
|
|
19
25
|
* resolves its physical bucket from the tenant's bucket row.
|
|
20
26
|
*/
|
|
21
27
|
export declare function resolveS3(options: PresignedUrlPluginOptions): S3Config;
|
|
22
|
-
/**
|
|
23
|
-
* Mint the physical S3 bucket name for a logical bucket's first provision.
|
|
24
|
-
*
|
|
25
|
-
* This is a naming *policy*, consulted exactly once per bucket — before the
|
|
26
|
-
* physical bucket exists. Once provisioned, the recorded `physical_name` on the
|
|
27
|
-
* row is authoritative and this function must not be consulted again.
|
|
28
|
-
*
|
|
29
|
-
* There is no fallback to the configured `s3.bucket`: a deployment-wide bucket
|
|
30
|
-
* name is not a tenant's storage, and silently minting one is how objects ended
|
|
31
|
-
* up in a bucket no database owned. A deployment that wants per-tenant buckets
|
|
32
|
-
* must supply the policy.
|
|
33
|
-
*/
|
|
34
|
-
export declare function mintPhysicalBucketName(options: PresignedUrlPluginOptions, databaseId: string, bucketKey: string): string;
|
|
35
28
|
/**
|
|
36
29
|
* Build the S3 config for a *known* physical bucket. `physicalName` is
|
|
37
|
-
* required — callers must resolve the coordinate
|
|
38
|
-
*
|
|
30
|
+
* required — callers must resolve the coordinate from the stored row value
|
|
31
|
+
* before getting here. No name is ever recomputed.
|
|
39
32
|
*/
|
|
40
33
|
export declare function resolveS3ForDatabase(options: PresignedUrlPluginOptions, storageConfig: StorageModuleConfig, physicalName: string): S3Config;
|
|
41
34
|
/**
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* physical name.
|
|
45
|
-
*
|
|
46
|
-
* Only called when the row has no `physical_name` yet. Afterwards the stored
|
|
47
|
-
* value is the durable coordinate: route resolution and every later read use
|
|
48
|
-
* it verbatim; nothing is recomputed.
|
|
49
|
-
*
|
|
50
|
-
* The record write runs in the system lane (privileged role, so it bypasses the
|
|
51
|
-
* RLS that stops request roles from UPDATE-ing bucket rows) — it is server
|
|
52
|
-
* bookkeeping, not request data. It still carries the tenant `database_id`
|
|
53
|
-
* claim, because the buckets table's catalog-sync trigger calls
|
|
54
|
-
* `jwt_private.current_database_id()` and would otherwise raise
|
|
55
|
-
* DATABASE_CLAIM_REQUIRED; `withRequestPgClient` applies that claim inside the
|
|
56
|
-
* write's transaction without switching off the privileged role.
|
|
57
|
-
* `bucket` (the cached config) is mutated in place so subsequent reads observe
|
|
58
|
-
* the recorded name without a DB round-trip.
|
|
35
|
+
* Return the reconciler's recorded physical name, or fail with a typed,
|
|
36
|
+
* retryable error while reconciliation is still pending.
|
|
59
37
|
*/
|
|
60
|
-
export declare function
|
|
38
|
+
export declare function assertBucketReconciled(bucket: BucketConfig, databaseId: string): string;
|
package/esm/physical-bucket.js
CHANGED
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Physical bucket coordinates:
|
|
3
|
-
* an S3 config against
|
|
2
|
+
* Physical bucket coordinates: reading the reconciler's recorded name and
|
|
3
|
+
* building an S3 config against that known name.
|
|
4
4
|
*
|
|
5
5
|
* A logical bucket belongs to a tenant; a physical bucket is an S3 name. The
|
|
6
|
-
* mapping is recorded on the bucket row the
|
|
7
|
-
*
|
|
8
|
-
* name is ever recomputed from a prefix convention, and there is no
|
|
9
|
-
* environment-level bucket standing in for a tenant's.
|
|
6
|
+
* mapping is recorded on the bucket row by the storage reconciler, and that
|
|
7
|
+
* value is the only coordinate anything reads — no name is ever recomputed.
|
|
10
8
|
*/
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
9
|
+
export class StorageBucketNotReconciledError extends Error {
|
|
10
|
+
code = 'STORAGE_BUCKET_NOT_RECONCILED';
|
|
11
|
+
retryable = true;
|
|
12
|
+
extensions = {
|
|
13
|
+
code: 'STORAGE_BUCKET_NOT_RECONCILED',
|
|
14
|
+
retryable: true,
|
|
15
|
+
};
|
|
16
|
+
constructor(bucket, databaseId) {
|
|
17
|
+
super(`STORAGE_BUCKET_NOT_RECONCILED: bucket "${bucket.key}" (id=${bucket.id}) ` +
|
|
18
|
+
`for database ${databaseId} has not yet been reconciled; the reconciler has ` +
|
|
19
|
+
'not yet recorded a physical name');
|
|
20
|
+
this.name = 'StorageBucketNotReconciledError';
|
|
21
|
+
}
|
|
22
|
+
}
|
|
17
23
|
/**
|
|
18
24
|
* Resolve the plugin's S3 connection (credentials, endpoint, region), memoizing
|
|
19
25
|
* a lazy getter on first use.
|
|
@@ -30,31 +36,10 @@ export function resolveS3(options) {
|
|
|
30
36
|
}
|
|
31
37
|
return options.s3;
|
|
32
38
|
}
|
|
33
|
-
/**
|
|
34
|
-
* Mint the physical S3 bucket name for a logical bucket's first provision.
|
|
35
|
-
*
|
|
36
|
-
* This is a naming *policy*, consulted exactly once per bucket — before the
|
|
37
|
-
* physical bucket exists. Once provisioned, the recorded `physical_name` on the
|
|
38
|
-
* row is authoritative and this function must not be consulted again.
|
|
39
|
-
*
|
|
40
|
-
* There is no fallback to the configured `s3.bucket`: a deployment-wide bucket
|
|
41
|
-
* name is not a tenant's storage, and silently minting one is how objects ended
|
|
42
|
-
* up in a bucket no database owned. A deployment that wants per-tenant buckets
|
|
43
|
-
* must supply the policy.
|
|
44
|
-
*/
|
|
45
|
-
export function mintPhysicalBucketName(options, databaseId, bucketKey) {
|
|
46
|
-
if (!options.resolveBucketName) {
|
|
47
|
-
throw new Error('STORAGE_BUCKET_NAME_POLICY_MISSING: no resolveBucketName was configured, so there is ' +
|
|
48
|
-
`no name to provision for bucket "${bucketKey}" of database ${databaseId}. ` +
|
|
49
|
-
'Physical bucket naming is a deployment policy; the configured s3.bucket is a ' +
|
|
50
|
-
'connection default and is never a tenant bucket.');
|
|
51
|
-
}
|
|
52
|
-
return options.resolveBucketName(databaseId, bucketKey);
|
|
53
|
-
}
|
|
54
39
|
/**
|
|
55
40
|
* Build the S3 config for a *known* physical bucket. `physicalName` is
|
|
56
|
-
* required — callers must resolve the coordinate
|
|
57
|
-
*
|
|
41
|
+
* required — callers must resolve the coordinate from the stored row value
|
|
42
|
+
* before getting here. No name is ever recomputed.
|
|
58
43
|
*/
|
|
59
44
|
export function resolveS3ForDatabase(options, storageConfig, physicalName) {
|
|
60
45
|
const globalS3 = resolveS3(options);
|
|
@@ -71,46 +56,11 @@ export function resolveS3ForDatabase(options, storageConfig, physicalName) {
|
|
|
71
56
|
};
|
|
72
57
|
}
|
|
73
58
|
/**
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
* physical name.
|
|
77
|
-
*
|
|
78
|
-
* Only called when the row has no `physical_name` yet. Afterwards the stored
|
|
79
|
-
* value is the durable coordinate: route resolution and every later read use
|
|
80
|
-
* it verbatim; nothing is recomputed.
|
|
81
|
-
*
|
|
82
|
-
* The record write runs in the system lane (privileged role, so it bypasses the
|
|
83
|
-
* RLS that stops request roles from UPDATE-ing bucket rows) — it is server
|
|
84
|
-
* bookkeeping, not request data. It still carries the tenant `database_id`
|
|
85
|
-
* claim, because the buckets table's catalog-sync trigger calls
|
|
86
|
-
* `jwt_private.current_database_id()` and would otherwise raise
|
|
87
|
-
* DATABASE_CLAIM_REQUIRED; `withRequestPgClient` applies that claim inside the
|
|
88
|
-
* write's transaction without switching off the privileged role.
|
|
89
|
-
* `bucket` (the cached config) is mutated in place so subsequent reads observe
|
|
90
|
-
* the recorded name without a DB round-trip.
|
|
59
|
+
* Return the reconciler's recorded physical name, or fail with a typed,
|
|
60
|
+
* retryable error while reconciliation is still pending.
|
|
91
61
|
*/
|
|
92
|
-
export
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
try {
|
|
97
|
-
await options.ensureBucketProvisioned(s3BucketName, bucket.type, databaseId, allowedOrigins);
|
|
98
|
-
}
|
|
99
|
-
catch (err) {
|
|
100
|
-
// The first upload to a bucket is where an unreachable object store is
|
|
101
|
-
// discovered, and the transport's own message is routinely empty: name the
|
|
102
|
-
// endpoint it could not reach so the response says what is misconfigured.
|
|
103
|
-
throw s3FailureError('BUCKET_PROVISION_FAILED', { endpoint: resolveS3(options).endpoint, bucket: s3BucketName, databaseId }, err);
|
|
104
|
-
}
|
|
105
|
-
markS3BucketProvisioned(s3BucketName);
|
|
106
|
-
log.info(`Lazy-provisioned S3 bucket "${s3BucketName}" successfully`);
|
|
107
|
-
}
|
|
108
|
-
// Record the physical coordinate on the source row. The `physical_name IS NULL`
|
|
109
|
-
// guard keeps this idempotent and race-safe across concurrent first uploads.
|
|
110
|
-
// The catalog-sync trigger on this UPDATE needs `jwt.claims.database_id`, so the
|
|
111
|
-
// write runs under the resolved database claim (privileged role preserved).
|
|
112
|
-
await withRequestPgClient(withPgClient, { 'jwt.claims.database_id': databaseId }, (client) => recordPhysicalName((query) => client.query(query), storageConfig.bucketsQualifiedName, bucket.id, s3BucketName));
|
|
113
|
-
bucket.physical_name = s3BucketName;
|
|
114
|
-
log.info(`Recorded physical_name="${s3BucketName}" on bucket ${bucket.id}`);
|
|
115
|
-
return s3BucketName;
|
|
62
|
+
export function assertBucketReconciled(bucket, databaseId) {
|
|
63
|
+
if (bucket.physical_name !== null)
|
|
64
|
+
return bucket.physical_name;
|
|
65
|
+
throw new StorageBucketNotReconciledError(bucket, databaseId);
|
|
116
66
|
}
|
package/esm/plugin.js
CHANGED
|
@@ -27,7 +27,7 @@ import { validateCustomKey } from './custom-key';
|
|
|
27
27
|
import { resolveDefaultBucket } from './default-bucket';
|
|
28
28
|
import { isLiveFileRow, statusSelectFragment } from './file-lifecycle';
|
|
29
29
|
import { buildFileProjection } from './managed-upload';
|
|
30
|
-
import {
|
|
30
|
+
import { assertBucketReconciled, resolveS3ForDatabase } from './physical-bucket';
|
|
31
31
|
import { withRequestPgClient } from './request-pg-client';
|
|
32
32
|
import { deleteS3Object, generatePresignedPutUrl } from './s3-signer';
|
|
33
33
|
import { recordManagedFile } from './storage-file-recorder';
|
|
@@ -199,11 +199,9 @@ export function createPresignedUrlPlugin(options) {
|
|
|
199
199
|
const bucket = await withRequestPgClient(vals.withPgClient, vals.pgSettings, (pgClient) => resolveUploadBucket(pgClient, storageConfig, databaseId, vals.bucketKey ?? null, vals.ownerId ?? null, vals.isPublic === true));
|
|
200
200
|
if (!bucket)
|
|
201
201
|
throw new Error('BUCKET_NOT_FOUND');
|
|
202
|
-
//
|
|
203
|
-
//
|
|
204
|
-
const physicalName = bucket
|
|
205
|
-
? await provisionAndRecordPhysicalBucket(options, vals.withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins)
|
|
206
|
-
: bucket.physical_name;
|
|
202
|
+
// The reconciler records the coordinate; consumers never
|
|
203
|
+
// recompute it from the logical bucket row.
|
|
204
|
+
const physicalName = assertBucketReconciled(bucket, databaseId);
|
|
207
205
|
const s3ForDb = resolveS3ForDatabase(options, storageConfig, physicalName);
|
|
208
206
|
// File row INSERT under the request role (RLS enforced).
|
|
209
207
|
return withRequestPgClient(vals.withPgClient, vals.pgSettings, (txClient) => processSingleFile(options, txClient, storageConfig, databaseId, bucket, s3ForDb, {
|
|
@@ -306,11 +304,9 @@ export function createPresignedUrlPlugin(options) {
|
|
|
306
304
|
if (totalSize > storageConfig.maxBulkTotalSize) {
|
|
307
305
|
throw new Error(`BULK_UPLOAD_SIZE_EXCEEDED: ${totalSize} bytes exceeds maximum of ${storageConfig.maxBulkTotalSize} bytes per batch`);
|
|
308
306
|
}
|
|
309
|
-
//
|
|
310
|
-
//
|
|
311
|
-
const physicalName = bucket
|
|
312
|
-
? await provisionAndRecordPhysicalBucket(options, vals.withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins)
|
|
313
|
-
: bucket.physical_name;
|
|
307
|
+
// The reconciler records the coordinate; consumers never
|
|
308
|
+
// recompute it from the logical bucket row.
|
|
309
|
+
const physicalName = assertBucketReconciled(bucket, databaseId);
|
|
314
310
|
const s3ForDb = resolveS3ForDatabase(options, storageConfig, physicalName);
|
|
315
311
|
// File row INSERTs under the request role (RLS enforced).
|
|
316
312
|
return withRequestPgClient(vals.withPgClient, vals.pgSettings, async (txClient) => {
|
|
@@ -91,14 +91,6 @@ export declare function getBucketConfig(pgClient: {
|
|
|
91
91
|
rows: unknown[];
|
|
92
92
|
}>;
|
|
93
93
|
}, storageConfig: StorageModuleConfig, databaseId: string, bucketKey: string, ownerId?: string): Promise<BucketConfig | null>;
|
|
94
|
-
/**
|
|
95
|
-
* Check whether an S3 bucket has already been provisioned (cached).
|
|
96
|
-
*/
|
|
97
|
-
export declare function isS3BucketProvisioned(s3BucketName: string): boolean;
|
|
98
|
-
/**
|
|
99
|
-
* Mark an S3 bucket as provisioned in the in-memory cache.
|
|
100
|
-
*/
|
|
101
|
-
export declare function markS3BucketProvisioned(s3BucketName: string): void;
|
|
102
94
|
/**
|
|
103
95
|
* Clear the storage module cache AND bucket cache.
|
|
104
96
|
* Useful for testing or schema changes.
|
|
@@ -260,34 +260,6 @@ export async function getBucketConfig(pgClient, storageConfig, databaseId, bucke
|
|
|
260
260
|
log.debug(`Cached bucket config for ${databaseId}:${bucketKey} (id=${config.id}, scope=${storageConfig.scope})`);
|
|
261
261
|
return config;
|
|
262
262
|
}
|
|
263
|
-
// --- S3 bucket existence cache ---
|
|
264
|
-
/**
|
|
265
|
-
* In-memory set of S3 bucket names that are known to exist.
|
|
266
|
-
*
|
|
267
|
-
* Used by the lazy provisioning logic in the presigned URL plugin:
|
|
268
|
-
* before generating a presigned PUT URL, the plugin checks this set.
|
|
269
|
-
* If the bucket name is absent, it calls `ensureBucketProvisioned`
|
|
270
|
-
* to create the S3 bucket, then adds the name here. Subsequent
|
|
271
|
-
* requests for the same bucket skip the provisioning entirely.
|
|
272
|
-
*
|
|
273
|
-
* No TTL needed — S3 buckets are never deleted during normal operation.
|
|
274
|
-
* The set resets on server restart, which is fine because the
|
|
275
|
-
* provisioner's createBucket is idempotent (handles "already exists").
|
|
276
|
-
*/
|
|
277
|
-
const provisionedBuckets = new Set();
|
|
278
|
-
/**
|
|
279
|
-
* Check whether an S3 bucket has already been provisioned (cached).
|
|
280
|
-
*/
|
|
281
|
-
export function isS3BucketProvisioned(s3BucketName) {
|
|
282
|
-
return provisionedBuckets.has(s3BucketName);
|
|
283
|
-
}
|
|
284
|
-
/**
|
|
285
|
-
* Mark an S3 bucket as provisioned in the in-memory cache.
|
|
286
|
-
*/
|
|
287
|
-
export function markS3BucketProvisioned(s3BucketName) {
|
|
288
|
-
provisionedBuckets.add(s3BucketName);
|
|
289
|
-
log.debug(`Marked S3 bucket "${s3BucketName}" as provisioned`);
|
|
290
|
-
}
|
|
291
263
|
/**
|
|
292
264
|
* Clear the storage module cache AND bucket cache.
|
|
293
265
|
* Useful for testing or schema changes.
|
|
@@ -295,7 +267,6 @@ export function markS3BucketProvisioned(s3BucketName) {
|
|
|
295
267
|
export function clearStorageModuleCache() {
|
|
296
268
|
storageModuleCache.clear();
|
|
297
269
|
bucketCache.clear();
|
|
298
|
-
provisionedBuckets.clear();
|
|
299
270
|
}
|
|
300
271
|
/**
|
|
301
272
|
* Clear cached bucket entries for a specific database.
|
package/esm/types.d.ts
CHANGED
|
@@ -12,10 +12,9 @@ export interface BucketConfig {
|
|
|
12
12
|
max_file_size: number | null;
|
|
13
13
|
allow_custom_keys: boolean;
|
|
14
14
|
/**
|
|
15
|
-
* The physical S3/MinIO bucket name recorded
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* reconstruct the name from a prefix convention.
|
|
15
|
+
* The physical S3/MinIO bucket name recorded by reconciliation. NULL until
|
|
16
|
+
* reconciliation completes. Once set, it is the source of truth for the
|
|
17
|
+
* physical bucket — reads never reconstruct the name.
|
|
19
18
|
*/
|
|
20
19
|
physical_name: string | null;
|
|
21
20
|
}
|
|
@@ -178,51 +177,10 @@ export interface S3Config {
|
|
|
178
177
|
* env-var reads and S3Client creation at module import time.
|
|
179
178
|
*/
|
|
180
179
|
export type S3ConfigOrGetter = S3Config | (() => S3Config);
|
|
181
|
-
/**
|
|
182
|
-
* Function to derive the actual S3 bucket name for a given database and bucket key.
|
|
183
|
-
*
|
|
184
|
-
* When provided, the presigned URL plugin calls this on every request
|
|
185
|
-
* to determine which S3 bucket to use — enabling per-(database, bucketKey)
|
|
186
|
-
* isolation. If not provided, falls back to `s3Config.bucket` (global).
|
|
187
|
-
*
|
|
188
|
-
* @param databaseId - The metaschema database UUID
|
|
189
|
-
* @param bucketKey - The logical bucket key (e.g., "public", "private")
|
|
190
|
-
* @returns The S3 bucket name for this database + bucket key
|
|
191
|
-
*/
|
|
192
|
-
export type BucketNameResolver = (databaseId: string, bucketKey: string) => string;
|
|
193
|
-
/**
|
|
194
|
-
* Callback to lazily provision an S3 bucket on first use.
|
|
195
|
-
*
|
|
196
|
-
* Called by the presigned URL plugin before generating a presigned PUT URL
|
|
197
|
-
* when the bucket has not been seen before (tracked in an in-memory cache).
|
|
198
|
-
* The implementation should create and fully configure the S3 bucket
|
|
199
|
-
* (privacy policies, CORS, lifecycle rules, etc.) — or no-op if the
|
|
200
|
-
* bucket already exists.
|
|
201
|
-
*
|
|
202
|
-
* @param bucketName - The S3 bucket name to provision
|
|
203
|
-
* @param accessType - The logical bucket type ('public', 'private', 'temp')
|
|
204
|
-
* @param databaseId - The metaschema database UUID
|
|
205
|
-
* @param allowedOrigins - Per-database CORS origins (from storage_module), or null to use global fallback
|
|
206
|
-
*/
|
|
207
|
-
export type EnsureBucketProvisioned = (bucketName: string, accessType: 'public' | 'private' | 'temp', databaseId: string, allowedOrigins: string[] | null) => Promise<void>;
|
|
208
180
|
/**
|
|
209
181
|
* Plugin options for the presigned URL plugin.
|
|
210
182
|
*/
|
|
211
183
|
export interface PresignedUrlPluginOptions {
|
|
212
184
|
/** S3 configuration (concrete or lazy getter) */
|
|
213
185
|
s3: S3ConfigOrGetter;
|
|
214
|
-
/**
|
|
215
|
-
* Optional function to resolve S3 bucket name per-database.
|
|
216
|
-
* When set, each database gets its own S3 bucket instead of sharing
|
|
217
|
-
* the global `s3Config.bucket`. The S3 credentials (client) remain shared.
|
|
218
|
-
*/
|
|
219
|
-
resolveBucketName?: BucketNameResolver;
|
|
220
|
-
/**
|
|
221
|
-
* Optional callback to lazily provision an S3 bucket on first upload.
|
|
222
|
-
* When set, the plugin calls this before generating a presigned PUT URL
|
|
223
|
-
* for any S3 bucket it hasn't seen yet (tracked in an in-memory cache).
|
|
224
|
-
* This enables graceful bucket creation without requiring buckets to
|
|
225
|
-
* exist at database provisioning time.
|
|
226
|
-
*/
|
|
227
|
-
ensureBucketProvisioned?: EnsureBucketProvisioned;
|
|
228
186
|
}
|
package/index.d.ts
CHANGED
|
@@ -34,11 +34,11 @@ export { createDownloadUrlPlugin } from './download-url-field';
|
|
|
34
34
|
export type { FileRefFieldBinding } from './file-ref-registry';
|
|
35
35
|
export { clearFileRefFieldCache, FileRefFieldNotRegisteredError, getFileRefFieldBinding } from './file-ref-registry';
|
|
36
36
|
export { assertUploadAllowedByBucket, buildFileProjection, type FileProjection, finalizeStagedUpload, type ManagedUploadTarget, resolveManagedUploadTarget, } from './managed-upload';
|
|
37
|
-
export {
|
|
37
|
+
export { assertBucketReconciled, resolveS3, resolveS3ForDatabase, StorageBucketNotReconciledError, } from './physical-bucket';
|
|
38
38
|
export { createPresignedUrlPlugin, PresignedUrlPlugin } from './plugin';
|
|
39
39
|
export { PresignedUrlPreset } from './preset';
|
|
40
40
|
export { type WithPgClient, withRequestPgClient } from './request-pg-client';
|
|
41
41
|
export { describeS3Failure, s3FailureError } from './s3-failure';
|
|
42
42
|
export { copyS3Object, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject, readObjectPrefix } from './s3-signer';
|
|
43
|
-
export { clearBucketCache, clearStorageModuleCache, getBucketConfig,
|
|
44
|
-
export type { BucketConfig,
|
|
43
|
+
export { clearBucketCache, clearStorageModuleCache, getBucketConfig, loadAllStorageModules, resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache';
|
|
44
|
+
export type { BucketConfig, PresignedUrlPluginOptions, RequestUploadUrlInput, RequestUploadUrlPayload, S3Config, S3ConfigOrGetter, StorageModuleConfig, } from './types';
|
package/index.js
CHANGED
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
* ```
|
|
29
29
|
*/
|
|
30
30
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
31
|
-
exports.resolveStorageModuleByFileId = exports.resolveStorageConfigFromCodec = exports.
|
|
31
|
+
exports.resolveStorageModuleByFileId = exports.resolveStorageConfigFromCodec = exports.loadAllStorageModules = exports.getBucketConfig = exports.clearStorageModuleCache = exports.clearBucketCache = exports.readObjectPrefix = exports.headObject = exports.generatePresignedPutUrl = exports.generatePresignedGetUrl = exports.deleteS3Object = exports.copyS3Object = exports.s3FailureError = exports.describeS3Failure = exports.withRequestPgClient = exports.PresignedUrlPreset = exports.PresignedUrlPlugin = exports.createPresignedUrlPlugin = exports.StorageBucketNotReconciledError = exports.resolveS3ForDatabase = exports.resolveS3 = exports.assertBucketReconciled = exports.resolveManagedUploadTarget = exports.finalizeStagedUpload = exports.buildFileProjection = exports.assertUploadAllowedByBucket = exports.getFileRefFieldBinding = exports.FileRefFieldNotRegisteredError = exports.clearFileRefFieldCache = exports.createDownloadUrlPlugin = exports.resolveDefaultBucket = exports.validateCustomKey = exports.confirmUploadedBytes = exports.CONFIRM_PREFIX_BYTES = void 0;
|
|
32
32
|
var confirm_upload_1 = require("./confirm-upload");
|
|
33
33
|
Object.defineProperty(exports, "CONFIRM_PREFIX_BYTES", { enumerable: true, get: function () { return confirm_upload_1.CONFIRM_PREFIX_BYTES; } });
|
|
34
34
|
Object.defineProperty(exports, "confirmUploadedBytes", { enumerable: true, get: function () { return confirm_upload_1.confirmUploadedBytes; } });
|
|
@@ -48,10 +48,10 @@ Object.defineProperty(exports, "buildFileProjection", { enumerable: true, get: f
|
|
|
48
48
|
Object.defineProperty(exports, "finalizeStagedUpload", { enumerable: true, get: function () { return managed_upload_1.finalizeStagedUpload; } });
|
|
49
49
|
Object.defineProperty(exports, "resolveManagedUploadTarget", { enumerable: true, get: function () { return managed_upload_1.resolveManagedUploadTarget; } });
|
|
50
50
|
var physical_bucket_1 = require("./physical-bucket");
|
|
51
|
-
Object.defineProperty(exports, "
|
|
52
|
-
Object.defineProperty(exports, "provisionAndRecordPhysicalBucket", { enumerable: true, get: function () { return physical_bucket_1.provisionAndRecordPhysicalBucket; } });
|
|
51
|
+
Object.defineProperty(exports, "assertBucketReconciled", { enumerable: true, get: function () { return physical_bucket_1.assertBucketReconciled; } });
|
|
53
52
|
Object.defineProperty(exports, "resolveS3", { enumerable: true, get: function () { return physical_bucket_1.resolveS3; } });
|
|
54
53
|
Object.defineProperty(exports, "resolveS3ForDatabase", { enumerable: true, get: function () { return physical_bucket_1.resolveS3ForDatabase; } });
|
|
54
|
+
Object.defineProperty(exports, "StorageBucketNotReconciledError", { enumerable: true, get: function () { return physical_bucket_1.StorageBucketNotReconciledError; } });
|
|
55
55
|
var plugin_1 = require("./plugin");
|
|
56
56
|
Object.defineProperty(exports, "createPresignedUrlPlugin", { enumerable: true, get: function () { return plugin_1.createPresignedUrlPlugin; } });
|
|
57
57
|
Object.defineProperty(exports, "PresignedUrlPlugin", { enumerable: true, get: function () { return plugin_1.PresignedUrlPlugin; } });
|
|
@@ -73,8 +73,6 @@ var storage_module_cache_1 = require("./storage-module-cache");
|
|
|
73
73
|
Object.defineProperty(exports, "clearBucketCache", { enumerable: true, get: function () { return storage_module_cache_1.clearBucketCache; } });
|
|
74
74
|
Object.defineProperty(exports, "clearStorageModuleCache", { enumerable: true, get: function () { return storage_module_cache_1.clearStorageModuleCache; } });
|
|
75
75
|
Object.defineProperty(exports, "getBucketConfig", { enumerable: true, get: function () { return storage_module_cache_1.getBucketConfig; } });
|
|
76
|
-
Object.defineProperty(exports, "isS3BucketProvisioned", { enumerable: true, get: function () { return storage_module_cache_1.isS3BucketProvisioned; } });
|
|
77
76
|
Object.defineProperty(exports, "loadAllStorageModules", { enumerable: true, get: function () { return storage_module_cache_1.loadAllStorageModules; } });
|
|
78
|
-
Object.defineProperty(exports, "markS3BucketProvisioned", { enumerable: true, get: function () { return storage_module_cache_1.markS3BucketProvisioned; } });
|
|
79
77
|
Object.defineProperty(exports, "resolveStorageConfigFromCodec", { enumerable: true, get: function () { return storage_module_cache_1.resolveStorageConfigFromCodec; } });
|
|
80
78
|
Object.defineProperty(exports, "resolveStorageModuleByFileId", { enumerable: true, get: function () { return storage_module_cache_1.resolveStorageModuleByFileId; } });
|
package/managed-upload.js
CHANGED
|
@@ -140,9 +140,7 @@ async function resolveManagedUploadTarget(args) {
|
|
|
140
140
|
'the multipart upload lane only writes content-addressed keys. Use the presigned upload ' +
|
|
141
141
|
'mutation with an explicit key.');
|
|
142
142
|
}
|
|
143
|
-
const physicalName = bucket
|
|
144
|
-
? await (0, physical_bucket_1.provisionAndRecordPhysicalBucket)(options, withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins)
|
|
145
|
-
: bucket.physical_name;
|
|
143
|
+
const physicalName = (0, physical_bucket_1.assertBucketReconciled)(bucket, databaseId);
|
|
146
144
|
return {
|
|
147
145
|
databaseId,
|
|
148
146
|
storageConfig,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "graphile-presigned-url-plugin",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.18.0",
|
|
4
4
|
"description": "Presigned URL upload plugin for PostGraphile v5 — requestUploadUrl mutation and downloadUrl computed field",
|
|
5
5
|
"author": "Constructive <developers@constructive.io>",
|
|
6
6
|
"homepage": "https://github.com/constructive-io/constructive",
|
|
@@ -62,5 +62,5 @@
|
|
|
62
62
|
"@types/node": "^22.19.11",
|
|
63
63
|
"makage": "^0.3.0"
|
|
64
64
|
},
|
|
65
|
-
"gitHead": "
|
|
65
|
+
"gitHead": "77d58bb93796be91d4e00156bb6847392584d3bc"
|
|
66
66
|
}
|
package/physical-bucket.d.ts
CHANGED
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Physical bucket coordinates:
|
|
3
|
-
* an S3 config against
|
|
2
|
+
* Physical bucket coordinates: reading the reconciler's recorded name and
|
|
3
|
+
* building an S3 config against that known name.
|
|
4
4
|
*
|
|
5
5
|
* A logical bucket belongs to a tenant; a physical bucket is an S3 name. The
|
|
6
|
-
* mapping is recorded on the bucket row the
|
|
7
|
-
*
|
|
8
|
-
* name is ever recomputed from a prefix convention, and there is no
|
|
9
|
-
* environment-level bucket standing in for a tenant's.
|
|
6
|
+
* mapping is recorded on the bucket row by the storage reconciler, and that
|
|
7
|
+
* value is the only coordinate anything reads — no name is ever recomputed.
|
|
10
8
|
*/
|
|
11
|
-
import { type WithPgClient } from './request-pg-client';
|
|
12
9
|
import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types';
|
|
10
|
+
export declare class StorageBucketNotReconciledError extends Error {
|
|
11
|
+
readonly code = "STORAGE_BUCKET_NOT_RECONCILED";
|
|
12
|
+
readonly retryable = true;
|
|
13
|
+
readonly extensions: {
|
|
14
|
+
code: string;
|
|
15
|
+
retryable: boolean;
|
|
16
|
+
};
|
|
17
|
+
constructor(bucket: BucketConfig, databaseId: string);
|
|
18
|
+
}
|
|
13
19
|
/**
|
|
14
20
|
* Resolve the plugin's S3 connection (credentials, endpoint, region), memoizing
|
|
15
21
|
* a lazy getter on first use.
|
|
@@ -19,42 +25,14 @@ import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleCo
|
|
|
19
25
|
* resolves its physical bucket from the tenant's bucket row.
|
|
20
26
|
*/
|
|
21
27
|
export declare function resolveS3(options: PresignedUrlPluginOptions): S3Config;
|
|
22
|
-
/**
|
|
23
|
-
* Mint the physical S3 bucket name for a logical bucket's first provision.
|
|
24
|
-
*
|
|
25
|
-
* This is a naming *policy*, consulted exactly once per bucket — before the
|
|
26
|
-
* physical bucket exists. Once provisioned, the recorded `physical_name` on the
|
|
27
|
-
* row is authoritative and this function must not be consulted again.
|
|
28
|
-
*
|
|
29
|
-
* There is no fallback to the configured `s3.bucket`: a deployment-wide bucket
|
|
30
|
-
* name is not a tenant's storage, and silently minting one is how objects ended
|
|
31
|
-
* up in a bucket no database owned. A deployment that wants per-tenant buckets
|
|
32
|
-
* must supply the policy.
|
|
33
|
-
*/
|
|
34
|
-
export declare function mintPhysicalBucketName(options: PresignedUrlPluginOptions, databaseId: string, bucketKey: string): string;
|
|
35
28
|
/**
|
|
36
29
|
* Build the S3 config for a *known* physical bucket. `physicalName` is
|
|
37
|
-
* required — callers must resolve the coordinate
|
|
38
|
-
*
|
|
30
|
+
* required — callers must resolve the coordinate from the stored row value
|
|
31
|
+
* before getting here. No name is ever recomputed.
|
|
39
32
|
*/
|
|
40
33
|
export declare function resolveS3ForDatabase(options: PresignedUrlPluginOptions, storageConfig: StorageModuleConfig, physicalName: string): S3Config;
|
|
41
34
|
/**
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* physical name.
|
|
45
|
-
*
|
|
46
|
-
* Only called when the row has no `physical_name` yet. Afterwards the stored
|
|
47
|
-
* value is the durable coordinate: route resolution and every later read use
|
|
48
|
-
* it verbatim; nothing is recomputed.
|
|
49
|
-
*
|
|
50
|
-
* The record write runs in the system lane (privileged role, so it bypasses the
|
|
51
|
-
* RLS that stops request roles from UPDATE-ing bucket rows) — it is server
|
|
52
|
-
* bookkeeping, not request data. It still carries the tenant `database_id`
|
|
53
|
-
* claim, because the buckets table's catalog-sync trigger calls
|
|
54
|
-
* `jwt_private.current_database_id()` and would otherwise raise
|
|
55
|
-
* DATABASE_CLAIM_REQUIRED; `withRequestPgClient` applies that claim inside the
|
|
56
|
-
* write's transaction without switching off the privileged role.
|
|
57
|
-
* `bucket` (the cached config) is mutated in place so subsequent reads observe
|
|
58
|
-
* the recorded name without a DB round-trip.
|
|
35
|
+
* Return the reconciler's recorded physical name, or fail with a typed,
|
|
36
|
+
* retryable error while reconciliation is still pending.
|
|
59
37
|
*/
|
|
60
|
-
export declare function
|
|
38
|
+
export declare function assertBucketReconciled(bucket: BucketConfig, databaseId: string): string;
|
package/physical-bucket.js
CHANGED
|
@@ -1,25 +1,32 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* Physical bucket coordinates:
|
|
4
|
-
* an S3 config against
|
|
3
|
+
* Physical bucket coordinates: reading the reconciler's recorded name and
|
|
4
|
+
* building an S3 config against that known name.
|
|
5
5
|
*
|
|
6
6
|
* A logical bucket belongs to a tenant; a physical bucket is an S3 name. The
|
|
7
|
-
* mapping is recorded on the bucket row the
|
|
8
|
-
*
|
|
9
|
-
* name is ever recomputed from a prefix convention, and there is no
|
|
10
|
-
* environment-level bucket standing in for a tenant's.
|
|
7
|
+
* mapping is recorded on the bucket row by the storage reconciler, and that
|
|
8
|
+
* value is the only coordinate anything reads — no name is ever recomputed.
|
|
11
9
|
*/
|
|
12
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.StorageBucketNotReconciledError = void 0;
|
|
13
12
|
exports.resolveS3 = resolveS3;
|
|
14
|
-
exports.mintPhysicalBucketName = mintPhysicalBucketName;
|
|
15
13
|
exports.resolveS3ForDatabase = resolveS3ForDatabase;
|
|
16
|
-
exports.
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
14
|
+
exports.assertBucketReconciled = assertBucketReconciled;
|
|
15
|
+
class StorageBucketNotReconciledError extends Error {
|
|
16
|
+
code = 'STORAGE_BUCKET_NOT_RECONCILED';
|
|
17
|
+
retryable = true;
|
|
18
|
+
extensions = {
|
|
19
|
+
code: 'STORAGE_BUCKET_NOT_RECONCILED',
|
|
20
|
+
retryable: true,
|
|
21
|
+
};
|
|
22
|
+
constructor(bucket, databaseId) {
|
|
23
|
+
super(`STORAGE_BUCKET_NOT_RECONCILED: bucket "${bucket.key}" (id=${bucket.id}) ` +
|
|
24
|
+
`for database ${databaseId} has not yet been reconciled; the reconciler has ` +
|
|
25
|
+
'not yet recorded a physical name');
|
|
26
|
+
this.name = 'StorageBucketNotReconciledError';
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
exports.StorageBucketNotReconciledError = StorageBucketNotReconciledError;
|
|
23
30
|
/**
|
|
24
31
|
* Resolve the plugin's S3 connection (credentials, endpoint, region), memoizing
|
|
25
32
|
* a lazy getter on first use.
|
|
@@ -36,31 +43,10 @@ function resolveS3(options) {
|
|
|
36
43
|
}
|
|
37
44
|
return options.s3;
|
|
38
45
|
}
|
|
39
|
-
/**
|
|
40
|
-
* Mint the physical S3 bucket name for a logical bucket's first provision.
|
|
41
|
-
*
|
|
42
|
-
* This is a naming *policy*, consulted exactly once per bucket — before the
|
|
43
|
-
* physical bucket exists. Once provisioned, the recorded `physical_name` on the
|
|
44
|
-
* row is authoritative and this function must not be consulted again.
|
|
45
|
-
*
|
|
46
|
-
* There is no fallback to the configured `s3.bucket`: a deployment-wide bucket
|
|
47
|
-
* name is not a tenant's storage, and silently minting one is how objects ended
|
|
48
|
-
* up in a bucket no database owned. A deployment that wants per-tenant buckets
|
|
49
|
-
* must supply the policy.
|
|
50
|
-
*/
|
|
51
|
-
function mintPhysicalBucketName(options, databaseId, bucketKey) {
|
|
52
|
-
if (!options.resolveBucketName) {
|
|
53
|
-
throw new Error('STORAGE_BUCKET_NAME_POLICY_MISSING: no resolveBucketName was configured, so there is ' +
|
|
54
|
-
`no name to provision for bucket "${bucketKey}" of database ${databaseId}. ` +
|
|
55
|
-
'Physical bucket naming is a deployment policy; the configured s3.bucket is a ' +
|
|
56
|
-
'connection default and is never a tenant bucket.');
|
|
57
|
-
}
|
|
58
|
-
return options.resolveBucketName(databaseId, bucketKey);
|
|
59
|
-
}
|
|
60
46
|
/**
|
|
61
47
|
* Build the S3 config for a *known* physical bucket. `physicalName` is
|
|
62
|
-
* required — callers must resolve the coordinate
|
|
63
|
-
*
|
|
48
|
+
* required — callers must resolve the coordinate from the stored row value
|
|
49
|
+
* before getting here. No name is ever recomputed.
|
|
64
50
|
*/
|
|
65
51
|
function resolveS3ForDatabase(options, storageConfig, physicalName) {
|
|
66
52
|
const globalS3 = resolveS3(options);
|
|
@@ -77,46 +63,11 @@ function resolveS3ForDatabase(options, storageConfig, physicalName) {
|
|
|
77
63
|
};
|
|
78
64
|
}
|
|
79
65
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
* physical name.
|
|
83
|
-
*
|
|
84
|
-
* Only called when the row has no `physical_name` yet. Afterwards the stored
|
|
85
|
-
* value is the durable coordinate: route resolution and every later read use
|
|
86
|
-
* it verbatim; nothing is recomputed.
|
|
87
|
-
*
|
|
88
|
-
* The record write runs in the system lane (privileged role, so it bypasses the
|
|
89
|
-
* RLS that stops request roles from UPDATE-ing bucket rows) — it is server
|
|
90
|
-
* bookkeeping, not request data. It still carries the tenant `database_id`
|
|
91
|
-
* claim, because the buckets table's catalog-sync trigger calls
|
|
92
|
-
* `jwt_private.current_database_id()` and would otherwise raise
|
|
93
|
-
* DATABASE_CLAIM_REQUIRED; `withRequestPgClient` applies that claim inside the
|
|
94
|
-
* write's transaction without switching off the privileged role.
|
|
95
|
-
* `bucket` (the cached config) is mutated in place so subsequent reads observe
|
|
96
|
-
* the recorded name without a DB round-trip.
|
|
66
|
+
* Return the reconciler's recorded physical name, or fail with a typed,
|
|
67
|
+
* retryable error while reconciliation is still pending.
|
|
97
68
|
*/
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
try {
|
|
103
|
-
await options.ensureBucketProvisioned(s3BucketName, bucket.type, databaseId, allowedOrigins);
|
|
104
|
-
}
|
|
105
|
-
catch (err) {
|
|
106
|
-
// The first upload to a bucket is where an unreachable object store is
|
|
107
|
-
// discovered, and the transport's own message is routinely empty: name the
|
|
108
|
-
// endpoint it could not reach so the response says what is misconfigured.
|
|
109
|
-
throw (0, s3_failure_1.s3FailureError)('BUCKET_PROVISION_FAILED', { endpoint: resolveS3(options).endpoint, bucket: s3BucketName, databaseId }, err);
|
|
110
|
-
}
|
|
111
|
-
(0, storage_module_cache_1.markS3BucketProvisioned)(s3BucketName);
|
|
112
|
-
log.info(`Lazy-provisioned S3 bucket "${s3BucketName}" successfully`);
|
|
113
|
-
}
|
|
114
|
-
// Record the physical coordinate on the source row. The `physical_name IS NULL`
|
|
115
|
-
// guard keeps this idempotent and race-safe across concurrent first uploads.
|
|
116
|
-
// The catalog-sync trigger on this UPDATE needs `jwt.claims.database_id`, so the
|
|
117
|
-
// write runs under the resolved database claim (privileged role preserved).
|
|
118
|
-
await (0, request_pg_client_1.withRequestPgClient)(withPgClient, { 'jwt.claims.database_id': databaseId }, (client) => (0, graphile_storage_registry_1.recordPhysicalName)((query) => client.query(query), storageConfig.bucketsQualifiedName, bucket.id, s3BucketName));
|
|
119
|
-
bucket.physical_name = s3BucketName;
|
|
120
|
-
log.info(`Recorded physical_name="${s3BucketName}" on bucket ${bucket.id}`);
|
|
121
|
-
return s3BucketName;
|
|
69
|
+
function assertBucketReconciled(bucket, databaseId) {
|
|
70
|
+
if (bucket.physical_name !== null)
|
|
71
|
+
return bucket.physical_name;
|
|
72
|
+
throw new StorageBucketNotReconciledError(bucket, databaseId);
|
|
122
73
|
}
|
package/plugin.js
CHANGED
|
@@ -203,11 +203,9 @@ function createPresignedUrlPlugin(options) {
|
|
|
203
203
|
const bucket = await (0, request_pg_client_1.withRequestPgClient)(vals.withPgClient, vals.pgSettings, (pgClient) => resolveUploadBucket(pgClient, storageConfig, databaseId, vals.bucketKey ?? null, vals.ownerId ?? null, vals.isPublic === true));
|
|
204
204
|
if (!bucket)
|
|
205
205
|
throw new Error('BUCKET_NOT_FOUND');
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
const physicalName = bucket
|
|
209
|
-
? await (0, physical_bucket_1.provisionAndRecordPhysicalBucket)(options, vals.withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins)
|
|
210
|
-
: bucket.physical_name;
|
|
206
|
+
// The reconciler records the coordinate; consumers never
|
|
207
|
+
// recompute it from the logical bucket row.
|
|
208
|
+
const physicalName = (0, physical_bucket_1.assertBucketReconciled)(bucket, databaseId);
|
|
211
209
|
const s3ForDb = (0, physical_bucket_1.resolveS3ForDatabase)(options, storageConfig, physicalName);
|
|
212
210
|
// File row INSERT under the request role (RLS enforced).
|
|
213
211
|
return (0, request_pg_client_1.withRequestPgClient)(vals.withPgClient, vals.pgSettings, (txClient) => processSingleFile(options, txClient, storageConfig, databaseId, bucket, s3ForDb, {
|
|
@@ -310,11 +308,9 @@ function createPresignedUrlPlugin(options) {
|
|
|
310
308
|
if (totalSize > storageConfig.maxBulkTotalSize) {
|
|
311
309
|
throw new Error(`BULK_UPLOAD_SIZE_EXCEEDED: ${totalSize} bytes exceeds maximum of ${storageConfig.maxBulkTotalSize} bytes per batch`);
|
|
312
310
|
}
|
|
313
|
-
//
|
|
314
|
-
//
|
|
315
|
-
const physicalName = bucket
|
|
316
|
-
? await (0, physical_bucket_1.provisionAndRecordPhysicalBucket)(options, vals.withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins)
|
|
317
|
-
: bucket.physical_name;
|
|
311
|
+
// The reconciler records the coordinate; consumers never
|
|
312
|
+
// recompute it from the logical bucket row.
|
|
313
|
+
const physicalName = (0, physical_bucket_1.assertBucketReconciled)(bucket, databaseId);
|
|
318
314
|
const s3ForDb = (0, physical_bucket_1.resolveS3ForDatabase)(options, storageConfig, physicalName);
|
|
319
315
|
// File row INSERTs under the request role (RLS enforced).
|
|
320
316
|
return (0, request_pg_client_1.withRequestPgClient)(vals.withPgClient, vals.pgSettings, async (txClient) => {
|
|
@@ -91,14 +91,6 @@ export declare function getBucketConfig(pgClient: {
|
|
|
91
91
|
rows: unknown[];
|
|
92
92
|
}>;
|
|
93
93
|
}, storageConfig: StorageModuleConfig, databaseId: string, bucketKey: string, ownerId?: string): Promise<BucketConfig | null>;
|
|
94
|
-
/**
|
|
95
|
-
* Check whether an S3 bucket has already been provisioned (cached).
|
|
96
|
-
*/
|
|
97
|
-
export declare function isS3BucketProvisioned(s3BucketName: string): boolean;
|
|
98
|
-
/**
|
|
99
|
-
* Mark an S3 bucket as provisioned in the in-memory cache.
|
|
100
|
-
*/
|
|
101
|
-
export declare function markS3BucketProvisioned(s3BucketName: string): void;
|
|
102
94
|
/**
|
|
103
95
|
* Clear the storage module cache AND bucket cache.
|
|
104
96
|
* Useful for testing or schema changes.
|
package/storage-module-cache.js
CHANGED
|
@@ -5,8 +5,6 @@ exports.loadAllStorageModules = loadAllStorageModules;
|
|
|
5
5
|
exports.resolveStorageConfigFromCodec = resolveStorageConfigFromCodec;
|
|
6
6
|
exports.storedPhysicalName = storedPhysicalName;
|
|
7
7
|
exports.getBucketConfig = getBucketConfig;
|
|
8
|
-
exports.isS3BucketProvisioned = isS3BucketProvisioned;
|
|
9
|
-
exports.markS3BucketProvisioned = markS3BucketProvisioned;
|
|
10
8
|
exports.clearStorageModuleCache = clearStorageModuleCache;
|
|
11
9
|
exports.clearBucketCache = clearBucketCache;
|
|
12
10
|
const logger_1 = require("@pgpmjs/logger");
|
|
@@ -271,34 +269,6 @@ async function getBucketConfig(pgClient, storageConfig, databaseId, bucketKey, o
|
|
|
271
269
|
log.debug(`Cached bucket config for ${databaseId}:${bucketKey} (id=${config.id}, scope=${storageConfig.scope})`);
|
|
272
270
|
return config;
|
|
273
271
|
}
|
|
274
|
-
// --- S3 bucket existence cache ---
|
|
275
|
-
/**
|
|
276
|
-
* In-memory set of S3 bucket names that are known to exist.
|
|
277
|
-
*
|
|
278
|
-
* Used by the lazy provisioning logic in the presigned URL plugin:
|
|
279
|
-
* before generating a presigned PUT URL, the plugin checks this set.
|
|
280
|
-
* If the bucket name is absent, it calls `ensureBucketProvisioned`
|
|
281
|
-
* to create the S3 bucket, then adds the name here. Subsequent
|
|
282
|
-
* requests for the same bucket skip the provisioning entirely.
|
|
283
|
-
*
|
|
284
|
-
* No TTL needed — S3 buckets are never deleted during normal operation.
|
|
285
|
-
* The set resets on server restart, which is fine because the
|
|
286
|
-
* provisioner's createBucket is idempotent (handles "already exists").
|
|
287
|
-
*/
|
|
288
|
-
const provisionedBuckets = new Set();
|
|
289
|
-
/**
|
|
290
|
-
* Check whether an S3 bucket has already been provisioned (cached).
|
|
291
|
-
*/
|
|
292
|
-
function isS3BucketProvisioned(s3BucketName) {
|
|
293
|
-
return provisionedBuckets.has(s3BucketName);
|
|
294
|
-
}
|
|
295
|
-
/**
|
|
296
|
-
* Mark an S3 bucket as provisioned in the in-memory cache.
|
|
297
|
-
*/
|
|
298
|
-
function markS3BucketProvisioned(s3BucketName) {
|
|
299
|
-
provisionedBuckets.add(s3BucketName);
|
|
300
|
-
log.debug(`Marked S3 bucket "${s3BucketName}" as provisioned`);
|
|
301
|
-
}
|
|
302
272
|
/**
|
|
303
273
|
* Clear the storage module cache AND bucket cache.
|
|
304
274
|
* Useful for testing or schema changes.
|
|
@@ -306,7 +276,6 @@ function markS3BucketProvisioned(s3BucketName) {
|
|
|
306
276
|
function clearStorageModuleCache() {
|
|
307
277
|
storageModuleCache.clear();
|
|
308
278
|
bucketCache.clear();
|
|
309
|
-
provisionedBuckets.clear();
|
|
310
279
|
}
|
|
311
280
|
/**
|
|
312
281
|
* Clear cached bucket entries for a specific database.
|
package/types.d.ts
CHANGED
|
@@ -12,10 +12,9 @@ export interface BucketConfig {
|
|
|
12
12
|
max_file_size: number | null;
|
|
13
13
|
allow_custom_keys: boolean;
|
|
14
14
|
/**
|
|
15
|
-
* The physical S3/MinIO bucket name recorded
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* reconstruct the name from a prefix convention.
|
|
15
|
+
* The physical S3/MinIO bucket name recorded by reconciliation. NULL until
|
|
16
|
+
* reconciliation completes. Once set, it is the source of truth for the
|
|
17
|
+
* physical bucket — reads never reconstruct the name.
|
|
19
18
|
*/
|
|
20
19
|
physical_name: string | null;
|
|
21
20
|
}
|
|
@@ -178,51 +177,10 @@ export interface S3Config {
|
|
|
178
177
|
* env-var reads and S3Client creation at module import time.
|
|
179
178
|
*/
|
|
180
179
|
export type S3ConfigOrGetter = S3Config | (() => S3Config);
|
|
181
|
-
/**
|
|
182
|
-
* Function to derive the actual S3 bucket name for a given database and bucket key.
|
|
183
|
-
*
|
|
184
|
-
* When provided, the presigned URL plugin calls this on every request
|
|
185
|
-
* to determine which S3 bucket to use — enabling per-(database, bucketKey)
|
|
186
|
-
* isolation. If not provided, falls back to `s3Config.bucket` (global).
|
|
187
|
-
*
|
|
188
|
-
* @param databaseId - The metaschema database UUID
|
|
189
|
-
* @param bucketKey - The logical bucket key (e.g., "public", "private")
|
|
190
|
-
* @returns The S3 bucket name for this database + bucket key
|
|
191
|
-
*/
|
|
192
|
-
export type BucketNameResolver = (databaseId: string, bucketKey: string) => string;
|
|
193
|
-
/**
|
|
194
|
-
* Callback to lazily provision an S3 bucket on first use.
|
|
195
|
-
*
|
|
196
|
-
* Called by the presigned URL plugin before generating a presigned PUT URL
|
|
197
|
-
* when the bucket has not been seen before (tracked in an in-memory cache).
|
|
198
|
-
* The implementation should create and fully configure the S3 bucket
|
|
199
|
-
* (privacy policies, CORS, lifecycle rules, etc.) — or no-op if the
|
|
200
|
-
* bucket already exists.
|
|
201
|
-
*
|
|
202
|
-
* @param bucketName - The S3 bucket name to provision
|
|
203
|
-
* @param accessType - The logical bucket type ('public', 'private', 'temp')
|
|
204
|
-
* @param databaseId - The metaschema database UUID
|
|
205
|
-
* @param allowedOrigins - Per-database CORS origins (from storage_module), or null to use global fallback
|
|
206
|
-
*/
|
|
207
|
-
export type EnsureBucketProvisioned = (bucketName: string, accessType: 'public' | 'private' | 'temp', databaseId: string, allowedOrigins: string[] | null) => Promise<void>;
|
|
208
180
|
/**
|
|
209
181
|
* Plugin options for the presigned URL plugin.
|
|
210
182
|
*/
|
|
211
183
|
export interface PresignedUrlPluginOptions {
|
|
212
184
|
/** S3 configuration (concrete or lazy getter) */
|
|
213
185
|
s3: S3ConfigOrGetter;
|
|
214
|
-
/**
|
|
215
|
-
* Optional function to resolve S3 bucket name per-database.
|
|
216
|
-
* When set, each database gets its own S3 bucket instead of sharing
|
|
217
|
-
* the global `s3Config.bucket`. The S3 credentials (client) remain shared.
|
|
218
|
-
*/
|
|
219
|
-
resolveBucketName?: BucketNameResolver;
|
|
220
|
-
/**
|
|
221
|
-
* Optional callback to lazily provision an S3 bucket on first upload.
|
|
222
|
-
* When set, the plugin calls this before generating a presigned PUT URL
|
|
223
|
-
* for any S3 bucket it hasn't seen yet (tracked in an in-memory cache).
|
|
224
|
-
* This enables graceful bucket creation without requiring buckets to
|
|
225
|
-
* exist at database provisioning time.
|
|
226
|
-
*/
|
|
227
|
-
ensureBucketProvisioned?: EnsureBucketProvisioned;
|
|
228
186
|
}
|