davepi-plugin-object-storage 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +267 -0
- package/index.js +326 -0
- package/lib/adapters/aws.js +144 -0
- package/lib/adapters/gcs.js +113 -0
- package/lib/adapters/index.js +29 -0
- package/lib/config.js +161 -0
- package/lib/keys.js +71 -0
- package/lib/reaper.js +115 -0
- package/lib/routes.js +209 -0
- package/lib/schema.js +124 -0
- package/package.json +50 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* AWS S3 adapter — also used for Cloudflare R2 and MinIO via the
|
|
5
|
+
* `endpoint` override. All three speak the same S3 wire protocol so
|
|
6
|
+
* one client + one presigner cover them. The differences are:
|
|
7
|
+
*
|
|
8
|
+
* - AWS: no endpoint override, virtual-host-style URLs.
|
|
9
|
+
* - R2: endpoint override to `https://<acct>.r2.cloudflarestorage.com`,
|
|
10
|
+
* virtual-host-style URLs supported.
|
|
11
|
+
* - MinIO: endpoint override to `http://<host>:9000`, MUST be
|
|
12
|
+
* path-style (the bucket sits in the URL path, not the host).
|
|
13
|
+
*
|
|
14
|
+
* `forcePathStyle` is the env knob; the readConfig default flips it on
|
|
15
|
+
* for MinIO and off elsewhere, but operators can override per-deploy
|
|
16
|
+
* for the rare bucket whose name isn't DNS-safe.
|
|
17
|
+
*
|
|
18
|
+
* The SDK is lazy-loaded so the gcs-only consumer doesn't pay the AWS
|
|
19
|
+
* SDK cost. (`@aws-sdk/client-s3` and the presigner are still hard deps
|
|
20
|
+
* of the plugin's package.json because the common case needs them; lazy
|
|
21
|
+
* load is about runtime startup, not install footprint.)
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
function createAwsAdapter(config, { sdkOverride } = {}) {
|
|
25
|
+
// See gcs.js: only `undefined` (no override) falls back to loadSdk();
|
|
26
|
+
// an explicit `null` means "SDK unavailable" and must not re-require.
|
|
27
|
+
const sdk = sdkOverride === undefined ? loadSdk() : sdkOverride;
|
|
28
|
+
// Mirror the gcs adapter's `if (!sdk) throw`: an explicit null
|
|
29
|
+
// override (or a malformed SDK object) must fail fast with a clear
|
|
30
|
+
// message rather than crashing on the `sdk.client` destructuring
|
|
31
|
+
// below with a cryptic "Cannot read properties of null".
|
|
32
|
+
if (!sdk || !sdk.client || !sdk.presigner) {
|
|
33
|
+
throw new Error(
|
|
34
|
+
'davepi-plugin-object-storage (aws adapter): AWS SDK is not available. ' +
|
|
35
|
+
'Ensure @aws-sdk/client-s3 and @aws-sdk/s3-request-presigner are installed, ' +
|
|
36
|
+
'or set S3_BACKEND to gcs.'
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
const {
|
|
40
|
+
S3Client,
|
|
41
|
+
PutObjectCommand,
|
|
42
|
+
GetObjectCommand,
|
|
43
|
+
HeadObjectCommand,
|
|
44
|
+
DeleteObjectCommand,
|
|
45
|
+
} = sdk.client;
|
|
46
|
+
const { getSignedUrl } = sdk.presigner;
|
|
47
|
+
|
|
48
|
+
if (!config.bucket) {
|
|
49
|
+
throw new Error('davepi-plugin-object-storage (aws adapter): S3_BUCKET is required');
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const region = config.region || 'us-east-1';
|
|
53
|
+
const clientOpts = {
|
|
54
|
+
region,
|
|
55
|
+
forcePathStyle: !!config.forcePathStyle,
|
|
56
|
+
};
|
|
57
|
+
if (config.endpoint) clientOpts.endpoint = config.endpoint;
|
|
58
|
+
if (config.accessKeyId && config.secretAccessKey) {
|
|
59
|
+
clientOpts.credentials = {
|
|
60
|
+
accessKeyId: config.accessKeyId,
|
|
61
|
+
secretAccessKey: config.secretAccessKey,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
// No credentials in env → fall through to the SDK's default credential
|
|
65
|
+
// chain (IRSA on EKS, EC2/ECS metadata, ~/.aws/credentials). That's
|
|
66
|
+
// intentional: hard-coded keys are the dev-machine path, IAM roles are
|
|
67
|
+
// the production path, and the SDK handles both without our help.
|
|
68
|
+
const client = new S3Client(clientOpts);
|
|
69
|
+
|
|
70
|
+
async function getSignedPutUrl({ key, contentType, expires }) {
|
|
71
|
+
const cmd = new PutObjectCommand({
|
|
72
|
+
Bucket: config.bucket,
|
|
73
|
+
Key: key,
|
|
74
|
+
ContentType: contentType,
|
|
75
|
+
});
|
|
76
|
+
return getSignedUrl(client, cmd, { expiresIn: expires });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function getSignedGetUrl({ key, expires }) {
|
|
80
|
+
const cmd = new GetObjectCommand({ Bucket: config.bucket, Key: key });
|
|
81
|
+
return getSignedUrl(client, cmd, { expiresIn: expires });
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function headObject({ key }) {
|
|
85
|
+
try {
|
|
86
|
+
const out = await client.send(
|
|
87
|
+
new HeadObjectCommand({ Bucket: config.bucket, Key: key })
|
|
88
|
+
);
|
|
89
|
+
return {
|
|
90
|
+
exists: true,
|
|
91
|
+
contentLength: typeof out.ContentLength === 'number' ? out.ContentLength : null,
|
|
92
|
+
contentType: out.ContentType || null,
|
|
93
|
+
etag: out.ETag || null,
|
|
94
|
+
};
|
|
95
|
+
} catch (err) {
|
|
96
|
+
const status = err && err.$metadata && err.$metadata.httpStatusCode;
|
|
97
|
+
if (status === 404 || (err && err.name === 'NotFound')) {
|
|
98
|
+
return { exists: false };
|
|
99
|
+
}
|
|
100
|
+
throw err;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function deleteObject({ key }) {
|
|
105
|
+
await client.send(
|
|
106
|
+
new DeleteObjectCommand({ Bucket: config.bucket, Key: key })
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function publicUrl({ key }) {
|
|
111
|
+
if (config.publicBaseUrl) {
|
|
112
|
+
return `${config.publicBaseUrl.replace(/\/+$/, '')}/${key}`;
|
|
113
|
+
}
|
|
114
|
+
if (config.endpoint) {
|
|
115
|
+
// Path-style URL for endpoint-overridden providers; safe for both
|
|
116
|
+
// R2's custom domain and MinIO's bare host.
|
|
117
|
+
return `${config.endpoint.replace(/\/+$/, '')}/${config.bucket}/${key}`;
|
|
118
|
+
}
|
|
119
|
+
return `https://${config.bucket}.s3.${region}.amazonaws.com/${key}`;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
name: `aws:${config.backend}`,
|
|
124
|
+
bucket: config.bucket,
|
|
125
|
+
region,
|
|
126
|
+
getSignedPutUrl,
|
|
127
|
+
getSignedGetUrl,
|
|
128
|
+
headObject,
|
|
129
|
+
deleteObject,
|
|
130
|
+
publicUrl,
|
|
131
|
+
_client: client, // exposed for advanced consumers via `plugin.adapter`
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function loadSdk() {
|
|
136
|
+
// Lazy require so a project that only uses the gcs backend doesn't
|
|
137
|
+
// execute the AWS SDK module graph at boot.
|
|
138
|
+
return {
|
|
139
|
+
client: require('@aws-sdk/client-s3'),
|
|
140
|
+
presigner: require('@aws-sdk/s3-request-presigner'),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
module.exports = { createAwsAdapter };
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Google Cloud Storage adapter.
|
|
5
|
+
*
|
|
6
|
+
* GCS exposes a presigned-URL surface ("V4 signed URLs") but with a
|
|
7
|
+
* different SDK + auth model than S3 — there's no drop-in shim. Hence
|
|
8
|
+
* a real adapter file rather than a parameterised aws.js variant.
|
|
9
|
+
*
|
|
10
|
+
* `@google-cloud/storage` is declared as an `optionalDependencies` so a
|
|
11
|
+
* consumer who doesn't run GCS doesn't have to install it. The require
|
|
12
|
+
* is wrapped — if the package isn't on the path, the plugin throws a
|
|
13
|
+
* clear "set S3_BACKEND=aws or install @google-cloud/storage" message
|
|
14
|
+
* at setup time rather than crashing with a cryptic MODULE_NOT_FOUND
|
|
15
|
+
* mid-request.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
function createGcsAdapter(config, { sdkOverride } = {}) {
|
|
19
|
+
// Only an absent override (`undefined`) falls back to loadSdk(); an
|
|
20
|
+
// explicit `null` means "SDK unavailable" (tests inject this to
|
|
21
|
+
// simulate @google-cloud/storage not being installed, regardless of
|
|
22
|
+
// whether it actually is on the path). Using `||` here would swallow
|
|
23
|
+
// that null and re-require the real package when it happens to be
|
|
24
|
+
// installed — e.g. as an optionalDependency that `npm install` pulls
|
|
25
|
+
// in on CI, which is exactly what made these tests fail at publish.
|
|
26
|
+
const sdk = sdkOverride === undefined ? loadSdk() : sdkOverride;
|
|
27
|
+
if (!sdk) {
|
|
28
|
+
throw new Error(
|
|
29
|
+
'davepi-plugin-object-storage (gcs adapter): @google-cloud/storage is not installed. ' +
|
|
30
|
+
'Add it to your dependencies, or set S3_BACKEND to aws / r2 / minio.'
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
const { Storage } = sdk;
|
|
34
|
+
|
|
35
|
+
if (!config.bucket) {
|
|
36
|
+
throw new Error('davepi-plugin-object-storage (gcs adapter): S3_BUCKET is required');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const storageOpts = {};
|
|
40
|
+
if (config.gcsProjectId) storageOpts.projectId = config.gcsProjectId;
|
|
41
|
+
if (config.gcsKeyFile) storageOpts.keyFilename = config.gcsKeyFile;
|
|
42
|
+
const client = new Storage(storageOpts);
|
|
43
|
+
const bucket = client.bucket(config.bucket);
|
|
44
|
+
|
|
45
|
+
async function getSignedPutUrl({ key, contentType, expires }) {
|
|
46
|
+
const [url] = await bucket.file(key).getSignedUrl({
|
|
47
|
+
version: 'v4',
|
|
48
|
+
action: 'write',
|
|
49
|
+
expires: Date.now() + expires * 1000,
|
|
50
|
+
contentType,
|
|
51
|
+
});
|
|
52
|
+
return url;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function getSignedGetUrl({ key, expires }) {
|
|
56
|
+
const [url] = await bucket.file(key).getSignedUrl({
|
|
57
|
+
version: 'v4',
|
|
58
|
+
action: 'read',
|
|
59
|
+
expires: Date.now() + expires * 1000,
|
|
60
|
+
});
|
|
61
|
+
return url;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async function headObject({ key }) {
|
|
65
|
+
try {
|
|
66
|
+
const [metadata] = await bucket.file(key).getMetadata();
|
|
67
|
+
const size = metadata && (metadata.size || metadata.Size);
|
|
68
|
+
return {
|
|
69
|
+
exists: true,
|
|
70
|
+
contentLength: size != null ? Number(size) : null,
|
|
71
|
+
contentType: (metadata && (metadata.contentType || metadata.ContentType)) || null,
|
|
72
|
+
etag: (metadata && (metadata.etag || metadata.Etag)) || null,
|
|
73
|
+
};
|
|
74
|
+
} catch (err) {
|
|
75
|
+
if (err && (err.code === 404 || err.statusCode === 404)) {
|
|
76
|
+
return { exists: false };
|
|
77
|
+
}
|
|
78
|
+
throw err;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function deleteObject({ key }) {
|
|
83
|
+
await bucket.file(key).delete({ ignoreNotFound: true });
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function publicUrl({ key }) {
|
|
87
|
+
if (config.publicBaseUrl) {
|
|
88
|
+
return `${config.publicBaseUrl.replace(/\/+$/, '')}/${key}`;
|
|
89
|
+
}
|
|
90
|
+
return `https://storage.googleapis.com/${config.bucket}/${key}`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return {
|
|
94
|
+
name: 'gcs',
|
|
95
|
+
bucket: config.bucket,
|
|
96
|
+
getSignedPutUrl,
|
|
97
|
+
getSignedGetUrl,
|
|
98
|
+
headObject,
|
|
99
|
+
deleteObject,
|
|
100
|
+
publicUrl,
|
|
101
|
+
_client: client,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function loadSdk() {
|
|
106
|
+
try {
|
|
107
|
+
return require('@google-cloud/storage');
|
|
108
|
+
} catch (_err) {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
module.exports = { createGcsAdapter };
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Adapter factory. Picks an adapter implementation based on the
|
|
5
|
+
* config's `backend` ('aws' | 'r2' | 'minio' | 'gcs'). R2 and MinIO
|
|
6
|
+
* share the AWS adapter — they speak the S3 wire protocol and only
|
|
7
|
+
* differ via the `endpoint` override that readConfig already wired up.
|
|
8
|
+
*
|
|
9
|
+
* The factory accepts an optional `sdkOverrides` for tests: pass
|
|
10
|
+
* `{ aws: { client, presigner }, gcs: { Storage } }` to swap the
|
|
11
|
+
* underlying SDKs without touching the require graph.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const { createAwsAdapter } = require('./aws');
|
|
15
|
+
const { createGcsAdapter } = require('./gcs');
|
|
16
|
+
|
|
17
|
+
function createAdapter(config, { sdkOverrides = {} } = {}) {
|
|
18
|
+
switch (config.backend) {
|
|
19
|
+
case 'gcs':
|
|
20
|
+
return createGcsAdapter(config, { sdkOverride: sdkOverrides.gcs });
|
|
21
|
+
case 'r2':
|
|
22
|
+
case 'minio':
|
|
23
|
+
case 'aws':
|
|
24
|
+
default:
|
|
25
|
+
return createAwsAdapter(config, { sdkOverride: sdkOverrides.aws });
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
module.exports = { createAdapter, createAwsAdapter, createGcsAdapter };
|
package/lib/config.js
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Parse env vars into a typed config object. All env-driven knobs live
|
|
5
|
+
* here so the plugin's setup() doesn't litter parseInt / parseBool /
|
|
6
|
+
* split-on-comma at the call sites.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const DEFAULTS = {
|
|
10
|
+
backend: 'aws',
|
|
11
|
+
putUrlTtlSeconds: 300,
|
|
12
|
+
getUrlTtlSeconds: 600,
|
|
13
|
+
maxBytes: 50 * 1024 * 1024, // 50 MiB
|
|
14
|
+
filePath: 'file',
|
|
15
|
+
fileVersion: 'v1',
|
|
16
|
+
routePrefix: '/api/files',
|
|
17
|
+
cascadeDelete: false,
|
|
18
|
+
verifyOnComplete: true,
|
|
19
|
+
reapEnabled: true,
|
|
20
|
+
// The reaper sweeps `pending` records that are older than the
|
|
21
|
+
// presigned PUT URL's lifetime by a wide margin. The default 3× multiplier
|
|
22
|
+
// gives clients comfortable headroom for slow networks (a 5-minute URL
|
|
23
|
+
// means 15-minute orphan retention) without keeping garbage indefinitely.
|
|
24
|
+
reapMultiplier: 3,
|
|
25
|
+
reapIntervalMs: 5 * 60 * 1000,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const BACKENDS = new Set(['aws', 'r2', 'minio', 'gcs']);
|
|
29
|
+
|
|
30
|
+
function parseBool(raw, fallback) {
|
|
31
|
+
if (raw === undefined || raw === null || raw === '') return fallback;
|
|
32
|
+
const v = String(raw).trim().toLowerCase();
|
|
33
|
+
if (['1', 'true', 'yes', 'on'].includes(v)) return true;
|
|
34
|
+
if (['0', 'false', 'no', 'off'].includes(v)) return false;
|
|
35
|
+
return fallback;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function parseInteger(raw, fallback, { min = 1 } = {}) {
|
|
39
|
+
if (raw === undefined || raw === null || raw === '') return fallback;
|
|
40
|
+
const n = parseInt(raw, 10);
|
|
41
|
+
if (!Number.isFinite(n) || n < min) return fallback;
|
|
42
|
+
return n;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function parseList(raw) {
|
|
46
|
+
if (raw === undefined || raw === null || raw === '') return [];
|
|
47
|
+
return String(raw)
|
|
48
|
+
.split(',')
|
|
49
|
+
.map((s) => s.trim())
|
|
50
|
+
.filter(Boolean);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Read the plugin's full env-driven config. Operators can also override
|
|
55
|
+
* any field by passing the same shape into `createPlugin({ env })`; the
|
|
56
|
+
* default export reads `process.env` so unconfigured projects can list
|
|
57
|
+
* the plugin in `davepi.plugins` and still boot.
|
|
58
|
+
*/
|
|
59
|
+
function readConfig(env = process.env) {
|
|
60
|
+
const backendRaw = (env.S3_BACKEND || DEFAULTS.backend).toLowerCase();
|
|
61
|
+
const backend = BACKENDS.has(backendRaw) ? backendRaw : DEFAULTS.backend;
|
|
62
|
+
|
|
63
|
+
return {
|
|
64
|
+
backend,
|
|
65
|
+
bucket: env.S3_BUCKET || null,
|
|
66
|
+
region: env.S3_REGION || env.AWS_REGION || null,
|
|
67
|
+
endpoint: env.S3_ENDPOINT || null,
|
|
68
|
+
accessKeyId: env.S3_ACCESS_KEY_ID || env.AWS_ACCESS_KEY_ID || null,
|
|
69
|
+
secretAccessKey: env.S3_SECRET_ACCESS_KEY || env.AWS_SECRET_ACCESS_KEY || null,
|
|
70
|
+
forcePathStyle: parseBool(env.S3_FORCE_PATH_STYLE, backend === 'minio'),
|
|
71
|
+
publicBaseUrl: env.S3_PUBLIC_BASE_URL || null,
|
|
72
|
+
putUrlTtlSeconds: parseInteger(env.S3_PUT_URL_TTL_SECONDS, DEFAULTS.putUrlTtlSeconds),
|
|
73
|
+
getUrlTtlSeconds: parseInteger(env.S3_GET_URL_TTL_SECONDS, DEFAULTS.getUrlTtlSeconds),
|
|
74
|
+
maxBytes: parseInteger(env.S3_MAX_BYTES, DEFAULTS.maxBytes),
|
|
75
|
+
allowedMime: parseList(env.S3_ALLOWED_MIME),
|
|
76
|
+
cascadeDelete: parseBool(env.S3_CASCADE_DELETE, DEFAULTS.cascadeDelete),
|
|
77
|
+
verifyOnComplete: parseBool(env.S3_VERIFY_ON_COMPLETE, DEFAULTS.verifyOnComplete),
|
|
78
|
+
filePath: env.S3_FILE_PATH || DEFAULTS.filePath,
|
|
79
|
+
fileVersion: env.S3_FILE_VERSION || DEFAULTS.fileVersion,
|
|
80
|
+
routePrefix: normalizePrefix(env.S3_ROUTE_PREFIX || DEFAULTS.routePrefix),
|
|
81
|
+
reapEnabled: parseBool(env.S3_REAP_ENABLED, DEFAULTS.reapEnabled),
|
|
82
|
+
reapMultiplier: parseInteger(env.S3_REAP_MULTIPLIER, DEFAULTS.reapMultiplier),
|
|
83
|
+
reapIntervalMs: parseInteger(env.S3_REAP_INTERVAL_MS, DEFAULTS.reapIntervalMs, { min: 1000 }),
|
|
84
|
+
gcsProjectId: env.GCS_PROJECT_ID || null,
|
|
85
|
+
gcsKeyFile: env.GCS_KEY_FILE || null,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function normalizePrefix(p) {
|
|
90
|
+
if (typeof p !== 'string' || !p) return DEFAULTS.routePrefix;
|
|
91
|
+
let out = p.startsWith('/') ? p : `/${p}`;
|
|
92
|
+
if (out.length > 1 && out.endsWith('/')) out = out.slice(0, -1);
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Verify the contentType passes the configured allowlist. An empty
|
|
98
|
+
* allowlist means "any". The wildcard form `image/*` matches every
|
|
99
|
+
* subtype the way standard Accept-headers expect.
|
|
100
|
+
*/
|
|
101
|
+
function mimeAllowed(contentType, allowedMime) {
|
|
102
|
+
if (!Array.isArray(allowedMime) || allowedMime.length === 0) return true;
|
|
103
|
+
if (typeof contentType !== 'string' || !contentType) return false;
|
|
104
|
+
const ct = contentType.toLowerCase();
|
|
105
|
+
for (const pat of allowedMime) {
|
|
106
|
+
const p = pat.toLowerCase();
|
|
107
|
+
if (p === ct) return true;
|
|
108
|
+
if (p.endsWith('/*')) {
|
|
109
|
+
const prefix = p.slice(0, -2);
|
|
110
|
+
if (ct.startsWith(`${prefix}/`)) return true;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Validate a presigned-upload request against the configured policy.
|
|
118
|
+
* Throws via the supplied `errors.ValidationError` on first failure;
|
|
119
|
+
* returns void on success. Both the REST `POST /upload-url` handler
|
|
120
|
+
* and the programmatic `createUploadUrl` route their input through
|
|
121
|
+
* this so the policy is enforced uniformly — a consumer who calls the
|
|
122
|
+
* programmatic API from a hook can't bypass the MIME allowlist or the
|
|
123
|
+
* max-bytes gate that the route enforces.
|
|
124
|
+
*
|
|
125
|
+
* `errors` is the framework's `utils/errors` module (or a stub with
|
|
126
|
+
* the same shape); passed in rather than required at module scope so
|
|
127
|
+
* the package's own unit tests can run without `davepi` installed.
|
|
128
|
+
*/
|
|
129
|
+
function validateUploadRequest({ contentType, size, config, errors }) {
|
|
130
|
+
const { ValidationError } = errors;
|
|
131
|
+
if (typeof contentType !== 'string' || !contentType) {
|
|
132
|
+
throw new ValidationError('contentType is required');
|
|
133
|
+
}
|
|
134
|
+
if (!mimeAllowed(contentType, config.allowedMime)) {
|
|
135
|
+
throw new ValidationError(
|
|
136
|
+
`contentType ${contentType} is not in S3_ALLOWED_MIME`
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
if (size !== undefined && size !== null) {
|
|
140
|
+
if (typeof size !== 'number' || size <= 0 || !Number.isFinite(size)) {
|
|
141
|
+
throw new ValidationError('size must be a positive number');
|
|
142
|
+
}
|
|
143
|
+
if (size > config.maxBytes) {
|
|
144
|
+
throw new ValidationError(
|
|
145
|
+
`size ${size} exceeds S3_MAX_BYTES (${config.maxBytes})`
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
module.exports = {
|
|
152
|
+
DEFAULTS,
|
|
153
|
+
BACKENDS,
|
|
154
|
+
readConfig,
|
|
155
|
+
parseBool,
|
|
156
|
+
parseInteger,
|
|
157
|
+
parseList,
|
|
158
|
+
mimeAllowed,
|
|
159
|
+
normalizePrefix,
|
|
160
|
+
validateUploadRequest,
|
|
161
|
+
};
|
package/lib/keys.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Object-key generator. The shape is:
|
|
5
|
+
*
|
|
6
|
+
* <userId>/<shortHash>/<safeOriginalName>
|
|
7
|
+
*
|
|
8
|
+
* The userId prefix means a flat `aws s3 ls` against the bucket
|
|
9
|
+
* immediately tells an operator which tenant a key belongs to without
|
|
10
|
+
* cross-referencing the DB. The 8-hex-char `shortHash` block prevents
|
|
11
|
+
* accidental collisions between two files a tenant uploads with the
|
|
12
|
+
* same name, and isolates listing-by-prefix to per-file granularity.
|
|
13
|
+
* The original name (slugified) trails so the storage layer's
|
|
14
|
+
* Content-Disposition can echo it back without extra DB work.
|
|
15
|
+
*
|
|
16
|
+
* The record `_id` remains the authoritative identifier; the key is
|
|
17
|
+
* convenience-encoding for humans + log readers.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const crypto = require('node:crypto');
|
|
21
|
+
|
|
22
|
+
const SAFE_NAME_RE = /[^a-zA-Z0-9._-]+/g;
|
|
23
|
+
const MAX_NAME_LEN = 128;
|
|
24
|
+
|
|
25
|
+
function slugifyName(name) {
|
|
26
|
+
if (typeof name !== 'string' || !name) return 'file';
|
|
27
|
+
// Strip any path component a careless client may have sent; Windows
|
|
28
|
+
// backslashes and Unix slashes both get split, and we keep only the
|
|
29
|
+
// basename. Then collapse whitespace + non-safe chars to underscore.
|
|
30
|
+
const base = name.split(/[\\/]/).pop() || 'file';
|
|
31
|
+
const cleaned = base.replace(SAFE_NAME_RE, '_').replace(/^_+|_+$/g, '');
|
|
32
|
+
const truncated = cleaned.slice(0, MAX_NAME_LEN);
|
|
33
|
+
return truncated || 'file';
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function shortHash() {
|
|
37
|
+
return crypto.randomBytes(4).toString('hex');
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Build a fresh storage key for an upload. The `userId` is required —
|
|
42
|
+
* the plugin's tenant-isolation invariant rests on every key carrying
|
|
43
|
+
* its owner as the first path component.
|
|
44
|
+
*/
|
|
45
|
+
function buildKey({ userId, originalName }) {
|
|
46
|
+
if (!userId) {
|
|
47
|
+
throw new Error('buildKey requires userId');
|
|
48
|
+
}
|
|
49
|
+
return `${userId}/${shortHash()}/${slugifyName(originalName || 'file')}`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Extract the userId from a key, or `null` if the shape is unexpected.
|
|
54
|
+
* Used by the cascade-delete subscriber to refuse to delete blobs whose
|
|
55
|
+
* stored key disagrees with the record's `userId` (defence in depth: a
|
|
56
|
+
* stamped record can't smuggle a foreign-tenant key past the storage
|
|
57
|
+
* layer).
|
|
58
|
+
*/
|
|
59
|
+
function userIdOfKey(key) {
|
|
60
|
+
if (typeof key !== 'string' || !key) return null;
|
|
61
|
+
const slash = key.indexOf('/');
|
|
62
|
+
if (slash < 1) return null;
|
|
63
|
+
return key.slice(0, slash);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
module.exports = {
|
|
67
|
+
buildKey,
|
|
68
|
+
slugifyName,
|
|
69
|
+
userIdOfKey,
|
|
70
|
+
shortHash,
|
|
71
|
+
};
|
package/lib/reaper.js
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Pending-record reaper.
|
|
5
|
+
*
|
|
6
|
+
* A `file` record is created in `status: 'pending'` when the upload-url
|
|
7
|
+
* route hands the client a presigned PUT URL. If the client never
|
|
8
|
+
* completes the upload (closed laptop, network drop, abandoned UI),
|
|
9
|
+
* the record sits in the DB and the partial blob — if any reached the
|
|
10
|
+
* bucket before the URL expired — sits in storage forever.
|
|
11
|
+
*
|
|
12
|
+
* This module exposes a `setInterval`-driven sweep:
|
|
13
|
+
*
|
|
14
|
+
* - finds `pending` records older than
|
|
15
|
+
* `putUrlTtlSeconds * reapMultiplier` seconds
|
|
16
|
+
* - calls the adapter's `deleteObject` on each key (best-effort —
|
|
17
|
+
* the object usually doesn't exist, in which case `headObject`
|
|
18
|
+
* reports 404 and we skip the delete call to save a round trip)
|
|
19
|
+
* - removes the DB record
|
|
20
|
+
*
|
|
21
|
+
* Operators can disable the in-process reaper by setting
|
|
22
|
+
* `S3_REAP_ENABLED=false` (e.g. for projects that run the cron plugin
|
|
23
|
+
* to drive cleanup at their own cadence; the reaper exports `runOnce`
|
|
24
|
+
* for those cases).
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
function createReaper({ getModel, adapter, config, log }) {
|
|
28
|
+
let timer = null;
|
|
29
|
+
let inflight = false;
|
|
30
|
+
|
|
31
|
+
const ttlMs = config.putUrlTtlSeconds * config.reapMultiplier * 1000;
|
|
32
|
+
|
|
33
|
+
async function runOnce({ now = Date.now() } = {}) {
|
|
34
|
+
if (inflight) return { skipped: true, reason: 'already running' };
|
|
35
|
+
inflight = true;
|
|
36
|
+
let deleted = 0;
|
|
37
|
+
try {
|
|
38
|
+
const Model = getModel();
|
|
39
|
+
if (!Model) return { deleted: 0, reason: 'no model' };
|
|
40
|
+
const cutoff = new Date(now - ttlMs);
|
|
41
|
+
// The framework's collection has `createdAt` from mongoose-timestamp.
|
|
42
|
+
// We deliberately use `createdAt` (not `updatedAt`) so a /complete
|
|
43
|
+
// retry that touches the doc doesn't reset the reap clock for a
|
|
44
|
+
// record that's still stuck in `pending`.
|
|
45
|
+
const stale = await Model.find({
|
|
46
|
+
status: 'pending',
|
|
47
|
+
createdAt: { $lt: cutoff },
|
|
48
|
+
}).limit(100);
|
|
49
|
+
|
|
50
|
+
for (const doc of stale) {
|
|
51
|
+
try {
|
|
52
|
+
// Always issue the delete: cheaper than a HEAD round-trip,
|
|
53
|
+
// and adapter implementations swallow 404s for us.
|
|
54
|
+
await adapter.deleteObject({ key: doc.key });
|
|
55
|
+
} catch (err) {
|
|
56
|
+
// Storage failure: log and skip removing the DB row, so the
|
|
57
|
+
// next sweep retries. The DB row stays in `pending` and
|
|
58
|
+
// remains over the cutoff threshold, so it's picked up again
|
|
59
|
+
// — no lost work.
|
|
60
|
+
if (log && typeof log.warn === 'function') {
|
|
61
|
+
log.warn(
|
|
62
|
+
{ err, plugin: 'object-storage', key: doc.key },
|
|
63
|
+
'davepi-plugin-object-storage: reaper failed to delete storage object; will retry'
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
await Model.deleteOne({ _id: doc._id });
|
|
70
|
+
deleted += 1;
|
|
71
|
+
} catch (err) {
|
|
72
|
+
if (log && typeof log.warn === 'function') {
|
|
73
|
+
log.warn(
|
|
74
|
+
{ err, plugin: 'object-storage', fileId: String(doc._id) },
|
|
75
|
+
'davepi-plugin-object-storage: reaper failed to delete pending record; will retry'
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return { deleted };
|
|
81
|
+
} finally {
|
|
82
|
+
inflight = false;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function start() {
|
|
87
|
+
if (timer) return;
|
|
88
|
+
if (!config.reapEnabled) return;
|
|
89
|
+
timer = setInterval(() => {
|
|
90
|
+
runOnce().catch((err) => {
|
|
91
|
+
// Last-resort: a thrown error escaping runOnce would crash the
|
|
92
|
+
// setInterval task on older Node versions. Belt-and-suspenders.
|
|
93
|
+
if (log && typeof log.error === 'function') {
|
|
94
|
+
log.error(
|
|
95
|
+
{ err, plugin: 'object-storage' },
|
|
96
|
+
'davepi-plugin-object-storage: reaper sweep threw unexpectedly'
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
}, config.reapIntervalMs);
|
|
101
|
+
// Don't pin the event loop — a process whose only remaining task
|
|
102
|
+
// is the reaper interval should still exit cleanly.
|
|
103
|
+
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function stop() {
|
|
107
|
+
if (!timer) return;
|
|
108
|
+
clearInterval(timer);
|
|
109
|
+
timer = null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return { start, stop, runOnce };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
module.exports = { createReaper };
|