@blamejs/core 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +230 -0
- package/LICENSE +201 -0
- package/LTS-CALENDAR.md +29 -0
- package/MIGRATING.md +7 -0
- package/NOTICE +59 -0
- package/README.md +100 -0
- package/bin/blamejs.js +13 -0
- package/index.js +253 -0
- package/lib/api-key.js +705 -0
- package/lib/api-snapshot.js +335 -0
- package/lib/app-shutdown.js +381 -0
- package/lib/app.js +364 -0
- package/lib/atomic-file.js +525 -0
- package/lib/audit-chain.js +168 -0
- package/lib/audit-sign.js +319 -0
- package/lib/audit-tools.js +682 -0
- package/lib/audit.js +753 -0
- package/lib/auth/jwt.js +280 -0
- package/lib/auth/oauth.js +691 -0
- package/lib/auth/passkey.js +185 -0
- package/lib/auth/password.js +139 -0
- package/lib/auth/totp.js +17 -0
- package/lib/auth-header.js +81 -0
- package/lib/backup/bundle.js +219 -0
- package/lib/backup/crypto.js +174 -0
- package/lib/backup/index.js +490 -0
- package/lib/backup/manifest.js +275 -0
- package/lib/bundler.js +295 -0
- package/lib/cache.js +819 -0
- package/lib/chain-writer.js +234 -0
- package/lib/cli-helpers.js +201 -0
- package/lib/cli.js +1377 -0
- package/lib/cluster-provider-db.js +245 -0
- package/lib/cluster-storage.js +166 -0
- package/lib/cluster.js +691 -0
- package/lib/consent.js +222 -0
- package/lib/constants.js +186 -0
- package/lib/cookies.js +293 -0
- package/lib/credential-hash.js +303 -0
- package/lib/crypto-field.js +159 -0
- package/lib/crypto.js +250 -0
- package/lib/db-query.js +297 -0
- package/lib/db-schema.js +250 -0
- package/lib/db.js +1054 -0
- package/lib/deprecate.js +226 -0
- package/lib/dev.js +324 -0
- package/lib/error-page.js +424 -0
- package/lib/events.js +135 -0
- package/lib/external-db.js +422 -0
- package/lib/forms.js +378 -0
- package/lib/framework-error.js +189 -0
- package/lib/framework-schema.js +604 -0
- package/lib/handlers.js +350 -0
- package/lib/html-balance.js +227 -0
- package/lib/http-client.js +615 -0
- package/lib/i18n.js +780 -0
- package/lib/jobs.js +181 -0
- package/lib/lazy-require.js +48 -0
- package/lib/log-stream-local.js +137 -0
- package/lib/log-stream-webhook.js +170 -0
- package/lib/log-stream.js +211 -0
- package/lib/log.js +355 -0
- package/lib/mail-bounce.js +507 -0
- package/lib/mail.js +701 -0
- package/lib/metrics.js +647 -0
- package/lib/middleware/api-encrypt.js +553 -0
- package/lib/middleware/attach-user.js +156 -0
- package/lib/middleware/body-parser.js +883 -0
- package/lib/middleware/bot-guard.js +148 -0
- package/lib/middleware/compression.js +436 -0
- package/lib/middleware/cors.js +236 -0
- package/lib/middleware/csp-nonce.js +332 -0
- package/lib/middleware/csrf-protect.js +275 -0
- package/lib/middleware/error-handler.js +46 -0
- package/lib/middleware/health.js +358 -0
- package/lib/middleware/index.js +52 -0
- package/lib/middleware/rate-limit.js +319 -0
- package/lib/middleware/request-id.js +53 -0
- package/lib/middleware/require-auth.js +95 -0
- package/lib/middleware/security-headers.js +91 -0
- package/lib/migrations.js +353 -0
- package/lib/mtls-ca.js +333 -0
- package/lib/mtls-engine-default.js +285 -0
- package/lib/nonce-store.js +177 -0
- package/lib/notify.js +643 -0
- package/lib/ntp-check.js +178 -0
- package/lib/object-store/azure-blob.js +467 -0
- package/lib/object-store/gcs.js +469 -0
- package/lib/object-store/http-put.js +153 -0
- package/lib/object-store/index.js +140 -0
- package/lib/object-store/local.js +163 -0
- package/lib/object-store/retry.js +15 -0
- package/lib/object-store/sigv4.js +535 -0
- package/lib/observability.js +114 -0
- package/lib/pagination.js +371 -0
- package/lib/parsers/index.js +64 -0
- package/lib/parsers/safe-csv.js +224 -0
- package/lib/parsers/safe-env.js +614 -0
- package/lib/parsers/safe-toml.js +745 -0
- package/lib/parsers/safe-xml.js +379 -0
- package/lib/parsers/safe-yaml.js +977 -0
- package/lib/permissions.js +430 -0
- package/lib/pqc-agent.js +85 -0
- package/lib/pqc-gate.js +266 -0
- package/lib/protocol-dispatcher.js +144 -0
- package/lib/queue-local.js +327 -0
- package/lib/queue.js +430 -0
- package/lib/redact.js +192 -0
- package/lib/render.js +193 -0
- package/lib/request-helpers.js +178 -0
- package/lib/restore-bundle.js +239 -0
- package/lib/restore-rollback.js +254 -0
- package/lib/restore.js +301 -0
- package/lib/retry.js +329 -0
- package/lib/router.js +437 -0
- package/lib/safe-async.js +520 -0
- package/lib/safe-buffer.js +162 -0
- package/lib/safe-json.js +532 -0
- package/lib/safe-schema.js +1176 -0
- package/lib/safe-sql.js +157 -0
- package/lib/safe-url.js +109 -0
- package/lib/scheduler.js +680 -0
- package/lib/seeders.js +622 -0
- package/lib/session.js +304 -0
- package/lib/slug.js +243 -0
- package/lib/static.js +268 -0
- package/lib/storage.js +470 -0
- package/lib/subject.js +281 -0
- package/lib/template.js +781 -0
- package/lib/testing.js +621 -0
- package/lib/totp.js +285 -0
- package/lib/tracing.js +484 -0
- package/lib/validate-opts.js +56 -0
- package/lib/vault/index.js +299 -0
- package/lib/vault/passphrase-ops.js +311 -0
- package/lib/vault/passphrase-source.js +198 -0
- package/lib/vault/rotate.js +761 -0
- package/lib/vault/wrap.js +289 -0
- package/lib/vendor/MANIFEST.json +84 -0
- package/lib/vendor/argon2/argon2.cjs +466 -0
- package/lib/vendor/argon2/argon2.d.cts +62 -0
- package/lib/vendor/argon2/package.json +1 -0
- package/lib/vendor/argon2/prebuilds/darwin-arm64/argon2.armv8.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/darwin-x64/argon2.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/freebsd-arm64/argon2.armv8.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/freebsd-x64/argon2.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-arm/argon2.armv7.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-arm/argon2.armv7.musl.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-arm64/argon2.armv8.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-arm64/argon2.armv8.musl.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-x64/argon2.glibc.node +0 -0
- package/lib/vendor/argon2/prebuilds/linux-x64/argon2.musl.node +0 -0
- package/lib/vendor/argon2/prebuilds/win32-x64/argon2.glibc.node +0 -0
- package/lib/vendor/noble-ciphers.cjs +9 -0
- package/lib/vendor/pki.cjs +181 -0
- package/lib/vendor/simplewebauthn-server.cjs +328 -0
- package/lib/webhook.js +632 -0
- package/lib/websocket-channels.js +413 -0
- package/lib/websocket.js +833 -0
- package/package.json +39 -0
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Google Cloud Storage (GCS) protocol adapter — native JSON API.
|
|
4
|
+
*
|
|
5
|
+
* Auth: service-account JSON (RSA-SHA256 signed JWT exchanged for an
|
|
6
|
+
* OAuth2 access token; tokens cached until ~5 min before expiry).
|
|
7
|
+
*
|
|
8
|
+
* Endpoint: https://storage.googleapis.com (override via config.endpoint
|
|
9
|
+
* for emulators / private endpoints).
|
|
10
|
+
*
|
|
11
|
+
* Config:
|
|
12
|
+
* {
|
|
13
|
+
* bucket: 'my-bucket' // required
|
|
14
|
+
* serviceAccount: { client_email, private_key, ... } // OR
|
|
15
|
+
* serviceAccountFile: '/path/to/sa.json' // OR
|
|
16
|
+
* scope: 'https://www.googleapis.com/auth/devstorage.read_write'
|
|
17
|
+
* endpoint: 'https://storage.googleapis.com'
|
|
18
|
+
* timeoutMs: C.TIME.seconds(30)
|
|
19
|
+
* }
|
|
20
|
+
*
|
|
21
|
+
* Reference:
|
|
22
|
+
* https://cloud.google.com/storage/docs/json_api/v1
|
|
23
|
+
* https://developers.google.com/identity/protocols/oauth2/service-account
|
|
24
|
+
*/
|
|
25
|
+
var fs = require("fs");
|
|
26
|
+
var nodeCrypto = require("crypto");
|
|
27
|
+
var { URL } = require("url");
|
|
28
|
+
var { Readable } = require("stream");
|
|
29
|
+
var safeJson = require("../safe-json");
|
|
30
|
+
var C = require("../constants");
|
|
31
|
+
var { ObjectStoreError } = require("../framework-error");
|
|
32
|
+
var httpClient = require("../http-client");
|
|
33
|
+
var safeUrl = require("../safe-url");
|
|
34
|
+
var authHeader = require("../auth-header");
|
|
35
|
+
var sigv4 = require("./sigv4");
|
|
36
|
+
|
|
37
|
+
// V4 query-string presigning. GCS uses the same canonical-request
|
|
38
|
+
// shape as AWS SigV4, differing only in:
|
|
39
|
+
// - algorithm: GOOG4-RSA-SHA256 (RSA service-account signing)
|
|
40
|
+
// - service: storage
|
|
41
|
+
// - region: auto (single global region for V4 presigning)
|
|
42
|
+
// - credential scope: {date}/{region}/storage/goog4_request
|
|
43
|
+
// - param prefix: X-Goog-* instead of X-Amz-*
|
|
44
|
+
// - credential value: client_email instead of accessKeyId
|
|
45
|
+
var GCS_V4_ALGORITHM = "GOOG4-RSA-SHA256";
|
|
46
|
+
var GCS_V4_SERVICE = "storage";
|
|
47
|
+
var GCS_V4_REGION = "auto";
|
|
48
|
+
|
|
49
|
+
// Same expiry bounds as SigV4 — V4 presigning shares the spec's 7-day cap.
|
|
50
|
+
var PRESIGN_DEFAULT_EXPIRES_SECONDS = 15 * 60;
|
|
51
|
+
var PRESIGN_MAX_EXPIRES_SECONDS = 7 * 24 * 60 * 60;
|
|
52
|
+
var PRESIGN_MIN_EXPIRES_SECONDS = 1;
|
|
53
|
+
|
|
54
|
+
var SERVICE_ACCOUNT_SCHEMA = {
|
|
55
|
+
type: "object",
|
|
56
|
+
required: ["client_email", "private_key"],
|
|
57
|
+
properties: {
|
|
58
|
+
client_email: { type: "string" },
|
|
59
|
+
private_key: { type: "string" },
|
|
60
|
+
token_uri: { type: "string" },
|
|
61
|
+
project_id: { type: "string" },
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
var DEFAULT_ENDPOINT = "https://storage.googleapis.com";
|
|
66
|
+
var TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token";
|
|
67
|
+
var DEFAULT_SCOPE = "https://www.googleapis.com/auth/devstorage.read_write";
|
|
68
|
+
var TOKEN_REFRESH_BUFFER = C.TIME.minutes(5); // refresh 5 min before expiry
|
|
69
|
+
|
|
70
|
+
var _err = ObjectStoreError.factory;
|
|
71
|
+
|
|
72
|
+
// ---- Generic HTTP helper (separate from sigv4's; no signing here) ----
|
|
73
|
+
|
|
74
|
+
function _httpRequest(method, urlObj, headers, body, opts) {
|
|
75
|
+
return httpClient.request({
|
|
76
|
+
method: method,
|
|
77
|
+
url: urlObj,
|
|
78
|
+
headers: headers,
|
|
79
|
+
body: body,
|
|
80
|
+
idleTimeoutMs: opts && opts.timeoutMs,
|
|
81
|
+
maxResponseBytes: opts && opts.maxResponseBytes,
|
|
82
|
+
errorClass: ObjectStoreError,
|
|
83
|
+
allowedProtocols: opts && opts.allowedProtocols,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// ---- JWT signing for service-account auth ----
|
|
88
|
+
|
|
89
|
+
function _base64UrlEncode(buf) {
|
|
90
|
+
return Buffer.from(buf).toString("base64")
|
|
91
|
+
.replace(/=+$/g, "")
|
|
92
|
+
.replace(/\+/g, "-")
|
|
93
|
+
.replace(/\//g, "_");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function _signJwt(serviceAccount, scope, audience) {
|
|
97
|
+
var nowSec = Math.floor(Date.now() / 1000);
|
|
98
|
+
var header = { alg: "RS256", typ: "JWT" };
|
|
99
|
+
var claim = {
|
|
100
|
+
iss: serviceAccount.client_email,
|
|
101
|
+
scope: scope,
|
|
102
|
+
aud: audience || TOKEN_ENDPOINT,
|
|
103
|
+
iat: nowSec,
|
|
104
|
+
exp: nowSec + 3600,
|
|
105
|
+
};
|
|
106
|
+
var headerB64 = _base64UrlEncode(JSON.stringify(header));
|
|
107
|
+
var claimB64 = _base64UrlEncode(JSON.stringify(claim));
|
|
108
|
+
var signingInput = headerB64 + "." + claimB64;
|
|
109
|
+
|
|
110
|
+
var signer = nodeCrypto.createSign("RSA-SHA256");
|
|
111
|
+
signer.update(signingInput);
|
|
112
|
+
signer.end();
|
|
113
|
+
var signature = signer.sign(serviceAccount.private_key);
|
|
114
|
+
return signingInput + "." + _base64UrlEncode(signature);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ---- Public adapter factory ----
|
|
118
|
+
|
|
119
|
+
function create(config) {
|
|
120
|
+
if (!config) throw new Error("gcs protocol requires config");
|
|
121
|
+
if (!config.bucket) throw new Error("gcs: bucket is required");
|
|
122
|
+
|
|
123
|
+
var serviceAccount = config.serviceAccount;
|
|
124
|
+
if (!serviceAccount && config.serviceAccountFile) {
|
|
125
|
+
try {
|
|
126
|
+
serviceAccount = safeJson.parse(fs.readFileSync(config.serviceAccountFile), { schema: SERVICE_ACCOUNT_SCHEMA });
|
|
127
|
+
} catch (e) {
|
|
128
|
+
throw new Error("gcs: failed to read serviceAccountFile '" + config.serviceAccountFile + "': " + e.message);
|
|
129
|
+
}
|
|
130
|
+
} else if (serviceAccount && (!serviceAccount.client_email || !serviceAccount.private_key)) {
|
|
131
|
+
throw new Error("gcs: serviceAccount with { client_email, private_key } is required (or serviceAccountFile pointing to one)");
|
|
132
|
+
}
|
|
133
|
+
if (!serviceAccount) {
|
|
134
|
+
throw new Error("gcs: serviceAccount with { client_email, private_key } is required (or serviceAccountFile pointing to one)");
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
var endpoint = config.endpoint || DEFAULT_ENDPOINT;
|
|
138
|
+
if (endpoint.endsWith("/")) endpoint = endpoint.slice(0, -1);
|
|
139
|
+
var tokenEndpoint = config.tokenEndpoint || TOKEN_ENDPOINT;
|
|
140
|
+
var bucket = config.bucket;
|
|
141
|
+
var scope = config.scope || DEFAULT_SCOPE;
|
|
142
|
+
var timeoutMs = config.timeoutMs;
|
|
143
|
+
// HTTPS-only by default — google APIs are always HTTPS. Operators with
|
|
144
|
+
// an emulator / private fake-GCS endpoint opt in via config.allowedProtocols.
|
|
145
|
+
var allowedProtocols = config.allowedProtocols || safeUrl.ALLOW_HTTP_TLS;
|
|
146
|
+
safeUrl.parse(endpoint, { allowedProtocols: allowedProtocols, errorClass: ObjectStoreError });
|
|
147
|
+
safeUrl.parse(tokenEndpoint, { allowedProtocols: allowedProtocols, errorClass: ObjectStoreError });
|
|
148
|
+
var reqOpts = { timeoutMs: timeoutMs, allowedProtocols: allowedProtocols };
|
|
149
|
+
|
|
150
|
+
// ---- Token cache ----
|
|
151
|
+
var cachedToken = null; // { accessToken, expiresAt }
|
|
152
|
+
|
|
153
|
+
async function _ensureToken() {
|
|
154
|
+
if (cachedToken && Date.now() < cachedToken.expiresAt - TOKEN_REFRESH_BUFFER) {
|
|
155
|
+
return cachedToken.accessToken;
|
|
156
|
+
}
|
|
157
|
+
var assertion = _signJwt(serviceAccount, scope, tokenEndpoint);
|
|
158
|
+
var bodyStr = "grant_type=" + encodeURIComponent("urn:ietf:params:oauth:grant-type:jwt-bearer") +
|
|
159
|
+
"&assertion=" + encodeURIComponent(assertion);
|
|
160
|
+
var bodyBuf = Buffer.from(bodyStr, "utf8");
|
|
161
|
+
var res = await _httpRequest(
|
|
162
|
+
"POST",
|
|
163
|
+
new URL(tokenEndpoint),
|
|
164
|
+
{
|
|
165
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
166
|
+
"Content-Length": String(bodyBuf.length),
|
|
167
|
+
},
|
|
168
|
+
bodyBuf,
|
|
169
|
+
reqOpts
|
|
170
|
+
);
|
|
171
|
+
var tokenResp = safeJson.parse(res.body);
|
|
172
|
+
if (!tokenResp.access_token) {
|
|
173
|
+
throw _err("AUTH_FAILED", "GCS token endpoint returned no access_token: " + res.body.toString("utf8"), true);
|
|
174
|
+
}
|
|
175
|
+
var expiresInMs = (tokenResp.expires_in || 3600) * 1000;
|
|
176
|
+
cachedToken = {
|
|
177
|
+
accessToken: tokenResp.access_token,
|
|
178
|
+
expiresAt: Date.now() + expiresInMs,
|
|
179
|
+
};
|
|
180
|
+
return cachedToken.accessToken;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function _objectUrl(key, params) {
|
|
184
|
+
var u = new URL(endpoint + "/storage/v1/b/" + encodeURIComponent(bucket) +
|
|
185
|
+
"/o/" + encodeURIComponent(key));
|
|
186
|
+
if (params) {
|
|
187
|
+
Object.keys(params).forEach(function (k) { u.searchParams.set(k, params[k]); });
|
|
188
|
+
}
|
|
189
|
+
return u;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function _uploadUrl(key) {
|
|
193
|
+
var u = new URL(endpoint + "/upload/storage/v1/b/" + encodeURIComponent(bucket) + "/o");
|
|
194
|
+
u.searchParams.set("uploadType", "media");
|
|
195
|
+
u.searchParams.set("name", key);
|
|
196
|
+
return u;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function _listUrl(prefix, opts) {
|
|
200
|
+
var u = new URL(endpoint + "/storage/v1/b/" + encodeURIComponent(bucket) + "/o");
|
|
201
|
+
if (prefix) u.searchParams.set("prefix", prefix);
|
|
202
|
+
if (opts && opts.maxResults) u.searchParams.set("maxResults", String(opts.maxResults));
|
|
203
|
+
if (opts && opts.pageToken) u.searchParams.set("pageToken", opts.pageToken);
|
|
204
|
+
return u;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// ---- Operations ----
|
|
208
|
+
|
|
209
|
+
async function put(key, body, opts) {
|
|
210
|
+
var token = await _ensureToken();
|
|
211
|
+
var url = _uploadUrl(key);
|
|
212
|
+
var buf = Buffer.isBuffer(body) ? body : Buffer.from(typeof body === "string" ? body : "", "utf8");
|
|
213
|
+
var contentType = (opts && opts.contentType) || "application/octet-stream";
|
|
214
|
+
var res = await _httpRequest("POST", url, Object.assign(
|
|
215
|
+
authHeader.bearer(token),
|
|
216
|
+
{ "Content-Type": contentType,
|
|
217
|
+
"Content-Length": String(buf.length) }
|
|
218
|
+
), buf, reqOpts);
|
|
219
|
+
var meta = safeJson.parse(res.body);
|
|
220
|
+
return { size: parseInt(meta.size || buf.length, 10), etag: meta.etag };
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async function get(key) {
|
|
224
|
+
var token = await _ensureToken();
|
|
225
|
+
var url = _objectUrl(key, { alt: "media" });
|
|
226
|
+
var res = await _httpRequest("GET", url, authHeader.bearer(token), null, reqOpts);
|
|
227
|
+
return res.body;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function getStream(key) {
|
|
231
|
+
return Readable.from(get(key));
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function head(key) {
|
|
235
|
+
var token = await _ensureToken();
|
|
236
|
+
var url = _objectUrl(key);
|
|
237
|
+
var res = await _httpRequest("GET", url, authHeader.bearer(token), null, reqOpts);
|
|
238
|
+
var meta = safeJson.parse(res.body);
|
|
239
|
+
return {
|
|
240
|
+
size: parseInt(meta.size, 10),
|
|
241
|
+
etag: meta.etag,
|
|
242
|
+
lastModified: meta.updated ? Date.parse(meta.updated) : null,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
async function deleteKey(key) {
|
|
247
|
+
var token = await _ensureToken();
|
|
248
|
+
var url = _objectUrl(key);
|
|
249
|
+
try {
|
|
250
|
+
await _httpRequest("DELETE", url, authHeader.bearer(token), null, reqOpts);
|
|
251
|
+
return true;
|
|
252
|
+
} catch (e) {
|
|
253
|
+
if (e.statusCode === 404) return false;
|
|
254
|
+
throw e;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async function list(prefix, opts) {
|
|
259
|
+
opts = opts || {};
|
|
260
|
+
var token = await _ensureToken();
|
|
261
|
+
var url = _listUrl(prefix, { maxResults: opts.maxResults, pageToken: opts.continuationToken });
|
|
262
|
+
var res = await _httpRequest("GET", url, authHeader.bearer(token), null, reqOpts);
|
|
263
|
+
var listResp = safeJson.parse(res.body);
|
|
264
|
+
var items = (listResp.items || []).map(function (item) {
|
|
265
|
+
return {
|
|
266
|
+
key: item.name,
|
|
267
|
+
size: parseInt(item.size, 10),
|
|
268
|
+
lastModified: item.updated ? Date.parse(item.updated) : null,
|
|
269
|
+
};
|
|
270
|
+
});
|
|
271
|
+
return {
|
|
272
|
+
items: items,
|
|
273
|
+
truncated: !!listResp.nextPageToken,
|
|
274
|
+
continuationToken: listResp.nextPageToken || null,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function _v4Presign(method, opts) {
|
|
279
|
+
opts = opts || {};
|
|
280
|
+
if (!opts.key || typeof opts.key !== "string") {
|
|
281
|
+
throw _err("INVALID_KEY", "presigned URL: key is required", true);
|
|
282
|
+
}
|
|
283
|
+
if (opts.key.indexOf("\0") !== -1) {
|
|
284
|
+
throw _err("INVALID_KEY", "null byte in key", true);
|
|
285
|
+
}
|
|
286
|
+
var expiresIn = opts.expiresIn != null ? opts.expiresIn : PRESIGN_DEFAULT_EXPIRES_SECONDS;
|
|
287
|
+
if (typeof expiresIn !== "number" ||
|
|
288
|
+
expiresIn < PRESIGN_MIN_EXPIRES_SECONDS ||
|
|
289
|
+
expiresIn > PRESIGN_MAX_EXPIRES_SECONDS) {
|
|
290
|
+
throw _err("INVALID_EXPIRES",
|
|
291
|
+
"presigned URL: expiresIn must be a number of seconds between " +
|
|
292
|
+
PRESIGN_MIN_EXPIRES_SECONDS + " and " + PRESIGN_MAX_EXPIRES_SECONDS +
|
|
293
|
+
" (7 days, V4 hard cap)", true);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// V4 presigned URLs target the XML API path-style endpoint
|
|
297
|
+
// (storage.googleapis.com/{bucket}/{key}), not the JSON API used
|
|
298
|
+
// for the in-process put/get methods. The XML endpoint is what
|
|
299
|
+
// honors the V4 signature.
|
|
300
|
+
var encodedKey = opts.key.split("/")
|
|
301
|
+
.map(function (s) { return sigv4.awsUriEncode(s, true); })
|
|
302
|
+
.join("/");
|
|
303
|
+
var url = new URL(endpoint + "/" + encodeURIComponent(bucket) + "/" + encodedKey);
|
|
304
|
+
|
|
305
|
+
var date = opts.date || new Date();
|
|
306
|
+
// Reuse SigV4's date formatters — the wire format is identical.
|
|
307
|
+
var iso = date.toISOString().replace(/[-:]/g, "");
|
|
308
|
+
var amzDate = iso.slice(0, 8) + "T" + iso.slice(9, 15) + "Z";
|
|
309
|
+
var dateStamp = date.toISOString().slice(0, 10).replace(/-/g, "");
|
|
310
|
+
var credentialScope = dateStamp + "/" + GCS_V4_REGION + "/" +
|
|
311
|
+
GCS_V4_SERVICE + "/goog4_request";
|
|
312
|
+
|
|
313
|
+
var headers = { host: url.host };
|
|
314
|
+
if (opts.contentType) headers["content-type"] = opts.contentType;
|
|
315
|
+
var signedHeadersStr = Object.keys(headers).sort().join(";");
|
|
316
|
+
|
|
317
|
+
url.searchParams.set("X-Goog-Algorithm", GCS_V4_ALGORITHM);
|
|
318
|
+
url.searchParams.set("X-Goog-Credential", serviceAccount.client_email + "/" + credentialScope);
|
|
319
|
+
url.searchParams.set("X-Goog-Date", amzDate);
|
|
320
|
+
url.searchParams.set("X-Goog-Expires", String(expiresIn));
|
|
321
|
+
url.searchParams.set("X-Goog-SignedHeaders", signedHeadersStr);
|
|
322
|
+
|
|
323
|
+
// Build the canonical request — identical shape to SigV4. The
|
|
324
|
+
// sigv4 export gives us the formatter so this stays in lockstep
|
|
325
|
+
// with the AWS implementation.
|
|
326
|
+
var canon = sigv4.canonicalRequest(method, url, headers, "UNSIGNED-PAYLOAD");
|
|
327
|
+
var stringToSign = [
|
|
328
|
+
GCS_V4_ALGORITHM,
|
|
329
|
+
amzDate,
|
|
330
|
+
credentialScope,
|
|
331
|
+
sigv4.sha256Hex(canon),
|
|
332
|
+
].join("\n");
|
|
333
|
+
|
|
334
|
+
// GCS V4 signs the stringToSign with the service-account's RSA
|
|
335
|
+
// private key (RSA-SHA256). Hex-encoded signature goes back into
|
|
336
|
+
// the URL.
|
|
337
|
+
var signer = nodeCrypto.createSign("RSA-SHA256");
|
|
338
|
+
signer.update(stringToSign);
|
|
339
|
+
signer.end();
|
|
340
|
+
var signature = signer.sign(serviceAccount.private_key).toString("hex");
|
|
341
|
+
|
|
342
|
+
url.searchParams.set("X-Goog-Signature", signature);
|
|
343
|
+
|
|
344
|
+
var clientHeaders = {};
|
|
345
|
+
if (opts.contentType) clientHeaders["Content-Type"] = opts.contentType;
|
|
346
|
+
|
|
347
|
+
return {
|
|
348
|
+
url: url.toString(),
|
|
349
|
+
method: method,
|
|
350
|
+
headers: clientHeaders,
|
|
351
|
+
expiresAt: date.getTime() + C.TIME.seconds(expiresIn),
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function presignedUploadUrl(opts) { return _v4Presign("PUT", opts); }
|
|
356
|
+
function presignedDownloadUrl(opts) { return _v4Presign("GET", opts); }
|
|
357
|
+
|
|
358
|
+
// GCS V4 POST policy presigning. Same shape as SigV4 POST policy
|
|
359
|
+
// (operator-uploaded multipart/form-data with policy + signature
|
|
360
|
+
// fields), but signed with the service account's RSA private key
|
|
361
|
+
// instead of an HMAC chain. The content-length-range condition is
|
|
362
|
+
// server-side enforced by GCS — clients sending bodies outside the
|
|
363
|
+
// declared range get a 403 from the bucket itself.
|
|
364
|
+
function presignedUploadPolicy(opts) {
|
|
365
|
+
opts = opts || {};
|
|
366
|
+
if (!opts.key || typeof opts.key !== "string") {
|
|
367
|
+
throw _err("INVALID_KEY", "presignedUploadPolicy: key is required", true);
|
|
368
|
+
}
|
|
369
|
+
if (opts.key.indexOf("\0") !== -1) {
|
|
370
|
+
throw _err("INVALID_KEY", "null byte in key", true);
|
|
371
|
+
}
|
|
372
|
+
if (typeof opts.maxBytes !== "number" || !Number.isFinite(opts.maxBytes) ||
|
|
373
|
+
opts.maxBytes <= 0) {
|
|
374
|
+
throw _err("INVALID_MAX_BYTES",
|
|
375
|
+
"presignedUploadPolicy: maxBytes (positive number of bytes) is required — " +
|
|
376
|
+
"POST-form policy enforces body size via the content-length-range condition; " +
|
|
377
|
+
"use presignedUploadUrl if size enforcement is not needed", true);
|
|
378
|
+
}
|
|
379
|
+
var minBytes = (typeof opts.minBytes === "number" && opts.minBytes >= 0) ? opts.minBytes : 0;
|
|
380
|
+
var expiresIn = opts.expiresIn != null ? opts.expiresIn : PRESIGN_DEFAULT_EXPIRES_SECONDS;
|
|
381
|
+
if (typeof expiresIn !== "number" ||
|
|
382
|
+
expiresIn < PRESIGN_MIN_EXPIRES_SECONDS ||
|
|
383
|
+
expiresIn > PRESIGN_MAX_EXPIRES_SECONDS) {
|
|
384
|
+
throw _err("INVALID_EXPIRES",
|
|
385
|
+
"presignedUploadPolicy: expiresIn must be a number of seconds between " +
|
|
386
|
+
PRESIGN_MIN_EXPIRES_SECONDS + " and " + PRESIGN_MAX_EXPIRES_SECONDS +
|
|
387
|
+
" (7 days, V4 hard cap)", true);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
var date = opts.date || new Date();
|
|
391
|
+
var iso = date.toISOString().replace(/[-:]/g, "");
|
|
392
|
+
var amzDate = iso.slice(0, 8) + "T" + iso.slice(9, 15) + "Z";
|
|
393
|
+
var dateStamp = date.toISOString().slice(0, 10).replace(/-/g, "");
|
|
394
|
+
var credentialScope = dateStamp + "/" + GCS_V4_REGION + "/" +
|
|
395
|
+
GCS_V4_SERVICE + "/goog4_request";
|
|
396
|
+
var credential = serviceAccount.client_email + "/" + credentialScope;
|
|
397
|
+
var expirationIso = new Date(date.getTime() + C.TIME.seconds(expiresIn)).toISOString();
|
|
398
|
+
|
|
399
|
+
var conditions = [
|
|
400
|
+
{ "bucket": bucket },
|
|
401
|
+
{ "key": opts.key },
|
|
402
|
+
{ "x-goog-algorithm": GCS_V4_ALGORITHM },
|
|
403
|
+
{ "x-goog-credential": credential },
|
|
404
|
+
{ "x-goog-date": amzDate },
|
|
405
|
+
["content-length-range", minBytes, opts.maxBytes],
|
|
406
|
+
];
|
|
407
|
+
if (opts.contentType) {
|
|
408
|
+
conditions.push({ "content-type": opts.contentType });
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
var policy = { expiration: expirationIso, conditions: conditions };
|
|
412
|
+
var policyJson = JSON.stringify(policy);
|
|
413
|
+
var policyB64 = Buffer.from(policyJson, "utf8").toString("base64");
|
|
414
|
+
|
|
415
|
+
// GCS V4: signature is hex(RSA-SHA256(privateKey, policyB64)).
|
|
416
|
+
var signer = nodeCrypto.createSign("RSA-SHA256");
|
|
417
|
+
signer.update(policyB64);
|
|
418
|
+
signer.end();
|
|
419
|
+
var signature = signer.sign(serviceAccount.private_key).toString("hex");
|
|
420
|
+
|
|
421
|
+
var fields = {
|
|
422
|
+
"key": opts.key,
|
|
423
|
+
"x-goog-algorithm": GCS_V4_ALGORITHM,
|
|
424
|
+
"x-goog-credential": credential,
|
|
425
|
+
"x-goog-date": amzDate,
|
|
426
|
+
"policy": policyB64,
|
|
427
|
+
"x-goog-signature": signature,
|
|
428
|
+
};
|
|
429
|
+
if (opts.contentType) fields["content-type"] = opts.contentType;
|
|
430
|
+
|
|
431
|
+
var url = new URL(endpoint + "/" + encodeURIComponent(bucket) + "/");
|
|
432
|
+
|
|
433
|
+
return {
|
|
434
|
+
url: url.toString(),
|
|
435
|
+
method: "POST",
|
|
436
|
+
fields: fields,
|
|
437
|
+
expiresAt: date.getTime() + C.TIME.seconds(expiresIn),
|
|
438
|
+
maxBytes: opts.maxBytes,
|
|
439
|
+
enforcement: "content-length-range",
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
return {
|
|
444
|
+
protocol: "gcs",
|
|
445
|
+
endpoint: endpoint,
|
|
446
|
+
bucket: bucket,
|
|
447
|
+
put: put,
|
|
448
|
+
get: get,
|
|
449
|
+
getStream: getStream,
|
|
450
|
+
head: head,
|
|
451
|
+
delete: deleteKey,
|
|
452
|
+
list: list,
|
|
453
|
+
presignedUploadUrl: presignedUploadUrl,
|
|
454
|
+
presignedDownloadUrl: presignedDownloadUrl,
|
|
455
|
+
presignedUploadPolicy: presignedUploadPolicy,
|
|
456
|
+
// Internal accessors for tests
|
|
457
|
+
_ensureToken: _ensureToken,
|
|
458
|
+
_signJwt: function () { return _signJwt(serviceAccount, scope, TOKEN_ENDPOINT); },
|
|
459
|
+
};
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
module.exports = {
|
|
463
|
+
create: create,
|
|
464
|
+
_signJwt: _signJwt,
|
|
465
|
+
_base64UrlEncode: _base64UrlEncode,
|
|
466
|
+
DEFAULT_ENDPOINT: DEFAULT_ENDPOINT,
|
|
467
|
+
TOKEN_ENDPOINT: TOKEN_ENDPOINT,
|
|
468
|
+
DEFAULT_SCOPE: DEFAULT_SCOPE,
|
|
469
|
+
};
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Generic HTTP PUT/GET protocol adapter.
|
|
4
|
+
*
|
|
5
|
+
* For backends that aren't S3-compatible but expose a simple HTTP key-value
|
|
6
|
+
* surface: internal stores, NAS appliances with HTTP frontends, or custom
|
|
7
|
+
* services. Uses Node's native https / http; PQC TLS group preference is
|
|
8
|
+
* applied to outbound connections via constants.TLS_GROUP_CURVE_STR.
|
|
9
|
+
*
|
|
10
|
+
* Authentication options:
|
|
11
|
+
* { auth: 'none' } (no Authorization header)
|
|
12
|
+
* { auth: 'bearer', token: '...' } (Authorization: Bearer ...)
|
|
13
|
+
* { auth: 'basic', username: '...', password: '...' }
|
|
14
|
+
* { auth: 'header', headers: { 'X-Auth-Key': '...' } } (arbitrary headers)
|
|
15
|
+
*
|
|
16
|
+
* Errors are surfaced as object-store errors with statusCode set so the
|
|
17
|
+
* retry layer can classify retryable vs permanent.
|
|
18
|
+
*/
|
|
19
|
+
var { Readable } = require("stream");
|
|
20
|
+
var { ObjectStoreError } = require("../framework-error");
|
|
21
|
+
var httpClient = require("../http-client");
|
|
22
|
+
var safeUrl = require("../safe-url");
|
|
23
|
+
var authHeader = require("../auth-header");
|
|
24
|
+
|
|
25
|
+
var _err = ObjectStoreError.factory;
|
|
26
|
+
|
|
27
|
+
// Auth-header construction is delegated to lib/auth-header for the
|
|
28
|
+
// none/bearer/basic triple. The "header" mode (pass-through arbitrary
|
|
29
|
+
// headers) is just header merging; kept here because it's documented as
|
|
30
|
+
// a value of the same config.auth knob.
|
|
31
|
+
function _authHeaders(authConfig) {
|
|
32
|
+
if (authConfig && authConfig.auth === "header") {
|
|
33
|
+
return Object.assign({}, authConfig.headers || {});
|
|
34
|
+
}
|
|
35
|
+
return authHeader.fromConfig(authConfig);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function _request(method, url, body, headers, opts) {
|
|
39
|
+
return httpClient.request({
|
|
40
|
+
method: method,
|
|
41
|
+
url: url,
|
|
42
|
+
headers: headers,
|
|
43
|
+
body: body,
|
|
44
|
+
idleTimeoutMs: opts && opts.timeoutMs,
|
|
45
|
+
errorClass: ObjectStoreError,
|
|
46
|
+
allowedProtocols: opts && opts.allowedProtocols,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function _keyToUrl(baseUrl, key) {
|
|
51
|
+
// Append key to baseUrl, ensuring exactly one slash between them.
|
|
52
|
+
// Disallow ../ to prevent path traversal at the URL level.
|
|
53
|
+
if (key.includes("..") || key.includes("\0")) {
|
|
54
|
+
throw _err("INVALID_KEY", "invalid characters in key", true);
|
|
55
|
+
}
|
|
56
|
+
var b = baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl;
|
|
57
|
+
var k = key.startsWith("/") ? key.slice(1) : key;
|
|
58
|
+
return b + "/" + k;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function create(config) {
|
|
62
|
+
if (!config || !config.baseUrl) {
|
|
63
|
+
throw new Error("http-put protocol requires { baseUrl }");
|
|
64
|
+
}
|
|
65
|
+
var baseUrl = config.baseUrl;
|
|
66
|
+
// Fail fast on misconfigured baseUrl — validate scheme + shape now, not
|
|
67
|
+
// at first put(). HTTPS-only by default; cleartext appliances opt in
|
|
68
|
+
// via config.allowedProtocols (safeUrl.ALLOW_HTTP_ALL).
|
|
69
|
+
var allowedProtocols = config.allowedProtocols || safeUrl.ALLOW_HTTP_TLS;
|
|
70
|
+
safeUrl.parse(baseUrl, {
|
|
71
|
+
allowedProtocols: allowedProtocols,
|
|
72
|
+
errorClass: ObjectStoreError,
|
|
73
|
+
});
|
|
74
|
+
var headers = _authHeaders(config);
|
|
75
|
+
var timeoutMs = config.timeoutMs;
|
|
76
|
+
var reqOpts = { timeoutMs: timeoutMs, allowedProtocols: allowedProtocols };
|
|
77
|
+
|
|
78
|
+
function put(key, body, _opts) {
|
|
79
|
+
var url = _keyToUrl(baseUrl, key);
|
|
80
|
+
var h = Object.assign({ "Content-Type": "application/octet-stream" }, headers);
|
|
81
|
+
return _request("PUT", url, body, h, reqOpts).then(function (res) {
|
|
82
|
+
var size = Buffer.isBuffer(body) ? body.length : null;
|
|
83
|
+
return { size: size, etag: res.headers.etag };
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function get(key) {
|
|
88
|
+
var url = _keyToUrl(baseUrl, key);
|
|
89
|
+
return _request("GET", url, null, headers, reqOpts).then(function (res) {
|
|
90
|
+
return res.body;
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function getStream(key) {
|
|
95
|
+
// Generic HTTP PUT has no streaming ergonomic path through Node's
|
|
96
|
+
// https that doesn't buffer; return a Readable wrapping the buffered
|
|
97
|
+
// body. Backends that support chunked transfer (e.g. SigV4) implement
|
|
98
|
+
// a real streaming getStream in their own adapter.
|
|
99
|
+
return Readable.from(get(key));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function head(key) {
|
|
103
|
+
var url = _keyToUrl(baseUrl, key);
|
|
104
|
+
return _request("HEAD", url, null, headers, reqOpts).then(function (res) {
|
|
105
|
+
return {
|
|
106
|
+
size: res.headers["content-length"] ? parseInt(res.headers["content-length"], 10) : null,
|
|
107
|
+
etag: res.headers.etag,
|
|
108
|
+
lastModified: res.headers["last-modified"] ? Date.parse(res.headers["last-modified"]) : null,
|
|
109
|
+
};
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function deleteKey(key) {
|
|
114
|
+
var url = _keyToUrl(baseUrl, key);
|
|
115
|
+
return _request("DELETE", url, null, headers, reqOpts).then(
|
|
116
|
+
function () { return true; },
|
|
117
|
+
function (e) {
|
|
118
|
+
if (e.statusCode === 404) return false;
|
|
119
|
+
throw e;
|
|
120
|
+
}
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function list(_prefix, _opts) {
|
|
125
|
+
// Generic HTTP PUT protocol has no listing convention. SigV4 will.
|
|
126
|
+
return Promise.reject(_err("NOT_SUPPORTED", "list() not supported by http-put protocol", true));
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function _presignNotSupported(_opts) {
|
|
130
|
+
// Generic HTTP PUT has no signing convention; the receiver is whatever
|
|
131
|
+
// endpoint the operator points at, with whatever auth it chose. Use
|
|
132
|
+
// protocol: 'sigv4' for an S3-compatible presigning workflow.
|
|
133
|
+
throw _err("PRESIGN_NOT_SUPPORTED",
|
|
134
|
+
"http-put backend does not support presigned URLs — switch to " +
|
|
135
|
+
"protocol: 'sigv4' for an S3-compatible signing flow", true);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
protocol: "http-put",
|
|
140
|
+
baseUrl: baseUrl,
|
|
141
|
+
put: put,
|
|
142
|
+
get: get,
|
|
143
|
+
getStream: getStream,
|
|
144
|
+
head: head,
|
|
145
|
+
delete: deleteKey,
|
|
146
|
+
list: list,
|
|
147
|
+
presignedUploadUrl: _presignNotSupported,
|
|
148
|
+
presignedDownloadUrl: _presignNotSupported,
|
|
149
|
+
presignedUploadPolicy: _presignNotSupported,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
module.exports = { create: create };
|