@cytario/web 2.1.7 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/app/.server/auth/README.md +0 -1
- package/app/.server/auth/authMiddleware.ts +6 -28
- package/app/.server/auth/getSessionCredentials.ts +11 -1
- package/app/.server/auth/sessionPolicy.ts +69 -0
- package/app/.server/corsPreflight.ts +136 -0
- package/app/.server/csp.ts +34 -0
- package/app/components/.client/ImageViewer/README.md +2 -2
- package/app/components/DirectoryView/DirectoryViewGrid.tsx +2 -11
- package/app/components/DirectoryView/DirectoryViewTree.tsx +18 -26
- package/app/components/DirectoryView/buildDirectoryTree.ts +89 -42
- package/app/components/DirectoryView/filterNodes.ts +4 -19
- package/app/components/DirectoryView/useLazyTreeNodes.ts +120 -0
- package/app/components/GlobalSearch/GlobalSearch.tsx +8 -3
- package/app/components/GlobalSearch/Suggestions.tsx +21 -3
- package/app/config.ts +0 -7
- package/app/entry.server.tsx +9 -4
- package/app/hooks/useInitConnections.ts +3 -3
- package/app/root.tsx +11 -6
- package/app/routes/connections/connection.form.tsx +9 -3
- package/app/routes/connections/connection.schema.ts +60 -11
- package/app/routes/connections/connections.clientLoader.ts +91 -0
- package/app/routes/connections/connections.loader.ts +19 -39
- package/app/routes/connections/connections.route.tsx +5 -0
- package/app/routes/connections/createConnection.action.ts +39 -13
- package/app/routes/connections/deleteConnection.action.ts +5 -1
- package/app/routes/connections/updateConnection.action.ts +50 -13
- package/app/routes/home/home.route.tsx +5 -1
- package/app/routes/layouts/protected.layout.tsx +15 -1
- package/app/routes/objects/objects.clientLoader.ts +73 -0
- package/app/routes/objects/objects.loader.ts +58 -92
- package/app/routes/objects/objects.route.tsx +41 -40
- package/app/routes/search.route.tsx +133 -28
- package/app/routes.ts +0 -4
- package/app/utils/connectionsStore/useConnectionsStore.ts +25 -62
- package/app/utils/credentialsRefresh.ts +53 -0
- package/app/utils/db/convertCsvToParquet.ts +24 -16
- package/app/utils/db/createDatabase.ts +25 -33
- package/app/utils/db/duckdbBundles.ts +42 -0
- package/app/utils/db/ensureSpatialLoaded.ts +28 -0
- package/app/utils/db/escapeSqlString.ts +4 -0
- package/app/utils/db/getBlobFromObjectNode.ts +15 -36
- package/app/utils/db/getTileDataWasm.ts +4 -5
- package/app/utils/db/sqlQueries.ts +6 -1
- package/app/utils/filterObjects.ts +2 -18
- package/app/utils/limitConcurrency.ts +28 -0
- package/app/utils/listObjectsClient.ts +318 -0
- package/app/utils/listingLimits.ts +7 -0
- package/app/utils/loadConnectionLevel.ts +50 -0
- package/app/utils/localFilesStore/useFileStore.ts +1 -6
- package/app/utils/pathUtils.ts +65 -0
- package/app/utils/resourceId.ts +14 -29
- package/app/utils/s3HostAllowlist.ts +116 -0
- package/app/utils/signedFetch.ts +159 -24
- package/package.json +2 -2
- package/prisma/seed.ts +3 -3
- package/public/duckdb-extensions/checksums.json +12 -0
- package/scripts/download-duckdb-extensions.mjs +188 -0
- package/scripts/prebuild.mjs +18 -10
- package/app/.server/auth/getPresignedUrl.ts +0 -21
- package/app/.server/auth/getS3Client.ts +0 -93
- package/app/routes/presign.route.tsx +0 -42
- package/app/utils/getObjects.ts +0 -24
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
import { Sha256 } from "@aws-crypto/sha256-browser";
|
|
2
|
+
import type { _Object } from "@aws-sdk/client-s3";
|
|
3
|
+
import type { Credentials } from "@aws-sdk/client-sts";
|
|
4
|
+
import { SignatureV4 } from "@smithy/signature-v4";
|
|
5
|
+
|
|
6
|
+
import { ExpiredCredentialsError, requestCredentialsRefresh } from "./credentialsRefresh";
|
|
7
|
+
import { filterObjects } from "./filterObjects";
|
|
8
|
+
import { DEFAULT_MAX_TOTAL } from "./listingLimits";
|
|
9
|
+
import { constructS3Url } from "./resourceId";
|
|
10
|
+
import { CorsLikelyError } from "./signedFetch";
|
|
11
|
+
import type { ConnectionConfig } from "~/.generated/client";
|
|
12
|
+
|
|
13
|
+
const DEFAULT_PAGE_SIZE = 1000;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* RFC 3986 strict percent-encoding — matches the canonical request SigV4
|
|
17
|
+
* builds. `URLSearchParams` form-encoding instead breaks signatures on any
|
|
18
|
+
* continuation token containing `+` (base64).
|
|
19
|
+
*/
|
|
20
|
+
function rfc3986Encode(value: string): string {
|
|
21
|
+
return encodeURIComponent(value).replace(
|
|
22
|
+
/[!*'()]/g,
|
|
23
|
+
(c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`,
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Serialise a query map using the same RFC-3986 encoding the signer applies
|
|
29
|
+
* canonically, so the wire URL byte-matches what was signed.
|
|
30
|
+
*/
|
|
31
|
+
function encodeWireQuery(query: Record<string, string>): string {
|
|
32
|
+
return Object.entries(query)
|
|
33
|
+
.map(([key, value]) => `${rfc3986Encode(key)}=${rfc3986Encode(value)}`)
|
|
34
|
+
.join("&");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface ListObjectsClientOptions {
|
|
38
|
+
query?: string | null;
|
|
39
|
+
prefix?: string;
|
|
40
|
+
/** When true, omit Delimiter (recursive flat listing). Default: false (one-level via Delimiter "/"). */
|
|
41
|
+
recursive?: boolean;
|
|
42
|
+
/** S3 page size. Default: 1000 (S3 max). */
|
|
43
|
+
maxKeys?: number;
|
|
44
|
+
/** Hard cap on total entries (contents + commonPrefixes) collected across pages. Default: 10000. */
|
|
45
|
+
maxTotal?: number;
|
|
46
|
+
/**
|
|
47
|
+
* Short-circuits pagination as soon as any object in a fetched page
|
|
48
|
+
* satisfies the predicate. Useful for "first match wins" scans.
|
|
49
|
+
*/
|
|
50
|
+
findFirst?: (obj: _Object) => boolean;
|
|
51
|
+
/** Aborts the in-flight pagination loop. */
|
|
52
|
+
signal?: AbortSignal;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface ListObjectsClientResult {
|
|
56
|
+
contents: _Object[];
|
|
57
|
+
commonPrefixes: string[];
|
|
58
|
+
isCapped: boolean;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function getElText(parent: Element, tag: string): string | undefined {
|
|
62
|
+
const el = parent.getElementsByTagName(tag)[0];
|
|
63
|
+
return el?.textContent ?? undefined;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function parseListResponse(xml: string): {
|
|
67
|
+
contents: _Object[];
|
|
68
|
+
commonPrefixes: string[];
|
|
69
|
+
isTruncated: boolean;
|
|
70
|
+
nextContinuationToken: string | undefined;
|
|
71
|
+
} {
|
|
72
|
+
const doc = new DOMParser().parseFromString(xml, "text/xml");
|
|
73
|
+
const root = doc.documentElement;
|
|
74
|
+
if (!root) {
|
|
75
|
+
throw new Error("Failed to parse ListBucketResult XML");
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Firefox injects `<parsererror>` as a child rather than replacing the root,
|
|
79
|
+
// so the nodeName check is insufficient.
|
|
80
|
+
if (doc.getElementsByTagName("parsererror").length > 0) {
|
|
81
|
+
throw new Error("Failed to parse ListBucketResult XML");
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (root.nodeName !== "ListBucketResult") {
|
|
85
|
+
const code = doc.getElementsByTagName("Code")[0]?.textContent;
|
|
86
|
+
const message = doc.getElementsByTagName("Message")[0]?.textContent;
|
|
87
|
+
throw new Error(`S3 error: ${code ?? "Unknown"} ${message ?? ""}`.trimEnd());
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const contents: _Object[] = Array.from(root.getElementsByTagName("Contents")).map((node) => {
|
|
91
|
+
const sizeText = getElText(node, "Size");
|
|
92
|
+
const lastModified = getElText(node, "LastModified");
|
|
93
|
+
return {
|
|
94
|
+
Key: getElText(node, "Key") ?? "",
|
|
95
|
+
LastModified: lastModified ? new Date(lastModified) : undefined,
|
|
96
|
+
ETag: getElText(node, "ETag"),
|
|
97
|
+
Size: sizeText !== undefined ? Number(sizeText) : undefined,
|
|
98
|
+
StorageClass: getElText(node, "StorageClass"),
|
|
99
|
+
} as _Object;
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const commonPrefixes: string[] = Array.from(root.getElementsByTagName("CommonPrefixes"))
|
|
103
|
+
.map((node) => getElText(node, "Prefix") ?? "")
|
|
104
|
+
.filter(Boolean);
|
|
105
|
+
|
|
106
|
+
return {
|
|
107
|
+
contents,
|
|
108
|
+
commonPrefixes,
|
|
109
|
+
isTruncated: getElText(root, "IsTruncated") === "true",
|
|
110
|
+
nextContinuationToken: getElText(root, "NextContinuationToken"),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function signedListBucketRequest({
|
|
115
|
+
credentials,
|
|
116
|
+
region,
|
|
117
|
+
bucketUrl,
|
|
118
|
+
query,
|
|
119
|
+
signal,
|
|
120
|
+
}: {
|
|
121
|
+
credentials: Credentials;
|
|
122
|
+
region: string;
|
|
123
|
+
bucketUrl: string;
|
|
124
|
+
query: Record<string, string>;
|
|
125
|
+
signal?: AbortSignal;
|
|
126
|
+
}): Promise<Response> {
|
|
127
|
+
if (!credentials.AccessKeyId || !credentials.SecretAccessKey) {
|
|
128
|
+
throw new Error("Invalid credentials: AccessKeyId and SecretAccessKey are required");
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const signer = new SignatureV4({
|
|
132
|
+
credentials: {
|
|
133
|
+
accessKeyId: credentials.AccessKeyId,
|
|
134
|
+
secretAccessKey: credentials.SecretAccessKey,
|
|
135
|
+
sessionToken: credentials.SessionToken,
|
|
136
|
+
},
|
|
137
|
+
region,
|
|
138
|
+
service: "s3",
|
|
139
|
+
sha256: Sha256,
|
|
140
|
+
// S3 paths are pre-encoded; the signer must not double-encode them.
|
|
141
|
+
uriEscapePath: false,
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
const parsed = new URL(bucketUrl);
|
|
145
|
+
|
|
146
|
+
const signed = await signer.sign({
|
|
147
|
+
method: "GET",
|
|
148
|
+
protocol: parsed.protocol,
|
|
149
|
+
hostname: parsed.hostname,
|
|
150
|
+
port: parsed.port ? parseInt(parsed.port) : undefined,
|
|
151
|
+
path: parsed.pathname,
|
|
152
|
+
query,
|
|
153
|
+
headers: { host: parsed.host },
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
const wireQuery = encodeWireQuery(query);
|
|
157
|
+
const wireUrl = `${parsed.origin}${parsed.pathname}?${wireQuery}`;
|
|
158
|
+
|
|
159
|
+
const browserOrigin =
|
|
160
|
+
typeof window !== "undefined" && typeof window.location?.origin === "string"
|
|
161
|
+
? window.location.origin
|
|
162
|
+
: "";
|
|
163
|
+
try {
|
|
164
|
+
return await fetch(wireUrl, {
|
|
165
|
+
method: "GET",
|
|
166
|
+
headers: signed.headers as Record<string, string>,
|
|
167
|
+
signal,
|
|
168
|
+
// Block redirect-following — `fetch` would otherwise re-send the
|
|
169
|
+
// Authorization header (and the STS token) to whatever host the 30x
|
|
170
|
+
// points at.
|
|
171
|
+
redirect: "error",
|
|
172
|
+
});
|
|
173
|
+
} catch (error) {
|
|
174
|
+
if (isLikelyCorsFailure(error)) {
|
|
175
|
+
throw new CorsLikelyError(parsed.host, browserOrigin, error);
|
|
176
|
+
}
|
|
177
|
+
throw error;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function isLikelyCorsFailure(error: unknown): boolean {
|
|
182
|
+
if (!(error instanceof TypeError)) return false;
|
|
183
|
+
const message = error.message ?? "";
|
|
184
|
+
return (
|
|
185
|
+
message.includes("Failed to fetch") ||
|
|
186
|
+
message.includes("Load failed") ||
|
|
187
|
+
message.includes("NetworkError when attempting to fetch resource") ||
|
|
188
|
+
message.includes("NetworkError")
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Detect expired-token bodies on a cloned response so the caller's downstream
|
|
194
|
+
* `.text()` still works. AWS: 400 + `ExpiredToken`; MinIO: 403 + `ExpiredTokenException`.
|
|
195
|
+
*/
|
|
196
|
+
async function isExpiredTokenResponse(response: Response): Promise<boolean> {
|
|
197
|
+
if (response.status !== 400 && response.status !== 403) return false;
|
|
198
|
+
try {
|
|
199
|
+
const body = await response.clone().text();
|
|
200
|
+
return (
|
|
201
|
+
body.includes("<Code>ExpiredToken</Code>") ||
|
|
202
|
+
body.includes("<Code>ExpiredTokenException</Code>")
|
|
203
|
+
);
|
|
204
|
+
} catch {
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Browser-side paginated `ListObjectsV2`. On `ExpiredToken` triggers one
|
|
211
|
+
* refresh through `requestCredentialsRefresh` and retries the page; a second
|
|
212
|
+
* failure surfaces as `ExpiredCredentialsError` for the UI to prompt re-auth.
|
|
213
|
+
*/
|
|
214
|
+
export async function listObjectsClient(
|
|
215
|
+
connectionConfig: Pick<ConnectionConfig, "name" | "bucketName" | "region" | "endpoint">,
|
|
216
|
+
credentials: Credentials,
|
|
217
|
+
options: ListObjectsClientOptions = {},
|
|
218
|
+
): Promise<ListObjectsClientResult> {
|
|
219
|
+
const {
|
|
220
|
+
query,
|
|
221
|
+
prefix,
|
|
222
|
+
recursive = false,
|
|
223
|
+
maxKeys = DEFAULT_PAGE_SIZE,
|
|
224
|
+
maxTotal = DEFAULT_MAX_TOTAL,
|
|
225
|
+
findFirst,
|
|
226
|
+
signal,
|
|
227
|
+
} = options;
|
|
228
|
+
|
|
229
|
+
const bucketUrl = constructS3Url(connectionConfig);
|
|
230
|
+
const region = connectionConfig.region || "eu-central-1";
|
|
231
|
+
const connectionName = connectionConfig.name;
|
|
232
|
+
|
|
233
|
+
let activeCredentials: Credentials = credentials;
|
|
234
|
+
|
|
235
|
+
const contents: _Object[] = [];
|
|
236
|
+
const commonPrefixes: string[] = [];
|
|
237
|
+
let continuationToken: string | undefined;
|
|
238
|
+
let isCapped = false;
|
|
239
|
+
let pageCount = 0;
|
|
240
|
+
|
|
241
|
+
do {
|
|
242
|
+
const queryParams: Record<string, string> = {
|
|
243
|
+
"list-type": "2",
|
|
244
|
+
"max-keys": String(maxKeys),
|
|
245
|
+
};
|
|
246
|
+
if (prefix) queryParams.prefix = prefix;
|
|
247
|
+
if (!recursive) queryParams.delimiter = "/";
|
|
248
|
+
if (continuationToken) queryParams["continuation-token"] = continuationToken;
|
|
249
|
+
|
|
250
|
+
let parsed: ReturnType<typeof parseListResponse>;
|
|
251
|
+
try {
|
|
252
|
+
let response = await signedListBucketRequest({
|
|
253
|
+
credentials: activeCredentials,
|
|
254
|
+
region,
|
|
255
|
+
bucketUrl,
|
|
256
|
+
query: queryParams,
|
|
257
|
+
signal,
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
if (await isExpiredTokenResponse(response)) {
|
|
261
|
+
if (!connectionName) {
|
|
262
|
+
throw new ExpiredCredentialsError(
|
|
263
|
+
"STS credentials expired and no connection name was provided to listObjectsClient.",
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
activeCredentials = await requestCredentialsRefresh(connectionName);
|
|
267
|
+
response = await signedListBucketRequest({
|
|
268
|
+
credentials: activeCredentials,
|
|
269
|
+
region,
|
|
270
|
+
bucketUrl,
|
|
271
|
+
query: queryParams,
|
|
272
|
+
signal,
|
|
273
|
+
});
|
|
274
|
+
if (await isExpiredTokenResponse(response)) {
|
|
275
|
+
throw new ExpiredCredentialsError(
|
|
276
|
+
"STS credentials expired and refresh did not yield a working session.",
|
|
277
|
+
connectionName,
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
if (!response.ok) {
|
|
283
|
+
throw new Error(`ListObjectsV2 failed: ${response.status} ${response.statusText}`);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// Body read + parse inside the guard so a mid-pagination failure
|
|
287
|
+
// returns a capped partial result instead of dropping prior pages.
|
|
288
|
+
const xml = await response.text();
|
|
289
|
+
parsed = parseListResponse(xml);
|
|
290
|
+
} catch (error) {
|
|
291
|
+
// ExpiredCredentialsError must escape so the UI can prompt re-auth.
|
|
292
|
+
if (error instanceof ExpiredCredentialsError) throw error;
|
|
293
|
+
if (pageCount === 0) throw error;
|
|
294
|
+
isCapped = true;
|
|
295
|
+
break;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
pageCount++;
|
|
299
|
+
|
|
300
|
+
for (const obj of parsed.contents) contents.push(obj);
|
|
301
|
+
for (const cp of parsed.commonPrefixes) commonPrefixes.push(cp);
|
|
302
|
+
|
|
303
|
+
if (findFirst && contents.some(findFirst)) break;
|
|
304
|
+
|
|
305
|
+
if (contents.length + commonPrefixes.length >= maxTotal) {
|
|
306
|
+
isCapped = parsed.isTruncated;
|
|
307
|
+
break;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
continuationToken = parsed.isTruncated ? parsed.nextContinuationToken : undefined;
|
|
311
|
+
} while (continuationToken);
|
|
312
|
+
|
|
313
|
+
return {
|
|
314
|
+
contents: filterObjects(contents, { query }),
|
|
315
|
+
commonPrefixes,
|
|
316
|
+
isCapped,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** Hard cap on total entries collected across paginated `ListObjectsV2` calls. */
|
|
2
|
+
export const DEFAULT_MAX_TOTAL = 10_000;
|
|
3
|
+
|
|
4
|
+
/** Shared truncation message so the cap value stays in sync with `DEFAULT_MAX_TOTAL`. */
|
|
5
|
+
export function formatTruncationMessage(name: string): string {
|
|
6
|
+
return `Listing for "${name}" was truncated at the ${DEFAULT_MAX_TOTAL.toLocaleString()}-entry cap. Some entries are not shown.`;
|
|
7
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { Credentials } from "@aws-sdk/client-sts";
|
|
2
|
+
|
|
3
|
+
import { listObjectsClient } from "./listObjectsClient";
|
|
4
|
+
import { resolveConnectionPrefix } from "./pathUtils";
|
|
5
|
+
import type { ConnectionConfig } from "~/.generated/client";
|
|
6
|
+
import { buildLevelTree, TreeNode } from "~/components/DirectoryView/buildDirectoryTree";
|
|
7
|
+
|
|
8
|
+
export interface LoadConnectionLevelArgs {
|
|
9
|
+
connectionConfig: ConnectionConfig;
|
|
10
|
+
credentials: Credentials;
|
|
11
|
+
connectionName: string;
|
|
12
|
+
/** Connection-relative path. May be empty for the bucket root. */
|
|
13
|
+
urlPath: string;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface LoadConnectionLevelResult {
|
|
18
|
+
nodes: TreeNode[];
|
|
19
|
+
isCapped: boolean;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* One-level browser-side listing for a connection. Shared by the object-
|
|
24
|
+
* browser client loader and the lazy-tree expansion hook.
|
|
25
|
+
*/
|
|
26
|
+
export async function loadConnectionLevel({
|
|
27
|
+
connectionConfig,
|
|
28
|
+
credentials,
|
|
29
|
+
connectionName,
|
|
30
|
+
urlPath: rawUrlPath,
|
|
31
|
+
signal,
|
|
32
|
+
}: LoadConnectionLevelArgs): Promise<LoadConnectionLevelResult> {
|
|
33
|
+
const { urlPath, prefix } = resolveConnectionPrefix(connectionConfig.prefix, rawUrlPath);
|
|
34
|
+
|
|
35
|
+
const { contents, commonPrefixes, isCapped } = await listObjectsClient(
|
|
36
|
+
connectionConfig,
|
|
37
|
+
credentials,
|
|
38
|
+
{ prefix, signal },
|
|
39
|
+
);
|
|
40
|
+
|
|
41
|
+
const nodes = buildLevelTree({
|
|
42
|
+
contents,
|
|
43
|
+
commonPrefixes,
|
|
44
|
+
connectionName,
|
|
45
|
+
prefix,
|
|
46
|
+
urlPath,
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
return { nodes, isCapped };
|
|
50
|
+
}
|
|
@@ -8,7 +8,6 @@ import {
|
|
|
8
8
|
import { create } from "zustand";
|
|
9
9
|
import { devtools } from "zustand/middleware";
|
|
10
10
|
|
|
11
|
-
// Create a custom IndexedDB store for file cache
|
|
12
11
|
const idbStore = createIdBStore("file-cache", "files");
|
|
13
12
|
|
|
14
13
|
export interface DownloadProgress {
|
|
@@ -47,7 +46,6 @@ export const useFileStore = create<FileStore>()(
|
|
|
47
46
|
saveFile: async (id: string, data: Uint8Array) => {
|
|
48
47
|
await idbSet(id, data, idbStore);
|
|
49
48
|
|
|
50
|
-
// Mark as complete
|
|
51
49
|
set(
|
|
52
50
|
(state) => ({
|
|
53
51
|
files: {
|
|
@@ -105,7 +103,6 @@ export const useFileStore = create<FileStore>()(
|
|
|
105
103
|
hydrate: async () => {
|
|
106
104
|
const allKeys = await idbKeys<string>(idbStore);
|
|
107
105
|
|
|
108
|
-
// Get sizes for all files
|
|
109
106
|
const filesWithSizes = await Promise.all(
|
|
110
107
|
allKeys.map(async (key) => {
|
|
111
108
|
const data = await idbGet<Uint8Array>(key, idbStore);
|
|
@@ -120,7 +117,6 @@ export const useFileStore = create<FileStore>()(
|
|
|
120
117
|
(state) => {
|
|
121
118
|
const files = { ...state.files };
|
|
122
119
|
|
|
123
|
-
// Add any keys from IndexedDB that aren't in the store
|
|
124
120
|
for (const { key, size } of filesWithSizes) {
|
|
125
121
|
if (!files[key]) {
|
|
126
122
|
files[key] = {
|
|
@@ -134,7 +130,6 @@ export const useFileStore = create<FileStore>()(
|
|
|
134
130
|
}
|
|
135
131
|
}
|
|
136
132
|
|
|
137
|
-
// Remove any keys from store that aren't in IndexedDB
|
|
138
133
|
for (const key of Object.keys(files)) {
|
|
139
134
|
if (!allKeys.includes(key)) {
|
|
140
135
|
delete files[key];
|
|
@@ -148,6 +143,6 @@ export const useFileStore = create<FileStore>()(
|
|
|
148
143
|
);
|
|
149
144
|
},
|
|
150
145
|
}),
|
|
151
|
-
{ name },
|
|
146
|
+
{ name, enabled: process.env.NODE_ENV !== "production" },
|
|
152
147
|
),
|
|
153
148
|
);
|
package/app/utils/pathUtils.ts
CHANGED
|
@@ -1,3 +1,14 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Reject NUL / CR / LF (illegal in S3 keys, common smuggling chars) and
|
|
5
|
+
* oversized inputs. `..` traversal is checked by `resolveConnectionPrefix`.
|
|
6
|
+
*/
|
|
7
|
+
export const prefixSchema = z
|
|
8
|
+
.string()
|
|
9
|
+
.max(1024, "Prefix exceeds 1024 characters")
|
|
10
|
+
.regex(/^[^\0\r\n]*$/, "Prefix contains illegal control characters");
|
|
11
|
+
|
|
1
12
|
export function getPrefix(path?: string) {
|
|
2
13
|
if (!path) return undefined;
|
|
3
14
|
if (path.endsWith("/")) return path;
|
|
@@ -8,3 +19,57 @@ export function getName(path?: string, bucketName?: string): string {
|
|
|
8
19
|
if (!path) return bucketName ?? "";
|
|
9
20
|
return path.split("/").pop() ?? "";
|
|
10
21
|
}
|
|
22
|
+
|
|
23
|
+
/** Thrown when a caller-supplied prefix escapes the connection's `prefix` boundary. */
|
|
24
|
+
export class ConnectionPrefixError extends Error {
|
|
25
|
+
constructor(message: string) {
|
|
26
|
+
super(message);
|
|
27
|
+
this.name = "ConnectionPrefixError";
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface ResolvedConnectionPrefix {
|
|
32
|
+
/** Normalized path relative to the connection root (no leading/trailing slash). */
|
|
33
|
+
urlPath: string;
|
|
34
|
+
/** Full S3 key path: `${connPrefix}/${urlPath}` with redundant slashes squashed. */
|
|
35
|
+
pathName: string;
|
|
36
|
+
/** S3 listing prefix — `pathName` with a trailing slash, or `undefined` for bucket root. */
|
|
37
|
+
prefix: string | undefined;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Compose the S3 listing prefix, asserting `rawUrlPath` stays under
|
|
42
|
+
* `connPrefix`. Defense in depth above STS / bucket policy. Throws
|
|
43
|
+
* `ConnectionPrefixError` on `..` segments or out-of-prefix paths.
|
|
44
|
+
*/
|
|
45
|
+
export function resolveConnectionPrefix(
|
|
46
|
+
connPrefix: string | null | undefined,
|
|
47
|
+
rawUrlPath: string,
|
|
48
|
+
): ResolvedConnectionPrefix {
|
|
49
|
+
const segments = rawUrlPath.split("/");
|
|
50
|
+
for (const seg of segments) {
|
|
51
|
+
if (seg === "..") {
|
|
52
|
+
throw new ConnectionPrefixError("Path traversal '..' is not allowed in prefix");
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const urlPath = segments.filter((s) => s !== "" && s !== ".").join("/");
|
|
57
|
+
|
|
58
|
+
const normalizedConnPrefix = (connPrefix ?? "").replace(/^\/+|\/+$/g, "");
|
|
59
|
+
|
|
60
|
+
const pathName = normalizedConnPrefix
|
|
61
|
+
? urlPath
|
|
62
|
+
? `${normalizedConnPrefix}/${urlPath}`
|
|
63
|
+
: normalizedConnPrefix
|
|
64
|
+
: urlPath;
|
|
65
|
+
|
|
66
|
+
if (normalizedConnPrefix && !pathName.startsWith(normalizedConnPrefix)) {
|
|
67
|
+
throw new ConnectionPrefixError("Resolved path escapes connection prefix");
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return {
|
|
71
|
+
urlPath,
|
|
72
|
+
pathName,
|
|
73
|
+
prefix: getPrefix(pathName),
|
|
74
|
+
};
|
|
75
|
+
}
|
package/app/utils/resourceId.ts
CHANGED
|
@@ -19,10 +19,7 @@ export function parseResourceId(resourceId: string): ResourceIdParts {
|
|
|
19
19
|
|
|
20
20
|
const connectionName = resourceId.slice(0, slashIndex);
|
|
21
21
|
|
|
22
|
-
const pathName = resourceId
|
|
23
|
-
.slice(slashIndex + 1)
|
|
24
|
-
// Strip leading slash
|
|
25
|
-
.replace(/^\/+/, "");
|
|
22
|
+
const pathName = resourceId.slice(slashIndex + 1).replace(/^\/+/, "");
|
|
26
23
|
|
|
27
24
|
if (!connectionName) {
|
|
28
25
|
throw new Error(`Invalid resourceId: "${resourceId}" — empty connectionName`);
|
|
@@ -46,38 +43,26 @@ export function buildConnectionPath(connectionName: string, pathName: string): s
|
|
|
46
43
|
}
|
|
47
44
|
|
|
48
45
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* The `s3Key` is the **full object key including any connection prefix**.
|
|
55
|
-
* If you have a prefix-relative `pathName` and a resourceId, use
|
|
56
|
-
* `selectHttpsUrl` / `resolveResourceId` — they rejoin the prefix for you.
|
|
57
|
-
*
|
|
58
|
-
* @example
|
|
59
|
-
* // AWS:
|
|
60
|
-
* constructS3Url({ bucketName: "my-bucket", region: "eu-central-1" }, "data/image.ome.tif")
|
|
61
|
-
* // → "https://s3.eu-central-1.amazonaws.com/my-bucket/data/image.ome.tif"
|
|
62
|
-
*
|
|
63
|
-
* @example
|
|
64
|
-
* // MinIO / R2 custom endpoint:
|
|
65
|
-
* constructS3Url({ bucketName: "b", endpoint: "http://localhost:9000" }, "x.zarr")
|
|
66
|
-
* // → "http://localhost:9000/b/x.zarr"
|
|
46
|
+
* Build the HTTPS URL for an S3 bucket or object. Always path-style (dotted
|
|
47
|
+
* bucket names break the vhost wildcard cert). `s3Key` is the full object
|
|
48
|
+
* key including any connection prefix; pass `""` for the bucket-level URL.
|
|
49
|
+
* Callers pass raw keys — path segments are URI-encoded internally.
|
|
67
50
|
*/
|
|
68
|
-
export function constructS3Url(
|
|
51
|
+
export function constructS3Url(
|
|
52
|
+
connectionConfig: Pick<ConnectionConfig, "bucketName" | "region" | "endpoint">,
|
|
53
|
+
s3Key: string = "",
|
|
54
|
+
): string {
|
|
69
55
|
const bucket = connectionConfig.bucketName;
|
|
70
|
-
const encodedPath = s3Key.split("/").map(encodeURIComponent).join("/");
|
|
71
|
-
|
|
72
56
|
const region = connectionConfig.region || "eu-central-1";
|
|
73
57
|
const endpoint = connectionConfig.endpoint?.replace(/\/$/, "");
|
|
74
58
|
|
|
75
59
|
const isAwsEndpoint = !endpoint || /\.amazonaws\.com$/i.test(endpoint);
|
|
60
|
+
const origin = isAwsEndpoint ? `https://s3.${region}.amazonaws.com` : endpoint;
|
|
76
61
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
return `https://s3.${region}.amazonaws.com/${bucket}/${encodedPath}`;
|
|
62
|
+
if (!s3Key) {
|
|
63
|
+
return `${origin}/${bucket}`;
|
|
80
64
|
}
|
|
81
65
|
|
|
82
|
-
|
|
66
|
+
const encodedPath = s3Key.split("/").map(encodeURIComponent).join("/");
|
|
67
|
+
return `${origin}/${bucket}/${encodedPath}`;
|
|
83
68
|
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared S3-host allowlist used by the CSP `connect-src` builder and the
|
|
3
|
+
* connection schema / CORS preflight probe.
|
|
4
|
+
*
|
|
5
|
+
* `CYTARIO_ALLOWED_S3_HOSTS` REPLACES the defaults entirely — deployers
|
|
6
|
+
* who want defaults plus extras must include them explicitly.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export const DEFAULT_S3_HOSTS = ["https://*.amazonaws.com", "https://*.cytario.com"];
|
|
10
|
+
|
|
11
|
+
// `process.env` as a default param crashes in the browser the moment zod
|
|
12
|
+
// invokes `isAllowedS3Host` from form validation.
|
|
13
|
+
const readProcessEnv = (): Record<string, string | undefined> => {
|
|
14
|
+
if (typeof process !== "undefined" && process.env) return process.env;
|
|
15
|
+
return {};
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Accept `https://host[:port]` and `https://*.host[:port]`. Only a leading
|
|
20
|
+
* `*.` wildcard label is allowed — embedded `*` would smuggle a permissive
|
|
21
|
+
* pattern past validation.
|
|
22
|
+
*/
|
|
23
|
+
const isWellFormedAllowlistEntry = (entry: string): boolean => {
|
|
24
|
+
if (!entry.startsWith("https://")) return false;
|
|
25
|
+
const rest = entry.slice("https://".length);
|
|
26
|
+
if (rest.length === 0) return false;
|
|
27
|
+
|
|
28
|
+
// URL parser balks on `*.` patterns, so split manually.
|
|
29
|
+
if (/[/?#]/.test(rest)) return false;
|
|
30
|
+
|
|
31
|
+
const [hostPart, portPart, ...extra] = rest.split(":");
|
|
32
|
+
if (extra.length > 0) return false;
|
|
33
|
+
if (portPart !== undefined && !/^\d+$/.test(portPart)) return false;
|
|
34
|
+
|
|
35
|
+
if (hostPart.length === 0) return false;
|
|
36
|
+
|
|
37
|
+
const labels = hostPart.split(".");
|
|
38
|
+
for (let i = 0; i < labels.length; i++) {
|
|
39
|
+
const label = labels[i];
|
|
40
|
+
if (label.length === 0) return false;
|
|
41
|
+
if (label === "*") {
|
|
42
|
+
if (i !== 0) return false;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (!/^[a-zA-Z0-9-]+$/.test(label)) return false;
|
|
46
|
+
}
|
|
47
|
+
return true;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Comma-separated `CYTARIO_ALLOWED_S3_HOSTS` overrides `DEFAULT_S3_HOSTS`
|
|
52
|
+
* entirely. Malformed entries are dropped with a `console.warn`.
|
|
53
|
+
*/
|
|
54
|
+
export function getAllowedS3Hosts(
|
|
55
|
+
env: Record<string, string | undefined> = readProcessEnv(),
|
|
56
|
+
): string[] {
|
|
57
|
+
const raw = env.CYTARIO_ALLOWED_S3_HOSTS ?? "";
|
|
58
|
+
const trimmed = raw.trim();
|
|
59
|
+
if (trimmed.length === 0) {
|
|
60
|
+
return [...DEFAULT_S3_HOSTS];
|
|
61
|
+
}
|
|
62
|
+
const overrides = trimmed
|
|
63
|
+
.split(",")
|
|
64
|
+
.map((s) => s.trim())
|
|
65
|
+
.filter((s) => s.length > 0)
|
|
66
|
+
.filter((entry) => {
|
|
67
|
+
if (isWellFormedAllowlistEntry(entry)) return true;
|
|
68
|
+
console.warn(`[s3HostAllowlist] Ignoring malformed CYTARIO_ALLOWED_S3_HOSTS entry: ${entry}`);
|
|
69
|
+
return false;
|
|
70
|
+
});
|
|
71
|
+
return overrides;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// `*.example.com` matches `foo.example.com` but not the bare parent, matching
|
|
75
|
+
// browser CORS / cookie rules.
|
|
76
|
+
const hostnameMatchesPattern = (hostname: string, patternHostname: string): boolean => {
|
|
77
|
+
if (patternHostname.startsWith("*.")) {
|
|
78
|
+
const suffix = patternHostname.slice(1);
|
|
79
|
+
return hostname.endsWith(suffix) && hostname.length > suffix.length;
|
|
80
|
+
}
|
|
81
|
+
return hostname === patternHostname;
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* https-only by design — the schema's dev http carve-out for localhost
|
|
86
|
+
* does not flow through here so the SSRF surface stays narrow.
|
|
87
|
+
*/
|
|
88
|
+
export function isAllowedS3Host(
|
|
89
|
+
url: string,
|
|
90
|
+
env: Record<string, string | undefined> = readProcessEnv(),
|
|
91
|
+
): boolean {
|
|
92
|
+
let parsed: URL;
|
|
93
|
+
try {
|
|
94
|
+
parsed = new URL(url);
|
|
95
|
+
} catch {
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
if (parsed.protocol !== "https:") return false;
|
|
99
|
+
|
|
100
|
+
const allowed = getAllowedS3Hosts(env);
|
|
101
|
+
for (const entry of allowed) {
|
|
102
|
+
let pattern: URL;
|
|
103
|
+
try {
|
|
104
|
+
pattern = new URL(entry);
|
|
105
|
+
} catch {
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (pattern.protocol !== "https:") continue;
|
|
109
|
+
// Pattern with no port matches any port; with a port, exact match required.
|
|
110
|
+
if (pattern.port && pattern.port !== parsed.port) continue;
|
|
111
|
+
if (hostnameMatchesPattern(parsed.hostname, pattern.hostname)) {
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return false;
|
|
116
|
+
}
|