@cytario/web 2.1.8 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/app/.server/auth/README.md +0 -1
- package/app/.server/auth/authMiddleware.ts +6 -28
- package/app/.server/auth/getSessionCredentials.ts +11 -1
- package/app/.server/auth/sessionPolicy.ts +69 -0
- package/app/.server/corsPreflight.ts +136 -0
- package/app/.server/csp.ts +34 -0
- package/app/components/.client/ImageViewer/README.md +2 -2
- package/app/components/DirectoryView/DirectoryViewGrid.tsx +2 -11
- package/app/components/DirectoryView/DirectoryViewTree.tsx +18 -26
- package/app/components/DirectoryView/buildDirectoryTree.ts +89 -42
- package/app/components/DirectoryView/filterNodes.ts +4 -19
- package/app/components/DirectoryView/useLazyTreeNodes.ts +120 -0
- package/app/components/GlobalSearch/GlobalSearch.tsx +8 -3
- package/app/components/GlobalSearch/Suggestions.tsx +21 -3
- package/app/config.ts +0 -7
- package/app/entry.server.tsx +9 -4
- package/app/hooks/useInitConnections.ts +3 -3
- package/app/root.tsx +11 -6
- package/app/routes/connections/connection.form.tsx +9 -3
- package/app/routes/connections/connection.schema.ts +60 -11
- package/app/routes/connections/connections.clientLoader.ts +91 -0
- package/app/routes/connections/connections.loader.ts +19 -39
- package/app/routes/connections/connections.route.tsx +5 -0
- package/app/routes/connections/createConnection.action.ts +39 -13
- package/app/routes/connections/deleteConnection.action.ts +5 -1
- package/app/routes/connections/updateConnection.action.ts +50 -13
- package/app/routes/home/home.route.tsx +5 -1
- package/app/routes/layouts/protected.layout.tsx +15 -1
- package/app/routes/objects/objects.clientLoader.ts +73 -0
- package/app/routes/objects/objects.loader.ts +58 -92
- package/app/routes/objects/objects.route.tsx +41 -40
- package/app/routes/search.route.tsx +133 -28
- package/app/routes.ts +0 -4
- package/app/utils/connectionsStore/useConnectionsStore.ts +25 -62
- package/app/utils/credentialsRefresh.ts +53 -0
- package/app/utils/db/convertCsvToParquet.ts +24 -16
- package/app/utils/db/createDatabase.ts +25 -33
- package/app/utils/db/duckdbBundles.ts +42 -0
- package/app/utils/db/ensureSpatialLoaded.ts +28 -0
- package/app/utils/db/escapeSqlString.ts +4 -0
- package/app/utils/db/getBlobFromObjectNode.ts +15 -36
- package/app/utils/db/getTileDataWasm.ts +4 -5
- package/app/utils/db/sqlQueries.ts +6 -1
- package/app/utils/filterObjects.ts +2 -18
- package/app/utils/limitConcurrency.ts +28 -0
- package/app/utils/listObjectsClient.ts +318 -0
- package/app/utils/listingLimits.ts +7 -0
- package/app/utils/loadConnectionLevel.ts +50 -0
- package/app/utils/localFilesStore/useFileStore.ts +1 -6
- package/app/utils/pathUtils.ts +65 -0
- package/app/utils/resourceId.ts +14 -29
- package/app/utils/s3HostAllowlist.ts +116 -0
- package/app/utils/signedFetch.ts +159 -24
- package/package.json +2 -2
- package/prisma/seed.ts +3 -3
- package/public/duckdb-extensions/checksums.json +12 -0
- package/scripts/download-duckdb-extensions.mjs +188 -0
- package/scripts/prebuild.mjs +18 -10
- package/app/.server/auth/getPresignedUrl.ts +0 -21
- package/app/.server/auth/getS3Client.ts +0 -93
- package/app/routes/presign.route.tsx +0 -42
- package/app/utils/getObjects.ts +0 -24
package/app/utils/signedFetch.ts
CHANGED
|
@@ -3,38 +3,116 @@ import type { Credentials } from "@aws-sdk/client-sts";
|
|
|
3
3
|
import { SignatureV4 } from "@smithy/signature-v4";
|
|
4
4
|
|
|
5
5
|
import type { ConnectionConfig } from "~/.generated/client";
|
|
6
|
+
import { ExpiredCredentialsError, requestCredentialsRefresh } from "~/utils/credentialsRefresh";
|
|
6
7
|
import { sanitizeHeaders } from "~/utils/sanitizeHeaders";
|
|
7
8
|
|
|
8
9
|
export type SignedFetch = (url: string, init?: RequestInit) => Promise<Response>;
|
|
9
10
|
|
|
10
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Lowercase header names the SigV4 signer emits. The CORS probe lists these
|
|
13
|
+
* in `Access-Control-Request-Headers` so the OPTIONS preflight carries the
|
|
14
|
+
* same set as the eventual GET. Keep in sync with `signer.sign` below.
|
|
15
|
+
*/
|
|
16
|
+
export const SIGNED_REQUEST_HEADERS = [
|
|
17
|
+
"authorization",
|
|
18
|
+
"x-amz-content-sha256",
|
|
19
|
+
"x-amz-date",
|
|
20
|
+
"x-amz-security-token",
|
|
21
|
+
] as const;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Thrown when a browser fetch fails before a Response — almost always a
|
|
25
|
+
* CORS misconfiguration on the bucket. Caller error paths instanceof-check
|
|
26
|
+
* this to surface a dedicated toast.
|
|
27
|
+
*/
|
|
28
|
+
export class CorsLikelyError extends Error {
|
|
29
|
+
public readonly host: string;
|
|
30
|
+
public readonly origin: string;
|
|
31
|
+
|
|
32
|
+
constructor(host: string, origin: string, cause?: unknown) {
|
|
33
|
+
super(
|
|
34
|
+
`Browser was blocked from reading "${host}" — likely a CORS misconfiguration on the bucket.`,
|
|
35
|
+
);
|
|
36
|
+
this.name = "CorsLikelyError";
|
|
37
|
+
this.host = host;
|
|
38
|
+
this.origin = origin;
|
|
39
|
+
if (cause !== undefined) {
|
|
40
|
+
(this as { cause?: unknown }).cause = cause;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function isLikelyCorsFailure(error: unknown): boolean {
|
|
46
|
+
if (!(error instanceof TypeError)) return false;
|
|
47
|
+
const message = error.message ?? "";
|
|
48
|
+
return (
|
|
49
|
+
message.includes("Failed to fetch") ||
|
|
50
|
+
message.includes("Load failed") ||
|
|
51
|
+
message.includes("NetworkError when attempting to fetch resource") ||
|
|
52
|
+
message.includes("NetworkError")
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function fetchWithCorsDetection(
|
|
57
|
+
url: string,
|
|
58
|
+
init: RequestInit,
|
|
59
|
+
host: string,
|
|
60
|
+
origin: string,
|
|
61
|
+
): Promise<Response> {
|
|
62
|
+
try {
|
|
63
|
+
return await fetch(url, init);
|
|
64
|
+
} catch (error) {
|
|
65
|
+
if (isLikelyCorsFailure(error)) {
|
|
66
|
+
throw new CorsLikelyError(host, origin, error);
|
|
67
|
+
}
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Detect expired-token bodies on a cloned response (the caller still owns the
|
|
74
|
+
* unread body). AWS: 400 + `ExpiredToken`; MinIO: 403 + `ExpiredTokenException`.
|
|
75
|
+
*/
|
|
76
|
+
async function isExpiredTokenResponse(response: Response): Promise<boolean> {
|
|
77
|
+
if (response.status !== 400 && response.status !== 403) return false;
|
|
78
|
+
try {
|
|
79
|
+
const body = await response.clone().text();
|
|
80
|
+
return (
|
|
81
|
+
body.includes("<Code>ExpiredToken</Code>") ||
|
|
82
|
+
body.includes("<Code>ExpiredTokenException</Code>")
|
|
83
|
+
);
|
|
84
|
+
} catch {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Tile/chunk bytes are immutable per object version → 7-day cache.
|
|
11
90
|
const IMAGE_DATA_CACHE_CONTROL = "private, max-age=604800";
|
|
12
91
|
|
|
13
|
-
// Sidecars
|
|
92
|
+
// Sidecars / overlays / JSON companions → 1-hour ceiling so analyst
|
|
14
93
|
// regenerations surface without an explicit purge.
|
|
15
94
|
const OTHER_DATA_CACHE_CONTROL = "private, max-age=3600";
|
|
16
95
|
|
|
17
|
-
// TIFF/OME-TIFF reads
|
|
18
|
-
// only (`image.zarr/0/0/0` or `image.zarr/0.0.0`).
|
|
96
|
+
// Matches TIFF/OME-TIFF reads and OME-Zarr chunk filenames (digits-only).
|
|
19
97
|
function isImageDataPath(pathname: string): boolean {
|
|
20
98
|
return /\.tiff?$/i.test(pathname) || /\/\d+(?:\.\d+)*$/.test(pathname);
|
|
21
99
|
}
|
|
22
100
|
|
|
23
101
|
/**
|
|
24
|
-
* SigV4-signing fetch. Credentials
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* Cache-Control directive — the browser HTTP cache then serves repeat reads
|
|
28
|
-
* without a network round-trip.
|
|
102
|
+
* SigV4-signing fetch. Credentials resolve lazily so the signer is rebuilt
|
|
103
|
+
* when `AccessKeyId` rotates. Every GET injects `response-cache-control` so
|
|
104
|
+
* the browser HTTP cache can serve repeat reads without a network round-trip.
|
|
29
105
|
*/
|
|
30
106
|
export function createSignedFetch(
|
|
31
107
|
getCredentials: () => Credentials,
|
|
32
108
|
connectionConfig: Pick<ConnectionConfig, "region">,
|
|
109
|
+
connectionName?: string,
|
|
33
110
|
): SignedFetch {
|
|
34
111
|
let cachedKeyId: string | undefined;
|
|
35
112
|
let signer: SignatureV4;
|
|
36
113
|
|
|
37
|
-
|
|
114
|
+
// Closured so the ExpiredToken retry path can rebuild with refreshed creds.
|
|
115
|
+
const buildSignedRequest = async (url: string, init?: RequestInit) => {
|
|
38
116
|
const credentials = getCredentials();
|
|
39
117
|
|
|
40
118
|
if (!credentials.AccessKeyId || !credentials.SecretAccessKey) {
|
|
@@ -51,33 +129,34 @@ export function createSignedFetch(
|
|
|
51
129
|
region: connectionConfig.region || "eu-central-1",
|
|
52
130
|
service: "s3",
|
|
53
131
|
sha256: Sha256,
|
|
132
|
+
// S3 paths are pre-encoded; the signer must not double-encode.
|
|
133
|
+
uriEscapePath: false,
|
|
54
134
|
});
|
|
55
135
|
cachedKeyId = credentials.AccessKeyId;
|
|
56
136
|
}
|
|
57
137
|
|
|
58
138
|
const parsed = new URL(url);
|
|
59
139
|
|
|
60
|
-
// Decode so the signer can re-encode canonically — without this,
|
|
61
|
-
// percent-encoded chars get double-encoded and the signature breaks.
|
|
62
|
-
const decodedPath = decodeURIComponent(parsed.pathname);
|
|
63
|
-
|
|
64
140
|
const cacheControl = isImageDataPath(parsed.pathname)
|
|
65
141
|
? IMAGE_DATA_CACHE_CONTROL
|
|
66
142
|
: OTHER_DATA_CACHE_CONTROL;
|
|
67
143
|
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
// sanitizeHeaders, and the merge order below puts signed headers LAST
|
|
71
|
-
// so a bypass of sanitizeHeaders still cannot override the signature.
|
|
144
|
+
// Signed headers merge LAST so a bypass of `sanitizeHeaders` cannot
|
|
145
|
+
// override the signature.
|
|
72
146
|
const callerHeaders = sanitizeHeaders(init?.headers as Record<string, string> | undefined);
|
|
73
147
|
|
|
148
|
+
// Pre-populate the signer's query map so caller-supplied params
|
|
149
|
+
// (e.g. `?versionId=...`) survive — otherwise the wire URL drops them.
|
|
150
|
+
const query: Record<string, string> = Object.fromEntries(parsed.searchParams);
|
|
151
|
+
query["response-cache-control"] = cacheControl;
|
|
152
|
+
|
|
74
153
|
const request = {
|
|
75
154
|
method: (init?.method as string) ?? "GET",
|
|
76
155
|
protocol: parsed.protocol,
|
|
77
156
|
hostname: parsed.hostname,
|
|
78
157
|
port: parsed.port ? parseInt(parsed.port) : undefined,
|
|
79
|
-
path:
|
|
80
|
-
query
|
|
158
|
+
path: parsed.pathname,
|
|
159
|
+
query,
|
|
81
160
|
headers: {
|
|
82
161
|
host: parsed.host,
|
|
83
162
|
},
|
|
@@ -85,9 +164,24 @@ export function createSignedFetch(
|
|
|
85
164
|
|
|
86
165
|
const signed = await signer.sign(request);
|
|
87
166
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
167
|
+
// RFC-3986 strict encoding — `URLSearchParams.toString()` form-encodes
|
|
168
|
+
// and would break the signature on reserved characters.
|
|
169
|
+
const wireQuery = Object.entries(query)
|
|
170
|
+
.map(
|
|
171
|
+
([key, value]) =>
|
|
172
|
+
`${encodeURIComponent(key).replace(/[!*'()]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}=${encodeURIComponent(
|
|
173
|
+
value,
|
|
174
|
+
).replace(/[!*'()]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}`,
|
|
175
|
+
)
|
|
176
|
+
.join("&");
|
|
177
|
+
const wireUrl = `${parsed.origin}${parsed.pathname}?${wireQuery}`;
|
|
178
|
+
|
|
179
|
+
const browserOrigin =
|
|
180
|
+
typeof window !== "undefined" && typeof window.location?.origin === "string"
|
|
181
|
+
? window.location.origin
|
|
182
|
+
: "";
|
|
183
|
+
|
|
184
|
+
const fetchInit: RequestInit = {
|
|
91
185
|
...init,
|
|
92
186
|
method: request.method,
|
|
93
187
|
headers: {
|
|
@@ -95,6 +189,47 @@ export function createSignedFetch(
|
|
|
95
189
|
...(signed.headers as Record<string, string>),
|
|
96
190
|
},
|
|
97
191
|
signal: init?.signal,
|
|
98
|
-
|
|
192
|
+
// Block redirect-following — `fetch` would otherwise re-send the
|
|
193
|
+
// Authorization header (and the STS token) to whatever host the 30x
|
|
194
|
+
// points at.
|
|
195
|
+
redirect: "error",
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
return { wireUrl, fetchInit, host: parsed.host, browserOrigin };
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
return async (url: string, init?: RequestInit): Promise<Response> => {
|
|
202
|
+
const first = await buildSignedRequest(url, init);
|
|
203
|
+
const response = await fetchWithCorsDetection(
|
|
204
|
+
first.wireUrl,
|
|
205
|
+
first.fetchInit,
|
|
206
|
+
first.host,
|
|
207
|
+
first.browserOrigin,
|
|
208
|
+
);
|
|
209
|
+
|
|
210
|
+
if (await isExpiredTokenResponse(response)) {
|
|
211
|
+
if (!connectionName) {
|
|
212
|
+
throw new ExpiredCredentialsError(
|
|
213
|
+
"STS credentials expired and no connection name was provided to signedFetch.",
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
await requestCredentialsRefresh(connectionName);
|
|
217
|
+
const retried = await buildSignedRequest(url, init);
|
|
218
|
+
const retryResponse = await fetchWithCorsDetection(
|
|
219
|
+
retried.wireUrl,
|
|
220
|
+
retried.fetchInit,
|
|
221
|
+
retried.host,
|
|
222
|
+
retried.browserOrigin,
|
|
223
|
+
);
|
|
224
|
+
if (await isExpiredTokenResponse(retryResponse)) {
|
|
225
|
+
throw new ExpiredCredentialsError(
|
|
226
|
+
"STS credentials expired and refresh did not yield a working session.",
|
|
227
|
+
connectionName,
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
return retryResponse;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
return response;
|
|
99
234
|
};
|
|
100
235
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cytario/web",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.2.0",
|
|
4
4
|
"description": "Cytario Web — scientific imaging data browser and viewer for OME-TIFF, OME-Zarr, Parquet and GeoTIFF on S3-compatible storage.",
|
|
5
5
|
"license": "AGPL-3.0",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"build:server": "tsup --config tsup.server.config.ts",
|
|
55
55
|
"codegen:check": "tsx scripts/codegen-check.ts",
|
|
56
56
|
"dev": "node bin/cytario-web.mjs dev",
|
|
57
|
-
"predev": "if [ -L node_modules/@cytario/design ]; then npm unlink @cytario/design --no-save 2>/dev/null && npm install @cytario/design --quiet; fi",
|
|
57
|
+
"predev": "node scripts/prebuild.mjs && if [ -L node_modules/@cytario/design ]; then npm unlink @cytario/design --no-save 2>/dev/null && npm install @cytario/design --quiet; fi",
|
|
58
58
|
"predev:design": "cd ../cytario-design && npm link --quiet && cd - > /dev/null && npm link @cytario/design --quiet && for p in react react-dom react-aria-components; do rm -rf ../cytario-design/node_modules/$p && ln -s \"$(pwd)/node_modules/$p\" ../cytario-design/node_modules/$p; done",
|
|
59
59
|
"dev:design": "concurrently -n tsup,storybook,web -c blue,magenta,green \"cd ../cytario-design && npx tsup --watch\" \"cd ../cytario-design && npm run dev\" \"node bin/cytario-web.mjs dev\"",
|
|
60
60
|
"lint": "eslint --cache --cache-location ./node_modules/.cache/eslint .",
|
package/prisma/seed.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Creates test fixtures in a fresh database. Runs automatically after
|
|
5
5
|
* `prisma db push` or explicitly via `npx prisma db seed`.
|
|
6
6
|
*
|
|
7
|
-
* Both connections point to the same bucket (
|
|
7
|
+
* Both connections point to the same bucket (shared-bucket-example).
|
|
8
8
|
* The prefixed connection tests prefix-based directory listing without
|
|
9
9
|
* requiring a separate bucket or cross-bucket auth setup.
|
|
10
10
|
*/
|
|
@@ -22,7 +22,7 @@ const TEST_PREFIX_CONNECTION_NAME = process.env.E2E_PREFIX_CONNECTION_NAME || "E
|
|
|
22
22
|
const SHARED_BUCKET = {
|
|
23
23
|
ownerScope: "cytario",
|
|
24
24
|
createdBy: "e2e-seed",
|
|
25
|
-
bucketName: "
|
|
25
|
+
bucketName: "shared-bucket-example",
|
|
26
26
|
provider: "aws" as const,
|
|
27
27
|
endpoint: "https://s3.eu-central-1.amazonaws.com",
|
|
28
28
|
roleArn: "arn:aws:iam::727043715722:role/keycloack-aws-test-iam-role",
|
|
@@ -46,7 +46,7 @@ async function seed() {
|
|
|
46
46
|
create: {
|
|
47
47
|
...SHARED_BUCKET,
|
|
48
48
|
name: TEST_PREFIX_CONNECTION_NAME,
|
|
49
|
-
prefix: "
|
|
49
|
+
prefix: "Alpha Lab",
|
|
50
50
|
},
|
|
51
51
|
});
|
|
52
52
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "SHA-256 checksums for DuckDB-WASM extensions mirrored locally. Pinned by scripts/download-duckdb-extensions.mjs — every entry must be reviewed and re-populated by hand at each DuckDB core version bump. Keys are `v<version>/<platform>/<ext>`.",
|
|
3
|
+
"v1.4.3/wasm_eh/httpfs": "5e76d1fb0779c4803ebfd6fb49b80ff833c11b9a6f55208a701e63a491df1d5c",
|
|
4
|
+
"v1.4.3/wasm_eh/parquet": "22765c8f7dc741cda2b571a66ac7bb355295d7d69a6c37e5315b265672984f55",
|
|
5
|
+
"v1.4.3/wasm_eh/spatial": "04b776946da64a15a7b14501790c75093e38f876acc46b2922f0daeb6aaa1d60",
|
|
6
|
+
"v1.4.3/wasm_mvp/httpfs": "88e7d15f1c6ac066cca28b4ee1046486a5110783dea82c28fc4f973924f12498",
|
|
7
|
+
"v1.4.3/wasm_mvp/parquet": "0785c6c95d003eff4faa7b3b4b660f02c9c92f6d68d135ddf330d42e3a650600",
|
|
8
|
+
"v1.4.3/wasm_mvp/spatial": "7a745cfc5259f69b46f077bc6afeb7a6aefb8ef8d8b336bb0b770e5449708bb4",
|
|
9
|
+
"v1.4.3/wasm_threads/httpfs": "20b0a02de0b0a7de2a846bc40d5b504f031580949c7e5828c9da672a8961d241",
|
|
10
|
+
"v1.4.3/wasm_threads/parquet": "4bae942b3a6d2b68f92203408c2b45b66cb107ec3fa119e0f000f368810ca98b",
|
|
11
|
+
"v1.4.3/wasm_threads/spatial": "83ad79d02f1dc5d72a0d5d63fa4a49e9f2a9413e947e889956f3ffabaee095c6"
|
|
12
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/* global console, process, fetch, Buffer */
|
|
3
|
+
/**
|
|
4
|
+
* Download the DuckDB-WASM extensions cytario relies on (httpfs, spatial)
|
|
5
|
+
* for every WASM build variant the runtime might pick, and write them
|
|
6
|
+
* under `public/duckdb-extensions/v<duckdb-version>/<platform>/...`.
|
|
7
|
+
*
|
|
8
|
+
* Why bundle:
|
|
9
|
+
* - Privacy: `INSTALL httpfs;` at session start otherwise leaks every
|
|
10
|
+
* user's IP + Referer to `extensions.duckdb.org` (Cloudflare). Same
|
|
11
|
+
* reason we removed the runtime `jsdelivr` fetch for the core WASM
|
|
12
|
+
* module (see `app/utils/db/duckdbBundles.ts`).
|
|
13
|
+
* - CSP: our `connect-src` allowlist intentionally excludes third-party
|
|
14
|
+
* CDNs; without local mirrors the page-load `INSTALL` blocks.
|
|
15
|
+
*
|
|
16
|
+
* Vite serves anything in `public/` from the document origin verbatim.
|
|
17
|
+
* `createDatabase.ts` points DuckDB at this local mirror via
|
|
18
|
+
* `SET custom_extension_repository='<origin>/duckdb-extensions/';`
|
|
19
|
+
* before issuing `INSTALL <ext>; LOAD <ext>;`. DuckDB then fetches
|
|
20
|
+
* `<repo>/v<version>/<platform>/<ext>.duckdb_extension.wasm`
|
|
21
|
+
* which matches the upstream `extensions.duckdb.org` layout.
|
|
22
|
+
*
|
|
23
|
+
* Supply-chain trust:
|
|
24
|
+
* `allowUnsignedExtensions: true` in `createDatabase.ts` disables
|
|
25
|
+
* DuckDB's own signature check, so this script is the ONLY integrity
|
|
26
|
+
* gate on the wasm payload. `public/duckdb-extensions/checksums.json`
|
|
27
|
+
* pins a SHA-256 per `{version, platform, ext}`. Behaviour:
|
|
28
|
+
* - Missing checksum entry → fail. Operator must add the entry by
|
|
29
|
+
* hand at every DuckDB core version bump (reviewed in a diff).
|
|
30
|
+
* - Existing local file with matching hash → skip.
|
|
31
|
+
* - Existing local file with mismatched hash → refuse to overwrite
|
|
32
|
+
* and abort, so a poisoned mirror is loud, not silent.
|
|
33
|
+
* - Fresh download with mismatched hash → refuse to write to disk.
|
|
34
|
+
*
|
|
35
|
+
* The DuckDB CORE version bundled inside `@duckdb/duckdb-wasm` is pinned
|
|
36
|
+
* here. The script aborts loudly when the duckdb-wasm major / minor
|
|
37
|
+
* changes — a manual review (and probably a version bump here) is the
|
|
38
|
+
* right response, not a silent re-download against an unverified URL.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
import { createHash } from "node:crypto";
|
|
42
|
+
import { mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
|
|
43
|
+
import { dirname, resolve } from "node:path";
|
|
44
|
+
import { fileURLToPath } from "node:url";
|
|
45
|
+
|
|
46
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
47
|
+
const REPO_ROOT = resolve(HERE, "..");
|
|
48
|
+
|
|
49
|
+
const DUCKDB_WASM_PACKAGE = resolve(
|
|
50
|
+
REPO_ROOT,
|
|
51
|
+
"node_modules",
|
|
52
|
+
"@duckdb",
|
|
53
|
+
"duckdb-wasm",
|
|
54
|
+
"package.json",
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
// duckdb-wasm v1.32.x embeds DuckDB v1.4.3 (verified by `strings ./*.wasm`).
|
|
58
|
+
// Bump this constant when bumping duckdb-wasm to a release that targets a
|
|
59
|
+
// different DuckDB core.
|
|
60
|
+
const SUPPORTED_DUCKDB_WASM = "1.32.";
|
|
61
|
+
const DUCKDB_CORE_VERSION = "1.4.3";
|
|
62
|
+
|
|
63
|
+
const PLATFORMS = ["wasm_mvp", "wasm_eh", "wasm_threads"];
|
|
64
|
+
// `parquet` is autoloaded by DuckDB on the first `parquet_scan(...)` —
|
|
65
|
+
// must be mirrored alongside the explicitly INSTALLed extensions.
|
|
66
|
+
const EXTENSIONS = ["httpfs", "spatial", "parquet"];
|
|
67
|
+
|
|
68
|
+
const UPSTREAM_REPO = "https://extensions.duckdb.org";
|
|
69
|
+
const OUTPUT_ROOT = resolve(REPO_ROOT, "public", "duckdb-extensions");
|
|
70
|
+
const CHECKSUMS_FILE = resolve(OUTPUT_ROOT, "checksums.json");
|
|
71
|
+
|
|
72
|
+
function checkVersion() {
|
|
73
|
+
const pkg = JSON.parse(readFileSync(DUCKDB_WASM_PACKAGE, "utf8"));
|
|
74
|
+
const v = pkg.version ?? "";
|
|
75
|
+
if (!v.startsWith(SUPPORTED_DUCKDB_WASM)) {
|
|
76
|
+
console.error(
|
|
77
|
+
`[duckdb-extensions] @duckdb/duckdb-wasm@${v} is not in the supported range (${SUPPORTED_DUCKDB_WASM}*).`,
|
|
78
|
+
);
|
|
79
|
+
console.error(
|
|
80
|
+
"[duckdb-extensions] Update DUCKDB_CORE_VERSION in scripts/download-duckdb-extensions.mjs after verifying the new mapping with `strings node_modules/@duckdb/duckdb-wasm/dist/duckdb-eh.wasm | grep '^v1\\.'`.",
|
|
81
|
+
);
|
|
82
|
+
process.exit(1);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function loadChecksums() {
|
|
87
|
+
if (!existsSync(CHECKSUMS_FILE)) {
|
|
88
|
+
throw new Error(
|
|
89
|
+
`Missing ${CHECKSUMS_FILE}. The integrity manifest must exist before any download — populate it manually at every DuckDB core version bump.`,
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
const raw = JSON.parse(readFileSync(CHECKSUMS_FILE, "utf8"));
|
|
93
|
+
// Strip JSON-comment-style keys so callers can iterate values safely.
|
|
94
|
+
const entries = Object.fromEntries(Object.entries(raw).filter(([k]) => !k.startsWith("$")));
|
|
95
|
+
return entries;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function sha256(buffer) {
|
|
99
|
+
return createHash("sha256").update(buffer).digest("hex");
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function downloadOne({ url, destination, key, expectedHash }) {
|
|
103
|
+
if (existsSync(destination)) {
|
|
104
|
+
const localHash = sha256(readFileSync(destination));
|
|
105
|
+
if (localHash === expectedHash) {
|
|
106
|
+
return { destination, key, skipped: true };
|
|
107
|
+
}
|
|
108
|
+
throw new Error(
|
|
109
|
+
`[duckdb-extensions] integrity mismatch for ${key}: on-disk SHA-256 ${localHash} does not match checksums.json ${expectedHash}. Refusing to overwrite — delete the file manually after investigating.`,
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
const res = await fetch(url);
|
|
113
|
+
if (!res.ok) {
|
|
114
|
+
throw new Error(`Failed to download ${url} — ${res.status} ${res.statusText}`);
|
|
115
|
+
}
|
|
116
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
117
|
+
const downloadedHash = sha256(buf);
|
|
118
|
+
if (downloadedHash !== expectedHash) {
|
|
119
|
+
throw new Error(
|
|
120
|
+
`[duckdb-extensions] integrity mismatch for ${key}: downloaded SHA-256 ${downloadedHash} does not match checksums.json ${expectedHash}. Refusing to write to disk.`,
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
mkdirSync(dirname(destination), { recursive: true });
|
|
124
|
+
writeFileSync(destination, buf);
|
|
125
|
+
return { destination, key, skipped: false, bytes: buf.byteLength };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async function main() {
|
|
129
|
+
checkVersion();
|
|
130
|
+
const checksums = loadChecksums();
|
|
131
|
+
|
|
132
|
+
const tasks = [];
|
|
133
|
+
for (const platform of PLATFORMS) {
|
|
134
|
+
for (const ext of EXTENSIONS) {
|
|
135
|
+
const key = `v${DUCKDB_CORE_VERSION}/${platform}/${ext}`;
|
|
136
|
+
const expectedHash = checksums[key];
|
|
137
|
+
if (!expectedHash) {
|
|
138
|
+
// Fail loudly: an operator bumping the DuckDB version must
|
|
139
|
+
// populate checksums.json by hand so the diff documents the
|
|
140
|
+
// new trust anchor. Silent allow would defeat the integrity gate.
|
|
141
|
+
throw new Error(
|
|
142
|
+
`[duckdb-extensions] no checksum entry for ${key} in public/duckdb-extensions/checksums.json. Populate it manually after verifying the upstream binary, then re-run.`,
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
const path = `/v${DUCKDB_CORE_VERSION}/${platform}/${ext}.duckdb_extension.wasm`;
|
|
146
|
+
tasks.push({
|
|
147
|
+
url: `${UPSTREAM_REPO}${path}`,
|
|
148
|
+
destination: resolve(
|
|
149
|
+
OUTPUT_ROOT,
|
|
150
|
+
`v${DUCKDB_CORE_VERSION}`,
|
|
151
|
+
platform,
|
|
152
|
+
`${ext}.duckdb_extension.wasm`,
|
|
153
|
+
),
|
|
154
|
+
key,
|
|
155
|
+
expectedHash,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// L-C: parallelize. `Promise.all` does not preserve console-log order,
|
|
161
|
+
// but each line is self-describing (it carries the destination path),
|
|
162
|
+
// so an operator reading the output can still tell which file did what.
|
|
163
|
+
const results = await Promise.all(tasks.map(downloadOne));
|
|
164
|
+
|
|
165
|
+
let downloaded = 0;
|
|
166
|
+
let skipped = 0;
|
|
167
|
+
for (const result of results) {
|
|
168
|
+
if (result.skipped) {
|
|
169
|
+
skipped++;
|
|
170
|
+
} else {
|
|
171
|
+
downloaded++;
|
|
172
|
+
console.log(
|
|
173
|
+
`[duckdb-extensions] ${result.bytes} bytes → ${result.destination.replace(REPO_ROOT + "/", "")}`,
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if (downloaded > 0) {
|
|
179
|
+
console.log(
|
|
180
|
+
`[duckdb-extensions] downloaded ${downloaded} extension(s), skipped ${skipped} (already present, integrity verified)`,
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
main().catch((err) => {
|
|
186
|
+
console.error(`[duckdb-extensions] ${err.message}`);
|
|
187
|
+
process.exit(1);
|
|
188
|
+
});
|
package/scripts/prebuild.mjs
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/* global process */
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
3
|
+
// Workspace-only build prep:
|
|
4
|
+
// 1. Regenerate `@cytario/plugin-api`'s `src/version.ts` from the latest
|
|
5
|
+
// `plugin-api-v*` git tag.
|
|
6
|
+
// 2. Download the DuckDB-WASM extensions (httpfs, spatial) into
|
|
7
|
+
// `public/duckdb-extensions/` so the runtime can `INSTALL` them from
|
|
8
|
+
// the cytario origin instead of `extensions.duckdb.org`.
|
|
9
|
+
// Both steps become no-ops inside a published `@cytario/web` install where
|
|
10
|
+
// the workspace source is not shipped.
|
|
7
11
|
|
|
8
12
|
import { existsSync } from "node:fs";
|
|
9
13
|
import { spawnSync } from "node:child_process";
|
|
@@ -11,12 +15,16 @@ import { dirname, resolve } from "node:path";
|
|
|
11
15
|
import { fileURLToPath } from "node:url";
|
|
12
16
|
|
|
13
17
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
14
|
-
const TARGET = resolve(HERE, "..", "packages", "plugin-api", "scripts", "write-version.mjs");
|
|
15
18
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
+
function run(targetRelative) {
|
|
20
|
+
const target = resolve(HERE, "..", targetRelative);
|
|
21
|
+
if (!existsSync(target)) return 0;
|
|
22
|
+
const result = spawnSync(process.execPath, [target], { stdio: "inherit" });
|
|
23
|
+
return result.status ?? 0;
|
|
19
24
|
}
|
|
20
25
|
|
|
21
|
-
const
|
|
22
|
-
|
|
26
|
+
const versionStatus = run("packages/plugin-api/scripts/write-version.mjs");
|
|
27
|
+
if (versionStatus !== 0) process.exit(versionStatus);
|
|
28
|
+
|
|
29
|
+
const extensionsStatus = run("scripts/download-duckdb-extensions.mjs");
|
|
30
|
+
process.exit(extensionsStatus);
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
|
2
|
-
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
3
|
-
|
|
4
|
-
import { ConnectionConfig } from "~/.generated/client";
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Generate a presigned URL for an object in S3.
|
|
8
|
-
*/
|
|
9
|
-
export const getPresignedUrl = async (
|
|
10
|
-
connectionConfig: ConnectionConfig,
|
|
11
|
-
s3Client: S3Client,
|
|
12
|
-
key: string,
|
|
13
|
-
) => {
|
|
14
|
-
const command = new GetObjectCommand({ Bucket: connectionConfig.bucketName, Key: key });
|
|
15
|
-
|
|
16
|
-
const url = await getSignedUrl(s3Client, command, {
|
|
17
|
-
expiresIn: 60 * 60 * 1, // 1 hour
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
return url;
|
|
21
|
-
};
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
import { S3Client } from "@aws-sdk/client-s3";
|
|
2
|
-
import { Credentials } from "@aws-sdk/client-sts";
|
|
3
|
-
import crypto from "crypto";
|
|
4
|
-
import { LRUCache } from "lru-cache";
|
|
5
|
-
|
|
6
|
-
import { ConnectionConfig } from "~/.generated/client";
|
|
7
|
-
import { isAwsS3Endpoint } from "~/utils/s3Provider";
|
|
8
|
-
|
|
9
|
-
interface CacheEntry {
|
|
10
|
-
client: S3Client;
|
|
11
|
-
userId: string;
|
|
12
|
-
bucketName: string;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
/**
|
|
16
|
-
* User-scoped S3Client cache with TTL and LRU eviction.
|
|
17
|
-
* Prevents cross-user data leakage and handles credential expiration.
|
|
18
|
-
*/
|
|
19
|
-
const s3ClientCache = new LRUCache<string, CacheEntry>({
|
|
20
|
-
max: 10000,
|
|
21
|
-
ttl: 3600000, // 1 hour in milliseconds (matches typical STS credential TTL)
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Creates a unique cache key scoped to user, bucket, and credential identity.
|
|
26
|
-
*/
|
|
27
|
-
const createCacheKey = (userId: string, bucketName: string, credentials: Credentials): string => {
|
|
28
|
-
// Hash credentials to detect when they change (e.g., after refresh)
|
|
29
|
-
const credHash = crypto
|
|
30
|
-
.createHash("sha256")
|
|
31
|
-
.update(`${credentials.AccessKeyId}:${credentials.SecretAccessKey}:${credentials.SessionToken}`)
|
|
32
|
-
.digest("hex")
|
|
33
|
-
.substring(0, 16);
|
|
34
|
-
|
|
35
|
-
return `${userId}:${bucketName}:${credHash}`;
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
export const getS3Client = async (
|
|
39
|
-
connectionConfig: ConnectionConfig,
|
|
40
|
-
credentials: Credentials,
|
|
41
|
-
userId: string,
|
|
42
|
-
): Promise<S3Client> => {
|
|
43
|
-
const { bucketName, region, endpoint } = connectionConfig;
|
|
44
|
-
const { AccessKeyId, SecretAccessKey, SessionToken } = credentials;
|
|
45
|
-
|
|
46
|
-
if (!AccessKeyId || !SecretAccessKey) throw Error("No Credentials");
|
|
47
|
-
if (!userId) throw Error("User ID is required for S3Client cache");
|
|
48
|
-
|
|
49
|
-
// Check cache first
|
|
50
|
-
const key = createCacheKey(userId, bucketName, credentials);
|
|
51
|
-
const cachedEntry = s3ClientCache.get(key);
|
|
52
|
-
if (cachedEntry) {
|
|
53
|
-
return cachedEntry.client;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// Use default region if null
|
|
57
|
-
const actualRegion = region ?? "eu-central-1";
|
|
58
|
-
|
|
59
|
-
// Detect if this is AWS S3 or a compatible service (MinIO, etc.)
|
|
60
|
-
const isAwsS3 = isAwsS3Endpoint(endpoint);
|
|
61
|
-
|
|
62
|
-
const s3Client = new S3Client({
|
|
63
|
-
region: actualRegion,
|
|
64
|
-
// Only set endpoint for non-AWS S3 services
|
|
65
|
-
...(endpoint && !isAwsS3 ? { endpoint } : {}),
|
|
66
|
-
credentials: {
|
|
67
|
-
accessKeyId: AccessKeyId,
|
|
68
|
-
secretAccessKey: SecretAccessKey,
|
|
69
|
-
sessionToken: SessionToken,
|
|
70
|
-
},
|
|
71
|
-
// Only use path style for non-AWS S3 services (MinIO, etc.)
|
|
72
|
-
forcePathStyle: !isAwsS3,
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
// Cache the client
|
|
76
|
-
s3ClientCache.set(key, {
|
|
77
|
-
client: s3Client,
|
|
78
|
-
userId,
|
|
79
|
-
bucketName,
|
|
80
|
-
});
|
|
81
|
-
|
|
82
|
-
return s3Client;
|
|
83
|
-
};
|
|
84
|
-
|
|
85
|
-
/** Remove all cached S3 clients for a given user + bucket (e.g. after config change). */
|
|
86
|
-
export const invalidateS3ClientsForBucket = (userId: string, bucketName: string): void => {
|
|
87
|
-
for (const key of s3ClientCache.keys()) {
|
|
88
|
-
const entry = s3ClientCache.peek(key);
|
|
89
|
-
if (entry && entry.userId === userId && entry.bucketName === bucketName) {
|
|
90
|
-
s3ClientCache.delete(key);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
};
|