@cytario/web 2.1.7 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +1 -1
  2. package/app/.server/auth/README.md +0 -1
  3. package/app/.server/auth/authMiddleware.ts +6 -28
  4. package/app/.server/auth/getSessionCredentials.ts +11 -1
  5. package/app/.server/auth/sessionPolicy.ts +69 -0
  6. package/app/.server/corsPreflight.ts +136 -0
  7. package/app/.server/csp.ts +34 -0
  8. package/app/components/.client/ImageViewer/README.md +2 -2
  9. package/app/components/DirectoryView/DirectoryViewGrid.tsx +2 -11
  10. package/app/components/DirectoryView/DirectoryViewTree.tsx +18 -26
  11. package/app/components/DirectoryView/buildDirectoryTree.ts +89 -42
  12. package/app/components/DirectoryView/filterNodes.ts +4 -19
  13. package/app/components/DirectoryView/useLazyTreeNodes.ts +120 -0
  14. package/app/components/GlobalSearch/GlobalSearch.tsx +8 -3
  15. package/app/components/GlobalSearch/Suggestions.tsx +21 -3
  16. package/app/config.ts +0 -7
  17. package/app/entry.server.tsx +9 -4
  18. package/app/hooks/useInitConnections.ts +3 -3
  19. package/app/root.tsx +11 -6
  20. package/app/routes/connections/connection.form.tsx +9 -3
  21. package/app/routes/connections/connection.schema.ts +60 -11
  22. package/app/routes/connections/connections.clientLoader.ts +91 -0
  23. package/app/routes/connections/connections.loader.ts +19 -39
  24. package/app/routes/connections/connections.route.tsx +5 -0
  25. package/app/routes/connections/createConnection.action.ts +39 -13
  26. package/app/routes/connections/deleteConnection.action.ts +5 -1
  27. package/app/routes/connections/updateConnection.action.ts +50 -13
  28. package/app/routes/home/home.route.tsx +5 -1
  29. package/app/routes/layouts/protected.layout.tsx +15 -1
  30. package/app/routes/objects/objects.clientLoader.ts +73 -0
  31. package/app/routes/objects/objects.loader.ts +58 -92
  32. package/app/routes/objects/objects.route.tsx +41 -40
  33. package/app/routes/search.route.tsx +133 -28
  34. package/app/routes.ts +0 -4
  35. package/app/utils/connectionsStore/useConnectionsStore.ts +25 -62
  36. package/app/utils/credentialsRefresh.ts +53 -0
  37. package/app/utils/db/convertCsvToParquet.ts +24 -16
  38. package/app/utils/db/createDatabase.ts +25 -33
  39. package/app/utils/db/duckdbBundles.ts +42 -0
  40. package/app/utils/db/ensureSpatialLoaded.ts +28 -0
  41. package/app/utils/db/escapeSqlString.ts +4 -0
  42. package/app/utils/db/getBlobFromObjectNode.ts +15 -36
  43. package/app/utils/db/getTileDataWasm.ts +4 -5
  44. package/app/utils/db/sqlQueries.ts +6 -1
  45. package/app/utils/filterObjects.ts +2 -18
  46. package/app/utils/limitConcurrency.ts +28 -0
  47. package/app/utils/listObjectsClient.ts +318 -0
  48. package/app/utils/listingLimits.ts +7 -0
  49. package/app/utils/loadConnectionLevel.ts +50 -0
  50. package/app/utils/localFilesStore/useFileStore.ts +1 -6
  51. package/app/utils/pathUtils.ts +65 -0
  52. package/app/utils/resourceId.ts +14 -29
  53. package/app/utils/s3HostAllowlist.ts +116 -0
  54. package/app/utils/signedFetch.ts +159 -24
  55. package/package.json +2 -2
  56. package/prisma/seed.ts +3 -3
  57. package/public/duckdb-extensions/checksums.json +12 -0
  58. package/scripts/download-duckdb-extensions.mjs +188 -0
  59. package/scripts/prebuild.mjs +18 -10
  60. package/app/.server/auth/getPresignedUrl.ts +0 -21
  61. package/app/.server/auth/getS3Client.ts +0 -93
  62. package/app/routes/presign.route.tsx +0 -42
  63. package/app/utils/getObjects.ts +0 -24
@@ -3,38 +3,116 @@ import type { Credentials } from "@aws-sdk/client-sts";
3
3
  import { SignatureV4 } from "@smithy/signature-v4";
4
4
 
5
5
  import type { ConnectionConfig } from "~/.generated/client";
6
+ import { ExpiredCredentialsError, requestCredentialsRefresh } from "~/utils/credentialsRefresh";
6
7
  import { sanitizeHeaders } from "~/utils/sanitizeHeaders";
7
8
 
8
9
  export type SignedFetch = (url: string, init?: RequestInit) => Promise<Response>;
9
10
 
10
- // Image tile/chunk bytes are immutable per object version → 7-day cache.
11
+ /**
12
+ * Lowercase header names the SigV4 signer emits. The CORS probe lists these
13
+ * in `Access-Control-Request-Headers` so the OPTIONS preflight carries the
14
+ * same set as the eventual GET. Keep in sync with `signer.sign` below.
15
+ */
16
+ export const SIGNED_REQUEST_HEADERS = [
17
+ "authorization",
18
+ "x-amz-content-sha256",
19
+ "x-amz-date",
20
+ "x-amz-security-token",
21
+ ] as const;
22
+
23
+ /**
24
+ * Thrown when a browser fetch fails before a Response — almost always a
25
+ * CORS misconfiguration on the bucket. Caller error paths instanceof-check
26
+ * this to surface a dedicated toast.
27
+ */
28
+ export class CorsLikelyError extends Error {
29
+ public readonly host: string;
30
+ public readonly origin: string;
31
+
32
+ constructor(host: string, origin: string, cause?: unknown) {
33
+ super(
34
+ `Browser was blocked from reading "${host}" — likely a CORS misconfiguration on the bucket.`,
35
+ );
36
+ this.name = "CorsLikelyError";
37
+ this.host = host;
38
+ this.origin = origin;
39
+ if (cause !== undefined) {
40
+ (this as { cause?: unknown }).cause = cause;
41
+ }
42
+ }
43
+ }
44
+
45
+ function isLikelyCorsFailure(error: unknown): boolean {
46
+ if (!(error instanceof TypeError)) return false;
47
+ const message = error.message ?? "";
48
+ return (
49
+ message.includes("Failed to fetch") ||
50
+ message.includes("Load failed") ||
51
+ message.includes("NetworkError when attempting to fetch resource") ||
52
+ message.includes("NetworkError")
53
+ );
54
+ }
55
+
56
+ async function fetchWithCorsDetection(
57
+ url: string,
58
+ init: RequestInit,
59
+ host: string,
60
+ origin: string,
61
+ ): Promise<Response> {
62
+ try {
63
+ return await fetch(url, init);
64
+ } catch (error) {
65
+ if (isLikelyCorsFailure(error)) {
66
+ throw new CorsLikelyError(host, origin, error);
67
+ }
68
+ throw error;
69
+ }
70
+ }
71
+
72
+ /**
73
+ * Detect expired-token bodies on a cloned response (the caller still owns the
74
+ * unread body). AWS: 400 + `ExpiredToken`; MinIO: 403 + `ExpiredTokenException`.
75
+ */
76
+ async function isExpiredTokenResponse(response: Response): Promise<boolean> {
77
+ if (response.status !== 400 && response.status !== 403) return false;
78
+ try {
79
+ const body = await response.clone().text();
80
+ return (
81
+ body.includes("<Code>ExpiredToken</Code>") ||
82
+ body.includes("<Code>ExpiredTokenException</Code>")
83
+ );
84
+ } catch {
85
+ return false;
86
+ }
87
+ }
88
+
89
+ // Tile/chunk bytes are immutable per object version → 7-day cache.
11
90
  const IMAGE_DATA_CACHE_CONTROL = "private, max-age=604800";
12
91
 
13
- // Sidecars, overlays, JSON companions → 1-hour ceiling so analyst
92
+ // Sidecars / overlays / JSON companions → 1-hour ceiling so analyst
14
93
  // regenerations surface without an explicit purge.
15
94
  const OTHER_DATA_CACHE_CONTROL = "private, max-age=3600";
16
95
 
17
- // TIFF/OME-TIFF reads, or OME-Zarr chunks whose last path segment is digits
18
- // only (`image.zarr/0/0/0` or `image.zarr/0.0.0`).
96
+ // Matches TIFF/OME-TIFF reads and OME-Zarr chunk filenames (digits-only).
19
97
  function isImageDataPath(pathname: string): boolean {
20
98
  return /\.tiff?$/i.test(pathname) || /\/\d+(?:\.\d+)*$/.test(pathname);
21
99
  }
22
100
 
23
101
  /**
24
- * SigV4-signing fetch. Credentials are resolved lazily per call so the signer
25
- * is recreated when AccessKeyId rotates. Every GET injects a
26
- * `response-cache-control` query override so the storage backend echoes a
27
- * Cache-Control directive — the browser HTTP cache then serves repeat reads
28
- * without a network round-trip.
102
+ * SigV4-signing fetch. Credentials resolve lazily so the signer is rebuilt
103
+ * when `AccessKeyId` rotates. Every GET injects `response-cache-control` so
104
+ * the browser HTTP cache can serve repeat reads without a network round-trip.
29
105
  */
30
106
  export function createSignedFetch(
31
107
  getCredentials: () => Credentials,
32
108
  connectionConfig: Pick<ConnectionConfig, "region">,
109
+ connectionName?: string,
33
110
  ): SignedFetch {
34
111
  let cachedKeyId: string | undefined;
35
112
  let signer: SignatureV4;
36
113
 
37
- return async (url: string, init?: RequestInit): Promise<Response> => {
114
+ // Closured so the ExpiredToken retry path can rebuild with refreshed creds.
115
+ const buildSignedRequest = async (url: string, init?: RequestInit) => {
38
116
  const credentials = getCredentials();
39
117
 
40
118
  if (!credentials.AccessKeyId || !credentials.SecretAccessKey) {
@@ -51,33 +129,34 @@ export function createSignedFetch(
51
129
  region: connectionConfig.region || "eu-central-1",
52
130
  service: "s3",
53
131
  sha256: Sha256,
132
+ // S3 paths are pre-encoded; the signer must not double-encode.
133
+ uriEscapePath: false,
54
134
  });
55
135
  cachedKeyId = credentials.AccessKeyId;
56
136
  }
57
137
 
58
138
  const parsed = new URL(url);
59
139
 
60
- // Decode so the signer can re-encode canonically — without this,
61
- // percent-encoded chars get double-encoded and the signature breaks.
62
- const decodedPath = decodeURIComponent(parsed.pathname);
63
-
64
140
  const cacheControl = isImageDataPath(parsed.pathname)
65
141
  ? IMAGE_DATA_CACHE_CONTROL
66
142
  : OTHER_DATA_CACHE_CONTROL;
67
143
 
68
- // Only the host header is signed Range etc. pass through unsigned to
69
- // avoid CORS/signature mismatches. Caller-supplied headers run through
70
- // sanitizeHeaders, and the merge order below puts signed headers LAST
71
- // so a bypass of sanitizeHeaders still cannot override the signature.
144
+ // Signed headers merge LAST so a bypass of `sanitizeHeaders` cannot
145
+ // override the signature.
72
146
  const callerHeaders = sanitizeHeaders(init?.headers as Record<string, string> | undefined);
73
147
 
148
+ // Pre-populate the signer's query map so caller-supplied params
149
+ // (e.g. `?versionId=...`) survive — otherwise the wire URL drops them.
150
+ const query: Record<string, string> = Object.fromEntries(parsed.searchParams);
151
+ query["response-cache-control"] = cacheControl;
152
+
74
153
  const request = {
75
154
  method: (init?.method as string) ?? "GET",
76
155
  protocol: parsed.protocol,
77
156
  hostname: parsed.hostname,
78
157
  port: parsed.port ? parseInt(parsed.port) : undefined,
79
- path: decodedPath,
80
- query: { "response-cache-control": cacheControl },
158
+ path: parsed.pathname,
159
+ query,
81
160
  headers: {
82
161
  host: parsed.host,
83
162
  },
@@ -85,9 +164,24 @@ export function createSignedFetch(
85
164
 
86
165
  const signed = await signer.sign(request);
87
166
 
88
- const wireUrl = `${parsed.origin}${parsed.pathname}?response-cache-control=${encodeURIComponent(cacheControl)}`;
89
-
90
- return fetch(wireUrl, {
167
+ // RFC-3986 strict encoding — `URLSearchParams.toString()` form-encodes
168
+ // and would break the signature on reserved characters.
169
+ const wireQuery = Object.entries(query)
170
+ .map(
171
+ ([key, value]) =>
172
+ `${encodeURIComponent(key).replace(/[!*'()]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}=${encodeURIComponent(
173
+ value,
174
+ ).replace(/[!*'()]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}`,
175
+ )
176
+ .join("&");
177
+ const wireUrl = `${parsed.origin}${parsed.pathname}?${wireQuery}`;
178
+
179
+ const browserOrigin =
180
+ typeof window !== "undefined" && typeof window.location?.origin === "string"
181
+ ? window.location.origin
182
+ : "";
183
+
184
+ const fetchInit: RequestInit = {
91
185
  ...init,
92
186
  method: request.method,
93
187
  headers: {
@@ -95,6 +189,47 @@ export function createSignedFetch(
95
189
  ...(signed.headers as Record<string, string>),
96
190
  },
97
191
  signal: init?.signal,
98
- });
192
+ // Block redirect-following — `fetch` would otherwise re-send the
193
+ // Authorization header (and the STS token) to whatever host the 30x
194
+ // points at.
195
+ redirect: "error",
196
+ };
197
+
198
+ return { wireUrl, fetchInit, host: parsed.host, browserOrigin };
199
+ };
200
+
201
+ return async (url: string, init?: RequestInit): Promise<Response> => {
202
+ const first = await buildSignedRequest(url, init);
203
+ const response = await fetchWithCorsDetection(
204
+ first.wireUrl,
205
+ first.fetchInit,
206
+ first.host,
207
+ first.browserOrigin,
208
+ );
209
+
210
+ if (await isExpiredTokenResponse(response)) {
211
+ if (!connectionName) {
212
+ throw new ExpiredCredentialsError(
213
+ "STS credentials expired and no connection name was provided to signedFetch.",
214
+ );
215
+ }
216
+ await requestCredentialsRefresh(connectionName);
217
+ const retried = await buildSignedRequest(url, init);
218
+ const retryResponse = await fetchWithCorsDetection(
219
+ retried.wireUrl,
220
+ retried.fetchInit,
221
+ retried.host,
222
+ retried.browserOrigin,
223
+ );
224
+ if (await isExpiredTokenResponse(retryResponse)) {
225
+ throw new ExpiredCredentialsError(
226
+ "STS credentials expired and refresh did not yield a working session.",
227
+ connectionName,
228
+ );
229
+ }
230
+ return retryResponse;
231
+ }
232
+
233
+ return response;
99
234
  };
100
235
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cytario/web",
3
- "version": "2.1.7",
3
+ "version": "2.2.0",
4
4
  "description": "Cytario Web — scientific imaging data browser and viewer for OME-TIFF, OME-Zarr, Parquet and GeoTIFF on S3-compatible storage.",
5
5
  "license": "AGPL-3.0",
6
6
  "sideEffects": false,
@@ -54,7 +54,7 @@
54
54
  "build:server": "tsup --config tsup.server.config.ts",
55
55
  "codegen:check": "tsx scripts/codegen-check.ts",
56
56
  "dev": "node bin/cytario-web.mjs dev",
57
- "predev": "if [ -L node_modules/@cytario/design ]; then npm unlink @cytario/design --no-save 2>/dev/null && npm install @cytario/design --quiet; fi",
57
+ "predev": "node scripts/prebuild.mjs && if [ -L node_modules/@cytario/design ]; then npm unlink @cytario/design --no-save 2>/dev/null && npm install @cytario/design --quiet; fi",
58
58
  "predev:design": "cd ../cytario-design && npm link --quiet && cd - > /dev/null && npm link @cytario/design --quiet && for p in react react-dom react-aria-components; do rm -rf ../cytario-design/node_modules/$p && ln -s \"$(pwd)/node_modules/$p\" ../cytario-design/node_modules/$p; done",
59
59
  "dev:design": "concurrently -n tsup,storybook,web -c blue,magenta,green \"cd ../cytario-design && npx tsup --watch\" \"cd ../cytario-design && npm run dev\" \"node bin/cytario-web.mjs dev\"",
60
60
  "lint": "eslint --cache --cache-location ./node_modules/.cache/eslint .",
package/prisma/seed.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  * Creates test fixtures in a fresh database. Runs automatically after
5
5
  * `prisma db push` or explicitly via `npx prisma db seed`.
6
6
  *
7
- * Both connections point to the same bucket (slashm-ultivue-exchange).
7
+ * Both connections point to the same bucket (shared-bucket-example).
8
8
  * The prefixed connection tests prefix-based directory listing without
9
9
  * requiring a separate bucket or cross-bucket auth setup.
10
10
  */
@@ -22,7 +22,7 @@ const TEST_PREFIX_CONNECTION_NAME = process.env.E2E_PREFIX_CONNECTION_NAME || "E
22
22
  const SHARED_BUCKET = {
23
23
  ownerScope: "cytario",
24
24
  createdBy: "e2e-seed",
25
- bucketName: "slashm-ultivue-exchange",
25
+ bucketName: "shared-bucket-example",
26
26
  provider: "aws" as const,
27
27
  endpoint: "https://s3.eu-central-1.amazonaws.com",
28
28
  roleArn: "arn:aws:iam::727043715722:role/keycloack-aws-test-iam-role",
@@ -46,7 +46,7 @@ async function seed() {
46
46
  create: {
47
47
  ...SHARED_BUCKET,
48
48
  name: TEST_PREFIX_CONNECTION_NAME,
49
- prefix: "Ascent Pharma Group",
49
+ prefix: "Alpha Lab",
50
50
  },
51
51
  });
52
52
 
@@ -0,0 +1,12 @@
1
+ {
2
+ "$comment": "SHA-256 checksums for DuckDB-WASM extensions mirrored locally. Pinned by scripts/download-duckdb-extensions.mjs — every entry must be reviewed and re-populated by hand at each DuckDB core version bump. Keys are `v<version>/<platform>/<ext>`.",
3
+ "v1.4.3/wasm_eh/httpfs": "5e76d1fb0779c4803ebfd6fb49b80ff833c11b9a6f55208a701e63a491df1d5c",
4
+ "v1.4.3/wasm_eh/parquet": "22765c8f7dc741cda2b571a66ac7bb355295d7d69a6c37e5315b265672984f55",
5
+ "v1.4.3/wasm_eh/spatial": "04b776946da64a15a7b14501790c75093e38f876acc46b2922f0daeb6aaa1d60",
6
+ "v1.4.3/wasm_mvp/httpfs": "88e7d15f1c6ac066cca28b4ee1046486a5110783dea82c28fc4f973924f12498",
7
+ "v1.4.3/wasm_mvp/parquet": "0785c6c95d003eff4faa7b3b4b660f02c9c92f6d68d135ddf330d42e3a650600",
8
+ "v1.4.3/wasm_mvp/spatial": "7a745cfc5259f69b46f077bc6afeb7a6aefb8ef8d8b336bb0b770e5449708bb4",
9
+ "v1.4.3/wasm_threads/httpfs": "20b0a02de0b0a7de2a846bc40d5b504f031580949c7e5828c9da672a8961d241",
10
+ "v1.4.3/wasm_threads/parquet": "4bae942b3a6d2b68f92203408c2b45b66cb107ec3fa119e0f000f368810ca98b",
11
+ "v1.4.3/wasm_threads/spatial": "83ad79d02f1dc5d72a0d5d63fa4a49e9f2a9413e947e889956f3ffabaee095c6"
12
+ }
@@ -0,0 +1,188 @@
1
+ #!/usr/bin/env node
2
+ /* global console, process, fetch, Buffer */
3
+ /**
4
+ * Download the DuckDB-WASM extensions cytario relies on (httpfs, spatial)
5
+ * for every WASM build variant the runtime might pick, and write them
6
+ * under `public/duckdb-extensions/v<duckdb-version>/<platform>/...`.
7
+ *
8
+ * Why bundle:
9
+ * - Privacy: `INSTALL httpfs;` at session start otherwise leaks every
10
+ * user's IP + Referer to `extensions.duckdb.org` (Cloudflare). Same
11
+ * reason we removed the runtime `jsdelivr` fetch for the core WASM
12
+ * module (see `app/utils/db/duckdbBundles.ts`).
13
+ * - CSP: our `connect-src` allowlist intentionally excludes third-party
14
+ * CDNs; without local mirrors the page-load `INSTALL` blocks.
15
+ *
16
+ * Vite serves anything in `public/` from the document origin verbatim.
17
+ * `createDatabase.ts` points DuckDB at this local mirror via
18
+ * `SET custom_extension_repository='<origin>/duckdb-extensions/';`
19
+ * before issuing `INSTALL <ext>; LOAD <ext>;`. DuckDB then fetches
20
+ * `<repo>/v<version>/<platform>/<ext>.duckdb_extension.wasm`
21
+ * which matches the upstream `extensions.duckdb.org` layout.
22
+ *
23
+ * Supply-chain trust:
24
+ * `allowUnsignedExtensions: true` in `createDatabase.ts` disables
25
+ * DuckDB's own signature check, so this script is the ONLY integrity
26
+ * gate on the wasm payload. `public/duckdb-extensions/checksums.json`
27
+ * pins a SHA-256 per `{version, platform, ext}`. Behaviour:
28
+ * - Missing checksum entry → fail. Operator must add the entry by
29
+ * hand at every DuckDB core version bump (reviewed in a diff).
30
+ * - Existing local file with matching hash → skip.
31
+ * - Existing local file with mismatched hash → refuse to overwrite
32
+ * and abort, so a poisoned mirror is loud, not silent.
33
+ * - Fresh download with mismatched hash → refuse to write to disk.
34
+ *
35
+ * The DuckDB CORE version bundled inside `@duckdb/duckdb-wasm` is pinned
36
+ * here. The script aborts loudly when the duckdb-wasm major / minor
37
+ * changes — a manual review (and probably a version bump here) is the
38
+ * right response, not a silent re-download against an unverified URL.
39
+ */
40
+
41
+ import { createHash } from "node:crypto";
42
+ import { mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
43
+ import { dirname, resolve } from "node:path";
44
+ import { fileURLToPath } from "node:url";
45
+
46
+ const HERE = dirname(fileURLToPath(import.meta.url));
47
+ const REPO_ROOT = resolve(HERE, "..");
48
+
49
+ const DUCKDB_WASM_PACKAGE = resolve(
50
+ REPO_ROOT,
51
+ "node_modules",
52
+ "@duckdb",
53
+ "duckdb-wasm",
54
+ "package.json",
55
+ );
56
+
57
+ // duckdb-wasm v1.32.x embeds DuckDB v1.4.3 (verified by `strings ./*.wasm`).
58
+ // Bump this constant when bumping duckdb-wasm to a release that targets a
59
+ // different DuckDB core.
60
+ const SUPPORTED_DUCKDB_WASM = "1.32.";
61
+ const DUCKDB_CORE_VERSION = "1.4.3";
62
+
63
+ const PLATFORMS = ["wasm_mvp", "wasm_eh", "wasm_threads"];
64
+ // `parquet` is autoloaded by DuckDB on the first `parquet_scan(...)` —
65
+ // must be mirrored alongside the explicitly INSTALLed extensions.
66
+ const EXTENSIONS = ["httpfs", "spatial", "parquet"];
67
+
68
+ const UPSTREAM_REPO = "https://extensions.duckdb.org";
69
+ const OUTPUT_ROOT = resolve(REPO_ROOT, "public", "duckdb-extensions");
70
+ const CHECKSUMS_FILE = resolve(OUTPUT_ROOT, "checksums.json");
71
+
72
+ function checkVersion() {
73
+ const pkg = JSON.parse(readFileSync(DUCKDB_WASM_PACKAGE, "utf8"));
74
+ const v = pkg.version ?? "";
75
+ if (!v.startsWith(SUPPORTED_DUCKDB_WASM)) {
76
+ console.error(
77
+ `[duckdb-extensions] @duckdb/duckdb-wasm@${v} is not in the supported range (${SUPPORTED_DUCKDB_WASM}*).`,
78
+ );
79
+ console.error(
80
+ "[duckdb-extensions] Update DUCKDB_CORE_VERSION in scripts/download-duckdb-extensions.mjs after verifying the new mapping with `strings node_modules/@duckdb/duckdb-wasm/dist/duckdb-eh.wasm | grep '^v1\\.'`.",
81
+ );
82
+ process.exit(1);
83
+ }
84
+ }
85
+
86
+ function loadChecksums() {
87
+ if (!existsSync(CHECKSUMS_FILE)) {
88
+ throw new Error(
89
+ `Missing ${CHECKSUMS_FILE}. The integrity manifest must exist before any download — populate it manually at every DuckDB core version bump.`,
90
+ );
91
+ }
92
+ const raw = JSON.parse(readFileSync(CHECKSUMS_FILE, "utf8"));
93
+ // Strip JSON-comment-style keys so callers can iterate values safely.
94
+ const entries = Object.fromEntries(Object.entries(raw).filter(([k]) => !k.startsWith("$")));
95
+ return entries;
96
+ }
97
+
98
+ function sha256(buffer) {
99
+ return createHash("sha256").update(buffer).digest("hex");
100
+ }
101
+
102
+ async function downloadOne({ url, destination, key, expectedHash }) {
103
+ if (existsSync(destination)) {
104
+ const localHash = sha256(readFileSync(destination));
105
+ if (localHash === expectedHash) {
106
+ return { destination, key, skipped: true };
107
+ }
108
+ throw new Error(
109
+ `[duckdb-extensions] integrity mismatch for ${key}: on-disk SHA-256 ${localHash} does not match checksums.json ${expectedHash}. Refusing to overwrite — delete the file manually after investigating.`,
110
+ );
111
+ }
112
+ const res = await fetch(url);
113
+ if (!res.ok) {
114
+ throw new Error(`Failed to download ${url} — ${res.status} ${res.statusText}`);
115
+ }
116
+ const buf = Buffer.from(await res.arrayBuffer());
117
+ const downloadedHash = sha256(buf);
118
+ if (downloadedHash !== expectedHash) {
119
+ throw new Error(
120
+ `[duckdb-extensions] integrity mismatch for ${key}: downloaded SHA-256 ${downloadedHash} does not match checksums.json ${expectedHash}. Refusing to write to disk.`,
121
+ );
122
+ }
123
+ mkdirSync(dirname(destination), { recursive: true });
124
+ writeFileSync(destination, buf);
125
+ return { destination, key, skipped: false, bytes: buf.byteLength };
126
+ }
127
+
128
+ async function main() {
129
+ checkVersion();
130
+ const checksums = loadChecksums();
131
+
132
+ const tasks = [];
133
+ for (const platform of PLATFORMS) {
134
+ for (const ext of EXTENSIONS) {
135
+ const key = `v${DUCKDB_CORE_VERSION}/${platform}/${ext}`;
136
+ const expectedHash = checksums[key];
137
+ if (!expectedHash) {
138
+ // Fail loudly: an operator bumping the DuckDB version must
139
+ // populate checksums.json by hand so the diff documents the
140
+ // new trust anchor. Silent allow would defeat the integrity gate.
141
+ throw new Error(
142
+ `[duckdb-extensions] no checksum entry for ${key} in public/duckdb-extensions/checksums.json. Populate it manually after verifying the upstream binary, then re-run.`,
143
+ );
144
+ }
145
+ const path = `/v${DUCKDB_CORE_VERSION}/${platform}/${ext}.duckdb_extension.wasm`;
146
+ tasks.push({
147
+ url: `${UPSTREAM_REPO}${path}`,
148
+ destination: resolve(
149
+ OUTPUT_ROOT,
150
+ `v${DUCKDB_CORE_VERSION}`,
151
+ platform,
152
+ `${ext}.duckdb_extension.wasm`,
153
+ ),
154
+ key,
155
+ expectedHash,
156
+ });
157
+ }
158
+ }
159
+
160
+ // L-C: parallelize. `Promise.all` does not preserve console-log order,
161
+ // but each line is self-describing (it carries the destination path),
162
+ // so an operator reading the output can still tell which file did what.
163
+ const results = await Promise.all(tasks.map(downloadOne));
164
+
165
+ let downloaded = 0;
166
+ let skipped = 0;
167
+ for (const result of results) {
168
+ if (result.skipped) {
169
+ skipped++;
170
+ } else {
171
+ downloaded++;
172
+ console.log(
173
+ `[duckdb-extensions] ${result.bytes} bytes → ${result.destination.replace(REPO_ROOT + "/", "")}`,
174
+ );
175
+ }
176
+ }
177
+
178
+ if (downloaded > 0) {
179
+ console.log(
180
+ `[duckdb-extensions] downloaded ${downloaded} extension(s), skipped ${skipped} (already present, integrity verified)`,
181
+ );
182
+ }
183
+ }
184
+
185
+ main().catch((err) => {
186
+ console.error(`[duckdb-extensions] ${err.message}`);
187
+ process.exit(1);
188
+ });
@@ -1,9 +1,13 @@
1
1
  #!/usr/bin/env node
2
2
  /* global process */
3
- // Regenerate `@cytario/plugin-api`'s `src/version.ts` from the latest
4
- // `plugin-api-v*` git tag — but only when the workspace source is
5
- // present. Becomes a no-op inside a published `@cytario/web` install
6
- // where the workspace source is not shipped.
3
+ // Workspace-only build prep:
4
+ // 1. Regenerate `@cytario/plugin-api`'s `src/version.ts` from the latest
5
+ // `plugin-api-v*` git tag.
6
+ // 2. Download the DuckDB-WASM extensions (httpfs, spatial) into
7
+ // `public/duckdb-extensions/` so the runtime can `INSTALL` them from
8
+ // the cytario origin instead of `extensions.duckdb.org`.
9
+ // Both steps become no-ops inside a published `@cytario/web` install where
10
+ // the workspace source is not shipped.
7
11
 
8
12
  import { existsSync } from "node:fs";
9
13
  import { spawnSync } from "node:child_process";
@@ -11,12 +15,16 @@ import { dirname, resolve } from "node:path";
11
15
  import { fileURLToPath } from "node:url";
12
16
 
13
17
  const HERE = dirname(fileURLToPath(import.meta.url));
14
- const TARGET = resolve(HERE, "..", "packages", "plugin-api", "scripts", "write-version.mjs");
15
18
 
16
- if (!existsSync(TARGET)) {
17
- // Published @cytario/web tarball plugin-api source is not shipped.
18
- process.exit(0);
19
+ function run(targetRelative) {
20
+ const target = resolve(HERE, "..", targetRelative);
21
+ if (!existsSync(target)) return 0;
22
+ const result = spawnSync(process.execPath, [target], { stdio: "inherit" });
23
+ return result.status ?? 0;
19
24
  }
20
25
 
21
- const result = spawnSync(process.execPath, [TARGET], { stdio: "inherit" });
22
- process.exit(result.status ?? 0);
26
+ const versionStatus = run("packages/plugin-api/scripts/write-version.mjs");
27
+ if (versionStatus !== 0) process.exit(versionStatus);
28
+
29
+ const extensionsStatus = run("scripts/download-duckdb-extensions.mjs");
30
+ process.exit(extensionsStatus);
@@ -1,21 +0,0 @@
1
- import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
2
- import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
3
-
4
- import { ConnectionConfig } from "~/.generated/client";
5
-
6
- /**
7
- * Generate a presigned URL for an object in S3.
8
- */
9
- export const getPresignedUrl = async (
10
- connectionConfig: ConnectionConfig,
11
- s3Client: S3Client,
12
- key: string,
13
- ) => {
14
- const command = new GetObjectCommand({ Bucket: connectionConfig.bucketName, Key: key });
15
-
16
- const url = await getSignedUrl(s3Client, command, {
17
- expiresIn: 60 * 60 * 1, // 1 hour
18
- });
19
-
20
- return url;
21
- };
@@ -1,93 +0,0 @@
1
- import { S3Client } from "@aws-sdk/client-s3";
2
- import { Credentials } from "@aws-sdk/client-sts";
3
- import crypto from "crypto";
4
- import { LRUCache } from "lru-cache";
5
-
6
- import { ConnectionConfig } from "~/.generated/client";
7
- import { isAwsS3Endpoint } from "~/utils/s3Provider";
8
-
9
- interface CacheEntry {
10
- client: S3Client;
11
- userId: string;
12
- bucketName: string;
13
- }
14
-
15
- /**
16
- * User-scoped S3Client cache with TTL and LRU eviction.
17
- * Prevents cross-user data leakage and handles credential expiration.
18
- */
19
- const s3ClientCache = new LRUCache<string, CacheEntry>({
20
- max: 10000,
21
- ttl: 3600000, // 1 hour in milliseconds (matches typical STS credential TTL)
22
- });
23
-
24
- /**
25
- * Creates a unique cache key scoped to user, bucket, and credential identity.
26
- */
27
- const createCacheKey = (userId: string, bucketName: string, credentials: Credentials): string => {
28
- // Hash credentials to detect when they change (e.g., after refresh)
29
- const credHash = crypto
30
- .createHash("sha256")
31
- .update(`${credentials.AccessKeyId}:${credentials.SecretAccessKey}:${credentials.SessionToken}`)
32
- .digest("hex")
33
- .substring(0, 16);
34
-
35
- return `${userId}:${bucketName}:${credHash}`;
36
- };
37
-
38
- export const getS3Client = async (
39
- connectionConfig: ConnectionConfig,
40
- credentials: Credentials,
41
- userId: string,
42
- ): Promise<S3Client> => {
43
- const { bucketName, region, endpoint } = connectionConfig;
44
- const { AccessKeyId, SecretAccessKey, SessionToken } = credentials;
45
-
46
- if (!AccessKeyId || !SecretAccessKey) throw Error("No Credentials");
47
- if (!userId) throw Error("User ID is required for S3Client cache");
48
-
49
- // Check cache first
50
- const key = createCacheKey(userId, bucketName, credentials);
51
- const cachedEntry = s3ClientCache.get(key);
52
- if (cachedEntry) {
53
- return cachedEntry.client;
54
- }
55
-
56
- // Use default region if null
57
- const actualRegion = region ?? "eu-central-1";
58
-
59
- // Detect if this is AWS S3 or a compatible service (MinIO, etc.)
60
- const isAwsS3 = isAwsS3Endpoint(endpoint);
61
-
62
- const s3Client = new S3Client({
63
- region: actualRegion,
64
- // Only set endpoint for non-AWS S3 services
65
- ...(endpoint && !isAwsS3 ? { endpoint } : {}),
66
- credentials: {
67
- accessKeyId: AccessKeyId,
68
- secretAccessKey: SecretAccessKey,
69
- sessionToken: SessionToken,
70
- },
71
- // Only use path style for non-AWS S3 services (MinIO, etc.)
72
- forcePathStyle: !isAwsS3,
73
- });
74
-
75
- // Cache the client
76
- s3ClientCache.set(key, {
77
- client: s3Client,
78
- userId,
79
- bucketName,
80
- });
81
-
82
- return s3Client;
83
- };
84
-
85
- /** Remove all cached S3 clients for a given user + bucket (e.g. after config change). */
86
- export const invalidateS3ClientsForBucket = (userId: string, bucketName: string): void => {
87
- for (const key of s3ClientCache.keys()) {
88
- const entry = s3ClientCache.peek(key);
89
- if (entry && entry.userId === userId && entry.bucketName === bucketName) {
90
- s3ClientCache.delete(key);
91
- }
92
- }
93
- };