@malloy-publisher/server 0.0.204 → 0.0.205
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.ts +10 -1
- package/dist/app/api-doc.yaml +133 -4
- package/dist/app/assets/{EnvironmentPage-CX06cjOF.js → EnvironmentPage-CAge6UHD.js} +1 -1
- package/dist/app/assets/HomePage-DhTe8qpa.js +1 -0
- package/dist/app/assets/{MainPage-nUJ9YatG.js → MainPage-CeTxxGex.js} +2 -2
- package/dist/app/assets/MaterializationsPage-CpDHB70t.js +1 -0
- package/dist/app/assets/ModelPage-D9sSMb75.js +1 -0
- package/dist/app/assets/{PackagePage-BaEVdEAG.js → PackagePage-LRqQWrFY.js} +1 -1
- package/dist/app/assets/{RouteError-BShQjZio.js → RouteError-xT6kuCNw.js} +1 -1
- package/dist/app/assets/{WorkbookPage-CBn6ZjJW.js → WorkbookPage-DsIh9svZ.js} +1 -1
- package/dist/app/assets/{core-DECXYL4E.es-OaRfXwuQ.js → core-C2sQrwVu.es-Bjem0hym.js} +1 -1
- package/dist/app/assets/{index-BLfPC1gy.js → index-BdOZDcce.js} +1 -1
- package/dist/app/assets/{index-Dy3YhAZQ.js → index-DHHAcY5o.js} +1 -1
- package/dist/app/assets/{index-DqiJ0bWp.js → index-RX3QOTde.js} +121 -121
- package/dist/app/assets/{index.umd-DAN9K8yC.js → index.umd-D2WH3D-f.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/runtime/publisher.js +318 -0
- package/dist/server.mjs +567 -194
- package/package.json +5 -4
- package/scripts/bake-duckdb-extensions.js +104 -0
- package/src/controller/watch-mode.controller.ts +176 -46
- package/src/errors.spec.ts +21 -0
- package/src/mcp/error_messages.spec.ts +35 -0
- package/src/mcp/error_messages.ts +14 -1
- package/src/mcp/handler_utils.ts +12 -0
- package/src/runtime/publisher.js +318 -0
- package/src/server.ts +479 -2
- package/src/service/authorize_integration.spec.ts +96 -2
- package/src/service/compile_authorize.spec.ts +85 -0
- package/src/service/environment.ts +63 -5
- package/src/service/environment_store.ts +142 -11
- package/src/service/model.ts +44 -0
- package/src/service/package.ts +17 -6
- package/src/storage/duckdb/DuckDBConnection.ts +70 -124
- package/tests/fixtures/authorize-compile/model.malloy +9 -0
- package/tests/fixtures/authorize-compile/publisher.json +4 -0
- package/tests/fixtures/html-pages-nopublic/model.malloy +1 -0
- package/tests/fixtures/html-pages-nopublic/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/data.csv +3 -0
- package/tests/fixtures/html-pages-test/public/assets/app.css +3 -0
- package/tests/fixtures/html-pages-test/public/data.json +1 -0
- package/tests/fixtures/html-pages-test/public/index.html +9 -0
- package/tests/fixtures/html-pages-test/public/sub/page2.html +9 -0
- package/tests/fixtures/html-pages-test/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/report.malloy +1 -0
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +92 -0
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +138 -0
- package/tests/integration/html_pages/html_pages.integration.spec.ts +378 -0
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +421 -0
- package/tests/unit/duckdb/attached_databases.test.ts +111 -0
- package/tests/unit/duckdb/duckdb_connection.test.ts +181 -0
- package/tests/unit/duckdb/repositories.test.ts +208 -0
- package/dist/app/assets/HomePage-CNFt_eUU.js +0 -1
- package/dist/app/assets/MaterializationsPage-B5goxVXW.js +0 -1
- package/dist/app/assets/ModelPage-Ba7Xh4lL.js +0 -1
package/src/server.ts
CHANGED
|
@@ -47,6 +47,7 @@ import { ManifestService } from "./service/manifest_service";
|
|
|
47
47
|
import { MaterializationService } from "./service/materialization_service";
|
|
48
48
|
import { normalizeQueryArray } from "./query_param_utils";
|
|
49
49
|
import { PackageMemoryGovernor } from "./service/package_memory_governor";
|
|
50
|
+
import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
|
|
50
51
|
|
|
51
52
|
export { normalizeQueryArray } from "./query_param_utils";
|
|
52
53
|
|
|
@@ -85,6 +86,14 @@ function parseArgs() {
|
|
|
85
86
|
i++;
|
|
86
87
|
} else if (arg === "--init") {
|
|
87
88
|
process.env.INITIALIZE_STORAGE = "true";
|
|
89
|
+
} else if (arg === "--watch-env" && args[i + 1]) {
|
|
90
|
+
// Append (don't overwrite) so multiple --watch-env flags compose
|
|
91
|
+
// and so an explicit env var pre-set still wins.
|
|
92
|
+
const existing = process.env.PUBLISHER_WATCH || "";
|
|
93
|
+
process.env.PUBLISHER_WATCH = existing
|
|
94
|
+
? `${existing},${args[i + 1]}`
|
|
95
|
+
: args[i + 1];
|
|
96
|
+
i++;
|
|
88
97
|
} else if (arg === "--help" || arg === "-h") {
|
|
89
98
|
console.log("Malloy Publisher Server");
|
|
90
99
|
console.log("");
|
|
@@ -115,6 +124,21 @@ function parseArgs() {
|
|
|
115
124
|
console.log(
|
|
116
125
|
" --init Initialize the storage (default: false)",
|
|
117
126
|
);
|
|
127
|
+
console.log(
|
|
128
|
+
" --watch-env <name> Enable dev-mode watch for the named environment.",
|
|
129
|
+
);
|
|
130
|
+
console.log(
|
|
131
|
+
" Mounts local-dir packages in-place (symlink, not",
|
|
132
|
+
);
|
|
133
|
+
console.log(
|
|
134
|
+
" copy) so source-edit live reload works. A comma-",
|
|
135
|
+
);
|
|
136
|
+
console.log(
|
|
137
|
+
" separated PUBLISHER_WATCH mounts all listed envs in",
|
|
138
|
+
);
|
|
139
|
+
console.log(
|
|
140
|
+
" place, but only the first one auto-reloads.",
|
|
141
|
+
);
|
|
118
142
|
console.log(" --help, -h Show this help message");
|
|
119
143
|
process.exit(0);
|
|
120
144
|
}
|
|
@@ -286,6 +310,290 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
|
|
|
286
310
|
}
|
|
287
311
|
});
|
|
288
312
|
|
|
313
|
+
// ---------------------------------------------------------------------------
|
|
314
|
+
// In-package HTML data apps
|
|
315
|
+
// ---------------------------------------------------------------------------
|
|
316
|
+
// These routes must come before the SPA catch-all and (in dev) the Vite proxy
|
|
317
|
+
// so that:
|
|
318
|
+
// - `/sdk/publisher.js` → Publisher runtime helper
|
|
319
|
+
// - `/environments/<env>/packages/<pkg>/<file.ext>` → static file from
|
|
320
|
+
// inside the package dir
|
|
321
|
+
// - `/api/v0/.../events` → live-reload SSE (registered in API routes
|
|
322
|
+
// below; this comment is the cross-reference)
|
|
323
|
+
|
|
324
|
+
// Serve the runtime helper that in-package HTML pages load via
|
|
325
|
+
// <script src="/sdk/publisher.js">. Path resolved once at module load.
|
|
326
|
+
const PUBLISHER_RUNTIME_PATH = path.join(
|
|
327
|
+
path.dirname(__filename_esm),
|
|
328
|
+
"runtime",
|
|
329
|
+
"publisher.js",
|
|
330
|
+
);
|
|
331
|
+
app.get("/sdk/publisher.js", (_req, res) => {
|
|
332
|
+
res.type("application/javascript");
|
|
333
|
+
// Short cache so live edits during local dev show up quickly. In
|
|
334
|
+
// production this file is content-stable per release.
|
|
335
|
+
res.setHeader("cache-control", "public, max-age=60");
|
|
336
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
337
|
+
res.sendFile(PUBLISHER_RUNTIME_PATH, (err) => {
|
|
338
|
+
if (err) {
|
|
339
|
+
logger.error("Failed to send publisher.js runtime", { error: err });
|
|
340
|
+
if (!res.headersSent) res.status(500).end();
|
|
341
|
+
}
|
|
342
|
+
});
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
// Serve files from inside a package directory at
|
|
346
|
+
// /environments/<env>/packages/<pkg>/<relative-path>
|
|
347
|
+
//
|
|
348
|
+
// This route fully owns its prefix — it does NOT fall through to the SPA on
|
|
349
|
+
// missing files, because doing so would mask 404s (and in dev mode the SPA
|
|
350
|
+
// catch-all errors out before it can reply). Behavior:
|
|
351
|
+
// - `/environments/<env>/packages/<pkg>` → 302 to `…/<pkg>/`
|
|
352
|
+
// - `/environments/<env>/packages/<pkg>/` → serve `<pkgRoot>/public/index.html`
|
|
353
|
+
// - `/environments/<env>/packages/<pkg>/foo/` → serve `<pkgRoot>/public/foo/index.html`
|
|
354
|
+
// - `/environments/<env>/packages/<pkg>/<file>` → serve `<pkgRoot>/public/<file>`, or 404
|
|
355
|
+
// Only the package's `public/` directory is web-served. Models, data files, and
|
|
356
|
+
// the publisher.json manifest live outside it and are never reachable here, so
|
|
357
|
+
// nothing can be downloaded around the per-model #(authorize) and query
|
|
358
|
+
// controls. The data stays reachable through the permission-checked query path.
|
|
359
|
+
|
|
360
|
+
async function serveFromPackage(
|
|
361
|
+
req: express.Request,
|
|
362
|
+
res: express.Response,
|
|
363
|
+
): Promise<void> {
|
|
364
|
+
const subPathRaw = (req.params as Record<string, string>)["0"] ?? "";
|
|
365
|
+
try {
|
|
366
|
+
const environment = await environmentStore.getEnvironment(
|
|
367
|
+
req.params.environmentName,
|
|
368
|
+
false,
|
|
369
|
+
);
|
|
370
|
+
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
371
|
+
// Only the package's public/ directory is web-served. Models, data, and
|
|
372
|
+
// the publisher.json manifest live outside it and are never reachable
|
|
373
|
+
// through this route. This single directory boundary is the whole
|
|
374
|
+
// access-control story for static files.
|
|
375
|
+
const publicRoot = path.join(pkg.getPackagePath(), "public");
|
|
376
|
+
|
|
377
|
+
// Directory-style fallback: empty path or trailing slash → look for
|
|
378
|
+
// index.html within that directory.
|
|
379
|
+
let subPath = subPathRaw;
|
|
380
|
+
if (subPath === "" || subPath.endsWith("/")) {
|
|
381
|
+
subPath = subPath + "index.html";
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// Resolve the requested file under public/ and reject anything that
|
|
385
|
+
// escapes it (`..`, encoded traversal) before touching the disk.
|
|
386
|
+
// safeJoinUnderRoot is the shared lexical-containment primitive (it throws
|
|
387
|
+
// BadRequestError on escape, surfaced as 400 by the outer catch); the
|
|
388
|
+
// realpath check below additionally catches symlinks inside public/ that
|
|
389
|
+
// point outward (403).
|
|
390
|
+
const fullPath = safeJoinUnderRoot(publicRoot, subPath);
|
|
391
|
+
|
|
392
|
+
// Containment check via realpath against the resolved public/ root.
|
|
393
|
+
// Catches symlinks inside public/ that point out (e.g. a malicious
|
|
394
|
+
// package shipping `public/leak -> /etc/passwd`), and tolerates the
|
|
395
|
+
// package root itself being a symlink (how watch-mode in-place mount
|
|
396
|
+
// works): realpath resolves it transparently and legitimate accesses
|
|
397
|
+
// inside public/ stay within realPublicRoot. Missing public/ dir or
|
|
398
|
+
// missing file: realpath throws ENOENT and we 404 cleanly instead of
|
|
399
|
+
// leaking via Express's default error handler.
|
|
400
|
+
const fsp = await import("fs/promises");
|
|
401
|
+
let realPublicRoot: string;
|
|
402
|
+
let realFullPath: string;
|
|
403
|
+
try {
|
|
404
|
+
realPublicRoot = await fsp.realpath(publicRoot);
|
|
405
|
+
realFullPath = await fsp.realpath(fullPath);
|
|
406
|
+
} catch {
|
|
407
|
+
if (!res.headersSent) {
|
|
408
|
+
// Generic 404 with no reflected request input (avoids reflecting
|
|
409
|
+
// user-controlled path/package name into the response body).
|
|
410
|
+
res.status(404).end();
|
|
411
|
+
}
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
const rel = path.relative(realPublicRoot, realFullPath);
|
|
415
|
+
if (rel.startsWith("..") || path.isAbsolute(rel)) {
|
|
416
|
+
res.status(403).end();
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// Framing policy only applies to HTML documents — setting it on CSS/JS/
|
|
421
|
+
// image assets is meaningless and needlessly strips their default
|
|
422
|
+
// SAMEORIGIN protection. Embeddability defaults to "*" so same-tenant
|
|
423
|
+
// embeds work out of the box, and is overridable via PUBLISHER_FRAME_ANCESTORS.
|
|
424
|
+
const ext = path.extname(realFullPath).toLowerCase();
|
|
425
|
+
if (ext === ".html" || ext === ".htm") {
|
|
426
|
+
const frameAncestors = process.env.PUBLISHER_FRAME_ANCESTORS || "*";
|
|
427
|
+
res.setHeader(
|
|
428
|
+
"Content-Security-Policy",
|
|
429
|
+
`frame-ancestors ${frameAncestors}`,
|
|
430
|
+
);
|
|
431
|
+
res.removeHeader("X-Frame-Options");
|
|
432
|
+
}
|
|
433
|
+
// Never let a served asset be MIME-sniffed into a different content type.
|
|
434
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
435
|
+
res.sendFile(realFullPath, (err) => {
|
|
436
|
+
if (err) {
|
|
437
|
+
// Own the 404 instead of letting Express fall through to a
|
|
438
|
+
// catch-all that may error.
|
|
439
|
+
if (!res.headersSent) {
|
|
440
|
+
// Generic 404, no reflected request input (see above).
|
|
441
|
+
res.status(404).end();
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
});
|
|
445
|
+
} catch (e) {
|
|
446
|
+
// Map service errors to their real status — a bad package name is a 400,
|
|
447
|
+
// memory back-pressure is a 503 — rather than flattening everything to
|
|
448
|
+
// 404. A genuine missing file is already handled by the realpath/sendFile
|
|
449
|
+
// 404 paths above; this catch only sees service-layer failures.
|
|
450
|
+
if (!res.headersSent) {
|
|
451
|
+
const { json, status } = internalErrorToHttpError(e as Error);
|
|
452
|
+
res.status(status).json(json);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// `/environments/<env>/packages/<pkg>` (no trailing slash, no path) redirect so
|
|
458
|
+
// relative URLs in the served HTML resolve as expected. Express's default loose
|
|
459
|
+
// matching also catches the trailing-slash form here, so only redirect URLs that
|
|
460
|
+
// don't already end with `/`.
|
|
461
|
+
//
|
|
462
|
+
// Build the target from the validated route params and the parsed query, not
|
|
463
|
+
// from the raw request URL, so it is always this same canonical, same-origin
|
|
464
|
+
// path with a trailing slash. That removes any open-redirect / header-injection
|
|
465
|
+
// surface from user-controlled input, with the slash placed before any query
|
|
466
|
+
// string (e.g. ?embed_token=...).
|
|
467
|
+
app.get(
|
|
468
|
+
"/environments/:environmentName/packages/:packageName",
|
|
469
|
+
(req, res, next) => {
|
|
470
|
+
if (req.path.endsWith("/")) return next();
|
|
471
|
+
const canonical =
|
|
472
|
+
`/environments/${encodeURIComponent(req.params.environmentName)}` +
|
|
473
|
+
`/packages/${encodeURIComponent(req.params.packageName)}/`;
|
|
474
|
+
const query = new URLSearchParams();
|
|
475
|
+
for (const [key, value] of Object.entries(req.query)) {
|
|
476
|
+
if (Array.isArray(value)) {
|
|
477
|
+
for (const v of value) query.append(key, String(v));
|
|
478
|
+
} else if (value !== undefined) {
|
|
479
|
+
query.append(key, String(value));
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
const qs = query.toString();
|
|
483
|
+
res.redirect(308, qs ? `${canonical}?${qs}` : canonical);
|
|
484
|
+
},
|
|
485
|
+
);
|
|
486
|
+
|
|
487
|
+
app.get(
|
|
488
|
+
"/environments/:environmentName/packages/:packageName/*",
|
|
489
|
+
serveFromPackage,
|
|
490
|
+
);
|
|
491
|
+
|
|
492
|
+
// List the static HTML pages bundled inside a package. Used by the SPA's
|
|
493
|
+
// package-detail view to surface a clickable list, and by anyone who wants
|
|
494
|
+
// to discover pages programmatically without scraping the directory.
|
|
495
|
+
//
|
|
496
|
+
// Returns a `Page[]` (see api-doc.yaml) — each item carries the relative
|
|
497
|
+
// `path`, the `packageName`, the page `title` (from its <title> tag), and a
|
|
498
|
+
// `resource` URL. `resource` is the root-relative static-serve URL (NOT under
|
|
499
|
+
// `${API_PREFIX}`) because pages are static assets served off the server root,
|
|
500
|
+
// unlike API resources such as `Package.resource`.
|
|
501
|
+
// Recursive depth is capped to keep this cheap for huge package directories.
|
|
502
|
+
const PAGES_DEPTH_CAP = 3;
|
|
503
|
+
type PageItem = {
|
|
504
|
+
resource: string;
|
|
505
|
+
packageName: string;
|
|
506
|
+
path: string;
|
|
507
|
+
title: string;
|
|
508
|
+
};
|
|
509
|
+
async function listPackagePages(
|
|
510
|
+
environmentName: string,
|
|
511
|
+
packageName: string,
|
|
512
|
+
publicRoot: string,
|
|
513
|
+
): Promise<PageItem[]> {
|
|
514
|
+
const fs = await import("fs/promises");
|
|
515
|
+
const out: PageItem[] = [];
|
|
516
|
+
|
|
517
|
+
// Resolve the public/ root once and reject any entry whose realpath escapes
|
|
518
|
+
// it. Same containment defense as serveFromPackage: catches symlinks inside
|
|
519
|
+
// public/ pointing outside (e.g. `public/leak -> ../report.malloy`) before we
|
|
520
|
+
// open and read the target's first 4KB for title extraction. A package with
|
|
521
|
+
// no public/ dir fails realpath here and yields an empty list.
|
|
522
|
+
let realPublicRoot: string;
|
|
523
|
+
try {
|
|
524
|
+
realPublicRoot = await fs.realpath(publicRoot);
|
|
525
|
+
} catch {
|
|
526
|
+
return out;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
async function walk(dir: string, depth: number) {
|
|
530
|
+
if (depth > PAGES_DEPTH_CAP) return;
|
|
531
|
+
let entries: import("fs").Dirent[];
|
|
532
|
+
try {
|
|
533
|
+
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
534
|
+
} catch {
|
|
535
|
+
return;
|
|
536
|
+
}
|
|
537
|
+
for (const entry of entries) {
|
|
538
|
+
if (entry.name.startsWith(".") || entry.name === "node_modules")
|
|
539
|
+
continue;
|
|
540
|
+
const full = path.join(dir, entry.name);
|
|
541
|
+
let realFull: string;
|
|
542
|
+
try {
|
|
543
|
+
realFull = await fs.realpath(full);
|
|
544
|
+
} catch {
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
const contained = path.relative(realPublicRoot, realFull);
|
|
548
|
+
if (contained.startsWith("..") || path.isAbsolute(contained)) continue;
|
|
549
|
+
if (entry.isDirectory()) {
|
|
550
|
+
await walk(full, depth + 1);
|
|
551
|
+
} else if (
|
|
552
|
+
entry.isFile() &&
|
|
553
|
+
(entry.name.endsWith(".html") || entry.name.endsWith(".htm"))
|
|
554
|
+
) {
|
|
555
|
+
const rel = path.relative(publicRoot, full).replace(/\\/g, "/");
|
|
556
|
+
// Cheap title extraction: read first 4KB and grep for <title>.
|
|
557
|
+
let title = rel;
|
|
558
|
+
try {
|
|
559
|
+
const fh = await fs.open(full, "r");
|
|
560
|
+
try {
|
|
561
|
+
const buf = Buffer.alloc(4096);
|
|
562
|
+
const { bytesRead } = await fh.read(buf, 0, 4096, 0);
|
|
563
|
+
const head = buf.slice(0, bytesRead).toString("utf8");
|
|
564
|
+
const m = head.match(/<title[^>]*>([^<]+)<\/title>/i);
|
|
565
|
+
if (m) title = m[1].trim();
|
|
566
|
+
} finally {
|
|
567
|
+
await fh.close();
|
|
568
|
+
}
|
|
569
|
+
} catch {
|
|
570
|
+
// ignore; fall back to relative path as title
|
|
571
|
+
}
|
|
572
|
+
out.push({
|
|
573
|
+
resource: `/environments/${environmentName}/packages/${packageName}/${rel}`,
|
|
574
|
+
packageName,
|
|
575
|
+
path: rel,
|
|
576
|
+
title,
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
await walk(publicRoot, 0);
|
|
583
|
+
out.sort((a, b) => {
|
|
584
|
+
// Surface index.html first, then alphabetical.
|
|
585
|
+
if (a.path === "index.html") return -1;
|
|
586
|
+
if (b.path === "index.html") return 1;
|
|
587
|
+
return a.path.localeCompare(b.path);
|
|
588
|
+
});
|
|
589
|
+
return out;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
// NOTE: route registration for /pages moved below the CORS middleware so
|
|
593
|
+
// cross-origin SDK consumers (e.g. a customer's React app pointing at
|
|
594
|
+
// `<ServerProvider baseURL="https://publisher.example.com/api/v0">`) get
|
|
595
|
+
// the proper CORS headers. See the registration after `app.use(cors(...))`.
|
|
596
|
+
|
|
289
597
|
// Only serve static files in production mode
|
|
290
598
|
// Otherwise we proxy to the React dev server
|
|
291
599
|
if (!isDevelopment) {
|
|
@@ -343,6 +651,34 @@ try {
|
|
|
343
651
|
// Register draining guard middleware - must be after health endpoints but before other routes
|
|
344
652
|
app.use(drainingGuard);
|
|
345
653
|
|
|
654
|
+
// /pages — registered here (post-CORS, post-body-parser, post-draining) so
|
|
655
|
+
// cross-origin SDK consumers and authenticated requests both work.
|
|
656
|
+
app.get(
|
|
657
|
+
`${API_PREFIX}/environments/:environmentName/packages/:packageName/pages`,
|
|
658
|
+
async (req, res) => {
|
|
659
|
+
try {
|
|
660
|
+
const environment = await environmentStore.getEnvironment(
|
|
661
|
+
req.params.environmentName,
|
|
662
|
+
false,
|
|
663
|
+
);
|
|
664
|
+
const pkg = await environment.getPackage(
|
|
665
|
+
req.params.packageName,
|
|
666
|
+
false,
|
|
667
|
+
);
|
|
668
|
+
const pages = await listPackagePages(
|
|
669
|
+
req.params.environmentName,
|
|
670
|
+
req.params.packageName,
|
|
671
|
+
path.join(pkg.getPackagePath(), "public"),
|
|
672
|
+
);
|
|
673
|
+
res.json(pages);
|
|
674
|
+
} catch (error) {
|
|
675
|
+
logger.error("Failed to list package pages", { error });
|
|
676
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
677
|
+
res.status(status).json(json);
|
|
678
|
+
}
|
|
679
|
+
},
|
|
680
|
+
);
|
|
681
|
+
|
|
346
682
|
app.get(`${API_PREFIX}/status`, async (_req, res) => {
|
|
347
683
|
try {
|
|
348
684
|
const status = await environmentStore.getStatus();
|
|
@@ -358,6 +694,86 @@ app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);
|
|
|
358
694
|
app.post(`${API_PREFIX}/watch-mode/start`, watchModeController.startWatching);
|
|
359
695
|
app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
|
|
360
696
|
|
|
697
|
+
// Live-reload Server-Sent Events stream for in-package HTML dashboards.
|
|
698
|
+
//
|
|
699
|
+
// This endpoint does NOT start watch mode on its own — that's an explicit
|
|
700
|
+
// opt-in (`--watch-env <name>` CLI flag, or `POST /api/v0/watch-mode/start`).
|
|
701
|
+
// Instead it reports whether watch mode is currently active for the requested
|
|
702
|
+
// env via a `mode` event and, if so, fans out file-change events to the
|
|
703
|
+
// browser. This avoids two earlier bugs:
|
|
704
|
+
// - Auto-starting from the request handler made arbitrary fetches reach
|
|
705
|
+
// in to mutate global watch-mode state (`event traversal — see below).
|
|
706
|
+
// - The runtime previously had no way to know "watch mode isn't running,
|
|
707
|
+
// don't expect reloads"; with the `mode` event it can choose to surface
|
|
708
|
+
// a small dev indicator (today: silent).
|
|
709
|
+
//
|
|
710
|
+
// Inputs are validated before any state lookup. Names that don't pass the
|
|
711
|
+
// canonical `assertSafePackageName` allowlist get 400 — preventing requests
|
|
712
|
+
// like `/api/v0/environments/%2e%2e/packages/x/events` from reaching the
|
|
713
|
+
// EnvironmentStore at all. We reuse the shared sanitizer rather than a local
|
|
714
|
+
// regex so the rules stay in one place (see path_safety.ts).
|
|
715
|
+
// Cap concurrent live-reload SSE connections so the endpoint can't be used to
|
|
716
|
+
// exhaust server sockets/memory with unbounded long-lived streams. Generous,
|
|
717
|
+
// since legitimate use is one stream per open dashboard tab.
|
|
718
|
+
const MAX_SSE_CONNECTIONS = 1000;
|
|
719
|
+
let sseConnectionCount = 0;
|
|
720
|
+
app.get(
|
|
721
|
+
`${API_PREFIX}/environments/:environmentName/packages/:packageName/events`,
|
|
722
|
+
async (req, res) => {
|
|
723
|
+
const env = req.params.environmentName;
|
|
724
|
+
const pkg = req.params.packageName;
|
|
725
|
+
try {
|
|
726
|
+
assertSafePackageName(env);
|
|
727
|
+
assertSafePackageName(pkg);
|
|
728
|
+
const environment = await environmentStore.getEnvironment(env, false);
|
|
729
|
+
await environment.getPackage(pkg, false); // 404 if missing
|
|
730
|
+
} catch (error) {
|
|
731
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
732
|
+
res.status(status).json(json);
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
if (sseConnectionCount >= MAX_SSE_CONNECTIONS) {
|
|
737
|
+
res.status(503).json({
|
|
738
|
+
code: 503,
|
|
739
|
+
message: "Too many live-reload connections; try again shortly.",
|
|
740
|
+
});
|
|
741
|
+
return;
|
|
742
|
+
}
|
|
743
|
+
sseConnectionCount++;
|
|
744
|
+
|
|
745
|
+
res.set({
|
|
746
|
+
"content-type": "text/event-stream",
|
|
747
|
+
"cache-control": "no-cache",
|
|
748
|
+
connection: "keep-alive",
|
|
749
|
+
// Disable proxy/CDN buffering so events flush immediately.
|
|
750
|
+
"x-accel-buffering": "no",
|
|
751
|
+
});
|
|
752
|
+
res.flushHeaders();
|
|
753
|
+
|
|
754
|
+
const watching = watchModeController.isWatching(env);
|
|
755
|
+
res.write("event: hello\ndata: connected\n\n");
|
|
756
|
+
res.write(`event: mode\ndata: ${watching ? "enabled" : "disabled"}\n\n`);
|
|
757
|
+
|
|
758
|
+
const key = `${env}/${pkg}`;
|
|
759
|
+
const send = () => {
|
|
760
|
+
res.write("event: changed\ndata: changed\n\n");
|
|
761
|
+
};
|
|
762
|
+
watchModeController.events.on(key, send);
|
|
763
|
+
// Keep the connection alive through idle proxies (heartbeat every 25s).
|
|
764
|
+
const heartbeat = setInterval(() => {
|
|
765
|
+
res.write(": heartbeat\n\n");
|
|
766
|
+
}, 25000);
|
|
767
|
+
const cleanup = () => {
|
|
768
|
+
clearInterval(heartbeat);
|
|
769
|
+
watchModeController.events.off(key, send);
|
|
770
|
+
sseConnectionCount--;
|
|
771
|
+
};
|
|
772
|
+
// "close" covers both clean and abrupt disconnects on Node >= 20.
|
|
773
|
+
req.on("close", cleanup);
|
|
774
|
+
},
|
|
775
|
+
);
|
|
776
|
+
|
|
361
777
|
app.get(`${API_PREFIX}/environments`, async (_req, res) => {
|
|
362
778
|
try {
|
|
363
779
|
res.status(200).json(await environmentStore.listEnvironments());
|
|
@@ -1414,7 +1830,30 @@ registerLegacyRoutes(app, {
|
|
|
1414
1830
|
|
|
1415
1831
|
// Modify the catch-all route to only serve index.html in production
|
|
1416
1832
|
if (!isDevelopment) {
|
|
1417
|
-
|
|
1833
|
+
const SPA_INDEX = path.resolve(ROOT, "index.html");
|
|
1834
|
+
app.get("*", (req, res) => {
|
|
1835
|
+
res.sendFile(SPA_INDEX, (err) => {
|
|
1836
|
+
if (!err) return;
|
|
1837
|
+
// The SPA bundle isn't built. This happens when running directly
|
|
1838
|
+
// from source (`bun run src/server.ts`) without first running
|
|
1839
|
+
// `bun run build:app`. Return a friendly placeholder rather than
|
|
1840
|
+
// a 500, and surface package URLs the user might be looking for.
|
|
1841
|
+
if (res.headersSent) return;
|
|
1842
|
+
res.status(404)
|
|
1843
|
+
.type("text/html")
|
|
1844
|
+
.send(
|
|
1845
|
+
`<!doctype html><meta charset="utf-8">
|
|
1846
|
+
<title>Publisher</title>
|
|
1847
|
+
<style>body{font:14px/1.4 -apple-system,system-ui,sans-serif;margin:40px;max-width:720px;color:#222}</style>
|
|
1848
|
+
<h1>Publisher is running, but the SPA bundle isn't built.</h1>
|
|
1849
|
+
<p>You requested <code>${req.path.replace(/[<>&]/g, (c) => ({ "<": "<", ">": ">", "&": "&" })[c] ?? c)}</code>.
|
|
1850
|
+
The Publisher API is available at <a href="/api/v0/environments">/api/v0/environments</a>.</p>
|
|
1851
|
+
<p>To get the Publisher web UI, run <code>cd packages/app && bunx vite build</code>
|
|
1852
|
+
or start the server with <code>NODE_ENV=development</code> after launching Vite on <code>:5173</code>.</p>
|
|
1853
|
+
<p>For in-package HTML data apps, browse to <code>/environments/<env>/packages/<pkg>/<file></code> directly.</p>`,
|
|
1854
|
+
);
|
|
1855
|
+
});
|
|
1856
|
+
});
|
|
1418
1857
|
}
|
|
1419
1858
|
|
|
1420
1859
|
app.use(
|
|
@@ -1448,7 +1887,7 @@ mainServer.timeout = 600000;
|
|
|
1448
1887
|
mainServer.keepAliveTimeout = 600000;
|
|
1449
1888
|
mainServer.headersTimeout = 600000;
|
|
1450
1889
|
|
|
1451
|
-
mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
|
|
1890
|
+
mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
|
|
1452
1891
|
const address = mainServer.address() as AddressInfo;
|
|
1453
1892
|
logger.info(
|
|
1454
1893
|
`Publisher server listening at http://${address.address}:${address.port}`,
|
|
@@ -1458,6 +1897,44 @@ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
|
|
|
1458
1897
|
"Running in development mode - proxying to React dev server at http://localhost:5173",
|
|
1459
1898
|
);
|
|
1460
1899
|
}
|
|
1900
|
+
// If `--watch-env <name>` (or PUBLISHER_WATCH=name1,name2) was passed,
|
|
1901
|
+
// wait for env initialization to settle, then start watch mode for each
|
|
1902
|
+
// named env. Packages in those envs are already mounted in-place via the
|
|
1903
|
+
// EnvironmentStore in-place path (see `loadEnvironmentIntoDisk`), so the
|
|
1904
|
+
// chokidar watcher will see edits to your source repo and fan them out
|
|
1905
|
+
// to any connected SSE clients.
|
|
1906
|
+
const watchEnvList = (process.env.PUBLISHER_WATCH || "")
|
|
1907
|
+
.split(",")
|
|
1908
|
+
.map((s) => s.trim())
|
|
1909
|
+
.filter((s) => s.length > 0);
|
|
1910
|
+
if (watchEnvList.length > 0) {
|
|
1911
|
+
// The watcher tracks exactly one env at a time (`WatchModeController`
|
|
1912
|
+
// holds a single chokidar instance). Every env in PUBLISHER_WATCH is
|
|
1913
|
+
// still mounted in place (live source) by the EnvironmentStore, but only
|
|
1914
|
+
// the first is watched, so the others do not auto-reload.
|
|
1915
|
+
if (watchEnvList.length > 1) {
|
|
1916
|
+
logger.warn(
|
|
1917
|
+
`Multiple watch environments requested (${watchEnvList.join(
|
|
1918
|
+
", ",
|
|
1919
|
+
)}); watch mode auto-reloads one at a time. Watching "${
|
|
1920
|
+
watchEnvList[0]
|
|
1921
|
+
}". The others are mounted in place (their source is live) but will not auto-reload. Pass a single --watch-env (or one PUBLISHER_WATCH value) to silence this.`,
|
|
1922
|
+
);
|
|
1923
|
+
}
|
|
1924
|
+
const envName = watchEnvList[0];
|
|
1925
|
+
try {
|
|
1926
|
+
await environmentStore.finishedInitialization;
|
|
1927
|
+
await watchModeController.ensureWatching(envName);
|
|
1928
|
+
logger.info(
|
|
1929
|
+
`Watch mode active for environment "${envName}" (in-place mount, source-edit live reload).`,
|
|
1930
|
+
);
|
|
1931
|
+
} catch (error) {
|
|
1932
|
+
logger.error(
|
|
1933
|
+
`Failed to start watch mode for environment "${envName}"`,
|
|
1934
|
+
{ error },
|
|
1935
|
+
);
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1461
1938
|
});
|
|
1462
1939
|
const mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
|
|
1463
1940
|
logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
|
|
@@ -7,8 +7,8 @@ import path from "path";
|
|
|
7
7
|
import { AccessDeniedError } from "../errors";
|
|
8
8
|
import { Model } from "./model";
|
|
9
9
|
|
|
10
|
-
// Introspection
|
|
11
|
-
//
|
|
10
|
+
// Introspection, compile-time validation, and the runtime gate for
|
|
11
|
+
// #(authorize) / ##(authorize).
|
|
12
12
|
|
|
13
13
|
const TEST_DIR = path.join(os.tmpdir(), "authorize-integration-tests");
|
|
14
14
|
const TEST_DB_DIR = path.join(TEST_DIR, "db");
|
|
@@ -277,6 +277,11 @@ source: gated is duckdb.table('customers')
|
|
|
277
277
|
// Names the source and surfaces the underlying Malloy reason.
|
|
278
278
|
expect(err?.message).toContain("gated");
|
|
279
279
|
expect(err?.message).toMatch(/NOPE|not declared/i);
|
|
280
|
+
// Redaction policy (pinned): the model-load 424 is author-facing, so it
|
|
281
|
+
// KEEPS the full expression text (needed to fix a malformed annotation).
|
|
282
|
+
// Only the runtime 403 redacts to the source name. If this assertion ever
|
|
283
|
+
// flips, the redaction split was changed — make it a conscious decision.
|
|
284
|
+
expect(err?.message).toContain("$NOPE = 'x'");
|
|
280
285
|
});
|
|
281
286
|
|
|
282
287
|
it("fails model load when an expression references a source field", async () => {
|
|
@@ -836,3 +841,92 @@ source: joiner is duckdb.table('customers') extend {
|
|
|
836
841
|
expect(result.data).toBeDefined();
|
|
837
842
|
});
|
|
838
843
|
});
|
|
844
|
+
|
|
845
|
+
// The /compile path gates via Model.assertAuthorizedForText (early,
|
|
846
|
+
// surface-syntax) and Model.assertAuthorizedForRunnable (compiled-source
|
|
847
|
+
// backstop). These are the enforcement primitives environment.compileSource
|
|
848
|
+
// calls; exercise them directly here.
|
|
849
|
+
describe("authorize compile-path gate", () => {
|
|
850
|
+
const CP_GATE = `##! experimental.givens
|
|
851
|
+
|
|
852
|
+
given:
|
|
853
|
+
ROLE :: string
|
|
854
|
+
|
|
855
|
+
#(authorize) "$ROLE = 'analyst'"
|
|
856
|
+
source: gated is duckdb.table('customers') extend { measure: c is count() }
|
|
857
|
+
|
|
858
|
+
source: open_src is duckdb.table('customers') extend { measure: c is count() }
|
|
859
|
+
`;
|
|
860
|
+
const CP_FILE_LEVEL = `##! experimental.givens
|
|
861
|
+
|
|
862
|
+
given:
|
|
863
|
+
ROLE :: string
|
|
864
|
+
|
|
865
|
+
##(authorize) "$ROLE = 'admin'"
|
|
866
|
+
|
|
867
|
+
source: declared is duckdb.table('customers') extend { measure: c is count() }
|
|
868
|
+
`;
|
|
869
|
+
|
|
870
|
+
async function cpModel(file: string, src: string): Promise<Model> {
|
|
871
|
+
await writeModel(file, src);
|
|
872
|
+
return Model.create("test-pkg", TEST_PKG_DIR, file, getConnections());
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
it("assertAuthorizedForText denies/allows a gated named source by its given", async () => {
|
|
876
|
+
const model = await cpModel("cp_gate.malloy", CP_GATE);
|
|
877
|
+
await expect(
|
|
878
|
+
model.assertAuthorizedForText("run: gated -> { aggregate: c }", {}),
|
|
879
|
+
).rejects.toBeInstanceOf(AccessDeniedError);
|
|
880
|
+
await expect(
|
|
881
|
+
model.assertAuthorizedForText("run: gated -> { aggregate: c }", {
|
|
882
|
+
ROLE: "analyst",
|
|
883
|
+
}),
|
|
884
|
+
).resolves.toBeUndefined();
|
|
885
|
+
});
|
|
886
|
+
|
|
887
|
+
it("assertAuthorizedForText leaves an ungated source unrestricted", async () => {
|
|
888
|
+
const model = await cpModel("cp_gate.malloy", CP_GATE);
|
|
889
|
+
await expect(
|
|
890
|
+
model.assertAuthorizedForText("run: open_src -> { aggregate: c }", {}),
|
|
891
|
+
).resolves.toBeUndefined();
|
|
892
|
+
});
|
|
893
|
+
|
|
894
|
+
it("assertAuthorizedForText applies the model-wide file-level gate to inline/unnamed text", async () => {
|
|
895
|
+
const model = await cpModel("cp_file.malloy", CP_FILE_LEVEL);
|
|
896
|
+
// No named source the regex recognizes -> undefined -> file-level gate.
|
|
897
|
+
await expect(
|
|
898
|
+
model.assertAuthorizedForText(
|
|
899
|
+
`run: duckdb.sql("SELECT 1 AS x") -> { aggregate: n is count() }`,
|
|
900
|
+
{},
|
|
901
|
+
),
|
|
902
|
+
).rejects.toBeInstanceOf(AccessDeniedError);
|
|
903
|
+
await expect(
|
|
904
|
+
model.assertAuthorizedForText(
|
|
905
|
+
`run: duckdb.sql("SELECT 1 AS x") -> { aggregate: n is count() }`,
|
|
906
|
+
{ ROLE: "admin" },
|
|
907
|
+
),
|
|
908
|
+
).resolves.toBeUndefined();
|
|
909
|
+
});
|
|
910
|
+
|
|
911
|
+
it("assertAuthorizedForRunnable gates the compiled-source structRef (alias backstop)", async () => {
|
|
912
|
+
const model = await cpModel("cp_gate.malloy", CP_GATE);
|
|
913
|
+
// Stub a runnable whose compiled query reads `gated` (e.g. via an alias
|
|
914
|
+
// the surface-syntax gate would miss).
|
|
915
|
+
const gatedRunnable = {
|
|
916
|
+
getPreparedQuery: async () => ({ _query: { structRef: "gated" } }),
|
|
917
|
+
};
|
|
918
|
+
await expect(
|
|
919
|
+
model.assertAuthorizedForRunnable(gatedRunnable, {}),
|
|
920
|
+
).rejects.toBeInstanceOf(AccessDeniedError);
|
|
921
|
+
await expect(
|
|
922
|
+
model.assertAuthorizedForRunnable(gatedRunnable, { ROLE: "analyst" }),
|
|
923
|
+
).resolves.toBeUndefined();
|
|
924
|
+
// Ungated compiled source -> unrestricted.
|
|
925
|
+
const openRunnable = {
|
|
926
|
+
getPreparedQuery: async () => ({ _query: { structRef: "open_src" } }),
|
|
927
|
+
};
|
|
928
|
+
await expect(
|
|
929
|
+
model.assertAuthorizedForRunnable(openRunnable, {}),
|
|
930
|
+
).resolves.toBeUndefined();
|
|
931
|
+
});
|
|
932
|
+
});
|