@malloy-publisher/server 0.0.205 → 0.0.207
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +394 -395
- package/dist/app/assets/{EnvironmentPage-CAge6UHD.js → EnvironmentPage-BScgHmkw.js} +1 -1
- package/dist/app/assets/{HomePage-DhTe8qpa.js → HomePage-CGedji_w.js} +1 -1
- package/dist/app/assets/{MainPage-CeTxxGex.js → MainPage-DWfF4jXW.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-CpDHB70t.js → MaterializationsPage-B9PDlk8c.js} +1 -1
- package/dist/app/assets/{ModelPage-D9sSMb75.js → ModelPage-BiNOgK_e.js} +1 -1
- package/dist/app/assets/{PackagePage-LRqQWrFY.js → PackagePage-DAN5V7gu.js} +1 -1
- package/dist/app/assets/{RouteError-xT6kuCNw.js → RouteError-CEnIzuKN.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DsIh9svZ.js → WorkbookPage-gA1ceqHP.js} +1 -1
- package/dist/app/assets/{core-C2sQrwVu.es-Bjem0hym.js → core-AOmIKwkc.es-Dclu1Fga.js} +1 -1
- package/dist/app/assets/{index-BdOZDcce.js → index-DGGe8UpP.js} +1 -1
- package/dist/app/assets/{index-RX3QOTde.js → index-DtlPzNxc.js} +127 -127
- package/dist/app/assets/{index-DHHAcY5o.js → index-uu6UpHd2.js} +1 -1
- package/dist/app/assets/{index.umd-D2WH3D-f.js → index.umd-DDq93YX4.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +18 -8
- package/dist/package_load_worker.mjs +19 -2
- package/dist/server.mjs +1272 -1299
- package/package.json +1 -1
- package/src/constants.ts +12 -0
- package/src/controller/materialization.controller.ts +79 -35
- package/src/controller/package.controller.spec.ts +179 -0
- package/src/controller/package.controller.ts +60 -73
- package/src/dto/package.dto.ts +16 -1
- package/src/errors.spec.ts +12 -0
- package/src/errors.ts +18 -0
- package/src/health.spec.ts +34 -1
- package/src/instrumentation.ts +33 -17
- package/src/materialization_metrics.ts +121 -0
- package/src/package_load/package_load_pool.ts +7 -1
- package/src/package_load/package_load_worker.ts +44 -4
- package/src/package_load/protocol.ts +7 -1
- package/src/server-old.ts +7 -149
- package/src/server.ts +9 -188
- package/src/service/authorize_integration.spec.ts +67 -0
- package/src/service/environment.ts +270 -12
- package/src/service/environment_store.spec.ts +0 -81
- package/src/service/environment_store.ts +0 -23
- package/src/service/explore_visibility.spec.ts +434 -0
- package/src/service/exports_probe.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +99 -0
- package/src/service/manifest_loader.ts +95 -0
- package/src/service/materialization_service.spec.ts +584 -500
- package/src/service/materialization_service.ts +839 -657
- package/src/service/model.ts +419 -15
- package/src/service/package.spec.ts +14 -2
- package/src/service/package.ts +339 -29
- package/src/service/package_rollback.spec.ts +190 -0
- package/src/service/package_worker_path.spec.ts +223 -0
- package/src/service/query_boundary.spec.ts +470 -0
- package/src/storage/DatabaseInterface.ts +35 -57
- package/src/storage/StorageManager.mock.ts +0 -9
- package/src/storage/StorageManager.ts +7 -290
- package/src/storage/duckdb/DuckDBRepository.ts +2 -35
- package/src/storage/duckdb/MaterializationRepository.ts +52 -27
- package/src/storage/duckdb/schema.ts +4 -20
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +194 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +369 -264
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
- package/src/controller/manifest.controller.ts +0 -38
- package/src/service/manifest_service.spec.ts +0 -206
- package/src/service/manifest_service.ts +0 -117
- package/src/service/materialized_table_gc.spec.ts +0 -384
- package/src/service/materialized_table_gc.ts +0 -231
- package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
- package/src/storage/duckdb/ManifestRepository.ts +0 -120
- package/src/storage/duckdb/manifest_store.spec.ts +0 -133
- package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
- package/tests/unit/storage/StorageManager.test.ts +0 -166
package/src/service/package.ts
CHANGED
|
@@ -48,8 +48,6 @@ type PackageConnectionInput =
|
|
|
48
48
|
| Map<string, Connection>
|
|
49
49
|
| (() => MalloyConfig);
|
|
50
50
|
|
|
51
|
-
const ENABLE_LIST_MODEL_COMPILATION = true;
|
|
52
|
-
|
|
53
51
|
export class Package {
|
|
54
52
|
private environmentName: string;
|
|
55
53
|
private packageName: string;
|
|
@@ -58,6 +56,17 @@ export class Package {
|
|
|
58
56
|
private models: Map<string, Model> = new Map();
|
|
59
57
|
private packagePath: string;
|
|
60
58
|
private malloyConfig: MalloyConfig;
|
|
59
|
+
// Build-manifest binding state (Malloy Persistence v0). When bound, these
|
|
60
|
+
// entries (buildId -> { tableName }) are what served queries use to route
|
|
61
|
+
// persist sources to their materialized physical tables; they are also reused
|
|
62
|
+
// by the /compile preview so previewed SQL matches executed SQL. Surfaced on
|
|
63
|
+
// /status (via getPackageMetadata) so the control plane can confirm a worker
|
|
64
|
+
// actually bound the distributed manifest instead of inferring it from logs.
|
|
65
|
+
private buildManifestEntries: BuildManifest["entries"] | undefined;
|
|
66
|
+
private manifestBindingStatus: "unbound" | "bound" | "live_fallback" =
|
|
67
|
+
"unbound";
|
|
68
|
+
private manifestEntryCount = 0;
|
|
69
|
+
private boundManifestUri: string | null = null;
|
|
61
70
|
private static meter = metrics.getMeter("publisher");
|
|
62
71
|
private static packageLoadHistogram = this.meter.createHistogram(
|
|
63
72
|
"malloy_package_load_duration",
|
|
@@ -83,6 +92,64 @@ export class Package {
|
|
|
83
92
|
this.databases = databases;
|
|
84
93
|
this.models = models;
|
|
85
94
|
this.malloyConfig = malloyConfig;
|
|
95
|
+
this.applyDiscoveryPolicyToModels();
|
|
96
|
+
this.applyQueryBoundaryToModels();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Push the discovery-curation policy down onto each Model. Curation (file
|
|
101
|
+
* listing via `explores` and within-file `export {}` filtering) is enabled
|
|
102
|
+
* only when `explores` is declared in publisher.json — absent/empty
|
|
103
|
+
* `explores` preserves legacy listings. Re-derived on reload and metadata
|
|
104
|
+
* PATCH (the inputs can change there).
|
|
105
|
+
*/
|
|
106
|
+
/** True when the package opts into curated discovery via a non-empty
|
|
107
|
+
* `explores`. Single source of truth so the curation/boundary/listing
|
|
108
|
+
* derivations can't drift out of sync. */
|
|
109
|
+
private exploresDeclared(): boolean {
|
|
110
|
+
const explores = this.packageMetadata.explores;
|
|
111
|
+
return !!(explores && explores.length > 0);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** The declared explore set, or null when discovery is uncurated. */
|
|
115
|
+
private exploreSet(): Set<string> | null {
|
|
116
|
+
const explores = this.packageMetadata.explores;
|
|
117
|
+
return explores && explores.length > 0 ? new Set(explores) : null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
private applyDiscoveryPolicyToModels(): void {
|
|
121
|
+
const curationEnabled = this.exploresDeclared();
|
|
122
|
+
for (const model of this.models.values()) {
|
|
123
|
+
model.setDiscoveryCuration(curationEnabled);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Push the package-level query-boundary policy down onto each Model so the
|
|
129
|
+
* query chokepoints can enforce it without a back-reference to the Package:
|
|
130
|
+
* `Model.getQueryResults` (the HTTP query route and the MCP tool) and the
|
|
131
|
+
* `/compile` path (via `assertQueryBoundaryForRunnable`). Derived once here
|
|
132
|
+
* (and on reload) rather than per query: the policy only changes when the
|
|
133
|
+
* manifest is (re)read.
|
|
134
|
+
*
|
|
135
|
+
* Policy: queryable == discoverable. The boundary is inert unless `explores`
|
|
136
|
+
* is declared (no curated surface ⇒ nothing to restrict) AND
|
|
137
|
+
* `queryableSources` is "declared" (the default; "all" decouples the axes).
|
|
138
|
+
* When active, a model file is a query entry point only if it is listed in
|
|
139
|
+
* `explores`; within-file curation (`export {}`) is read off each Model.
|
|
140
|
+
*/
|
|
141
|
+
private applyQueryBoundaryToModels(): void {
|
|
142
|
+
const exploresDeclared = this.exploresDeclared();
|
|
143
|
+
const exploreSet = this.exploreSet();
|
|
144
|
+
const mode =
|
|
145
|
+
this.packageMetadata.queryableSources === "all" ? "all" : "declared";
|
|
146
|
+
for (const [modelPath, model] of this.models) {
|
|
147
|
+
model.setQueryBoundary({
|
|
148
|
+
mode,
|
|
149
|
+
exploresDeclared,
|
|
150
|
+
isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
86
153
|
}
|
|
87
154
|
|
|
88
155
|
static async create(
|
|
@@ -251,6 +318,9 @@ export class Package {
|
|
|
251
318
|
name: outcome.packageMetadata.name,
|
|
252
319
|
description: outcome.packageMetadata.description,
|
|
253
320
|
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
|
|
321
|
+
explores: outcome.packageMetadata.explores,
|
|
322
|
+
queryableSources: outcome.packageMetadata.queryableSources,
|
|
323
|
+
manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
|
|
254
324
|
};
|
|
255
325
|
|
|
256
326
|
// Build live `Model`s from worker output. Any per-model compile
|
|
@@ -272,10 +342,18 @@ export class Package {
|
|
|
272
342
|
// cleans the package directory.
|
|
273
343
|
throw err;
|
|
274
344
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
345
|
+
const model = Model.fromSerialized(
|
|
346
|
+
packageName,
|
|
347
|
+
packagePath,
|
|
348
|
+
malloyConfig,
|
|
349
|
+
sm,
|
|
278
350
|
);
|
|
351
|
+
// Validate renderer tags on the main thread (the renderer is too heavy
|
|
352
|
+
// to load inside the pure-CPU package-load worker). A misconfigured tag
|
|
353
|
+
// throws a ModelCompilationError (424), aborting the whole load like any
|
|
354
|
+
// other per-model compile error above.
|
|
355
|
+
await model.validateRenderTags();
|
|
356
|
+
models.set(sm.modelPath, model);
|
|
279
357
|
}
|
|
280
358
|
|
|
281
359
|
const endTime = performance.now();
|
|
@@ -289,7 +367,7 @@ export class Package {
|
|
|
289
367
|
duration: formatDuration(executionTime),
|
|
290
368
|
});
|
|
291
369
|
|
|
292
|
-
|
|
370
|
+
const pkg = new Package(
|
|
293
371
|
environmentName,
|
|
294
372
|
packageName,
|
|
295
373
|
packagePath,
|
|
@@ -298,6 +376,21 @@ export class Package {
|
|
|
298
376
|
models,
|
|
299
377
|
malloyConfig,
|
|
300
378
|
);
|
|
379
|
+
|
|
380
|
+
// Fail-safe at load: a bad explores entry doesn't fail the package
|
|
381
|
+
// (its models still load and listModels hides the unmatched entry — it
|
|
382
|
+
// never falls back to listing everything). Warn so the misconfig is
|
|
383
|
+
// visible; the publish path rejects it outright (see package.controller).
|
|
384
|
+
const invalidMsg = pkg.formatInvalidExplores();
|
|
385
|
+
if (invalidMsg) {
|
|
386
|
+
logger.warn(`Package ${packageName} has invalid explores`, {
|
|
387
|
+
packageName,
|
|
388
|
+
detail: invalidMsg,
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
pkg.logEmptyDiscoveryWarnings();
|
|
392
|
+
|
|
393
|
+
return pkg;
|
|
301
394
|
}
|
|
302
395
|
|
|
303
396
|
public getPackageName(): string {
|
|
@@ -305,7 +398,171 @@ export class Package {
|
|
|
305
398
|
}
|
|
306
399
|
|
|
307
400
|
public getPackageMetadata(): ApiPackage {
|
|
308
|
-
|
|
401
|
+
// Overlay the server-computed fields onto the stored metadata: the
|
|
402
|
+
// explores misconfig warnings (loading is fail-safe — the package still
|
|
403
|
+
// serves with the bad entry hidden — so this is the only non-log signal
|
|
404
|
+
// that it's broken) and the manifest-binding state (so /status reflects
|
|
405
|
+
// whether persist sources are actually routed to materialized tables).
|
|
406
|
+
// Always returns a copy so these overlays never mutate the stored
|
|
407
|
+
// metadata; the binding fields are authoritative from the private state.
|
|
408
|
+
//
|
|
409
|
+
// `name` is the registered package name (the environment's identity for
|
|
410
|
+
// this package), not the value read from the package's own manifest —
|
|
411
|
+
// those can differ (e.g. a package installed under a different name than
|
|
412
|
+
// its publisher.json declares). `listPackages` already overrides it to
|
|
413
|
+
// the registered name; surfacing it here keeps the single-package GET
|
|
414
|
+
// consistent without relying on a returned-reference mutation.
|
|
415
|
+
const metadata: ApiPackage = {
|
|
416
|
+
...this.packageMetadata,
|
|
417
|
+
name: this.packageName,
|
|
418
|
+
manifestBindingStatus: this.manifestBindingStatus,
|
|
419
|
+
manifestEntryCount: this.manifestEntryCount,
|
|
420
|
+
boundManifestUri: this.boundManifestUri,
|
|
421
|
+
};
|
|
422
|
+
const warnings = this.exploreWarnings();
|
|
423
|
+
if (warnings.length > 0) {
|
|
424
|
+
metadata.exploresWarnings = warnings;
|
|
425
|
+
}
|
|
426
|
+
return metadata;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* The currently-bound build-manifest entries (buildId -> { tableName }), or
|
|
431
|
+
* undefined when the package is serving live. Reused by the /compile preview
|
|
432
|
+
* so previewed SQL gets the same persist-source -> physical-table routing as
|
|
433
|
+
* execution.
|
|
434
|
+
*/
|
|
435
|
+
public getBuildManifestEntries(): BuildManifest["entries"] | undefined {
|
|
436
|
+
return this.buildManifestEntries;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* Record the URI whose manifest is currently bound to the served models. May
|
|
441
|
+
* differ from `manifestLocation` after an in-memory auto-load following a
|
|
442
|
+
* materialization build (no URI), in which case it stays null.
|
|
443
|
+
*/
|
|
444
|
+
public setBoundManifestUri(uri: string | null): void {
|
|
445
|
+
this.boundManifestUri = uri;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* Mark that a configured `manifestLocation` could not be fetched/bound, so
|
|
450
|
+
* the package is serving live despite intending to be materialized-routed.
|
|
451
|
+
*/
|
|
452
|
+
public markManifestBindFailed(): void {
|
|
453
|
+
this.manifestBindingStatus = "live_fallback";
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* Store the entries applied by the latest (re)bind for reuse (/compile) and
|
|
458
|
+
* observability (/status). An empty map means the manifest was cleared, so
|
|
459
|
+
* the package reverts to live (unbound).
|
|
460
|
+
*/
|
|
461
|
+
private recordManifestBinding(
|
|
462
|
+
buildManifest: BuildManifest["entries"],
|
|
463
|
+
): void {
|
|
464
|
+
const count = Object.keys(buildManifest).length;
|
|
465
|
+
this.buildManifestEntries = count > 0 ? buildManifest : undefined;
|
|
466
|
+
this.manifestEntryCount = count;
|
|
467
|
+
this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
|
|
468
|
+
if (count === 0) {
|
|
469
|
+
this.boundManifestUri = null;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/**
|
|
474
|
+
* Declared `explores` (publisher.json) that don't resolve to a real
|
|
475
|
+
* `.malloy` model in this package, each with an actionable reason. Empty
|
|
476
|
+
* when explores is absent/empty or every entry resolves.
|
|
477
|
+
*
|
|
478
|
+
* The listing already fails safe — a non-resolving entry matches no model in
|
|
479
|
+
* `listModels`, so it hides rather than exposes. This surfaces *why*, so the
|
|
480
|
+
* load path can warn and the publish path can reject (see package.controller).
|
|
481
|
+
*/
|
|
482
|
+
public getInvalidExplores(
|
|
483
|
+
exploresOverride?: string[],
|
|
484
|
+
): { entry: string; reason: string }[] {
|
|
485
|
+
const declared = exploresOverride ?? this.packageMetadata.explores;
|
|
486
|
+
if (!declared || declared.length === 0) return [];
|
|
487
|
+
const malloyModels = new Set(
|
|
488
|
+
Array.from(this.models.keys()).filter((p) =>
|
|
489
|
+
p.endsWith(MODEL_FILE_SUFFIX),
|
|
490
|
+
),
|
|
491
|
+
);
|
|
492
|
+
const problems: { entry: string; reason: string }[] = [];
|
|
493
|
+
for (const entry of declared) {
|
|
494
|
+
if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
495
|
+
problems.push({
|
|
496
|
+
entry,
|
|
497
|
+
reason:
|
|
498
|
+
`notebooks are always public and cannot be explores. ` +
|
|
499
|
+
`Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`,
|
|
500
|
+
});
|
|
501
|
+
} else if (!malloyModels.has(entry)) {
|
|
502
|
+
problems.push({
|
|
503
|
+
entry,
|
|
504
|
+
reason:
|
|
505
|
+
`file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` +
|
|
506
|
+
`file relative to the package root (e.g. "index.malloy").`,
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return problems;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** One actionable message per invalid entry (empty when all resolve). */
|
|
514
|
+
public exploreWarnings(exploresOverride?: string[]): string[] {
|
|
515
|
+
return this.getInvalidExplores(exploresOverride).map(
|
|
516
|
+
(p) =>
|
|
517
|
+
`Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`,
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/**
|
|
522
|
+
* The {@link exploreWarnings} joined into one string, or "" if none.
|
|
523
|
+
* Newline-separated so multiple invalid entries stay one-per-line in the
|
|
524
|
+
* 400 message rather than running together.
|
|
525
|
+
*/
|
|
526
|
+
public formatInvalidExplores(exploresOverride?: string[]): string {
|
|
527
|
+
return this.exploreWarnings(exploresOverride).join("\n");
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* One message per LISTED model whose discovery surface is empty because it
|
|
532
|
+
* is import-only (imports other files, declares/re-exports nothing). Such a
|
|
533
|
+
* model renders a blank page, which reads as broken; the fix is an explicit
|
|
534
|
+
* re-export. Log-only (see loadViaWorker/reloadAllModels) — deliberately
|
|
535
|
+
* NOT part of exploreWarnings, which is strict-at-publish: import-only
|
|
536
|
+
* files are a legitimate pattern and must not block a publish. Hidden
|
|
537
|
+
* (non-listed) models are skipped — nobody browses them, so an empty
|
|
538
|
+
* surface there is just normal plumbing.
|
|
539
|
+
*/
|
|
540
|
+
public emptyDiscoveryWarnings(): string[] {
|
|
541
|
+
const exploreSet = this.exploreSet();
|
|
542
|
+
const warnings: string[] = [];
|
|
543
|
+
for (const [modelPath, model] of this.models) {
|
|
544
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) continue;
|
|
545
|
+
if (exploreSet && !exploreSet.has(modelPath)) continue;
|
|
546
|
+
if (model.hasEmptyDiscoverySurface()) {
|
|
547
|
+
warnings.push(
|
|
548
|
+
`Model "${modelPath}" is listed but exposes nothing: it only ` +
|
|
549
|
+
`imports other files and re-exports none of their sources. ` +
|
|
550
|
+
`Add e.g. 'export { source_name }' to surface sources on ` +
|
|
551
|
+
`this model.`,
|
|
552
|
+
);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
return warnings;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/** Log {@link emptyDiscoveryWarnings}; shared by load and reload. */
|
|
559
|
+
private logEmptyDiscoveryWarnings(): void {
|
|
560
|
+
for (const warning of this.emptyDiscoveryWarnings()) {
|
|
561
|
+
logger.warn(`Package ${this.packageName} has a blank-looking model`, {
|
|
562
|
+
packageName: this.packageName,
|
|
563
|
+
detail: warning,
|
|
564
|
+
});
|
|
565
|
+
}
|
|
309
566
|
}
|
|
310
567
|
|
|
311
568
|
public listDatabases(): ApiDatabase[] {
|
|
@@ -404,18 +661,67 @@ export class Package {
|
|
|
404
661
|
),
|
|
405
662
|
);
|
|
406
663
|
} else {
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
sm,
|
|
414
|
-
),
|
|
664
|
+
const model = Model.fromSerialized(
|
|
665
|
+
this.packageName,
|
|
666
|
+
this.packagePath,
|
|
667
|
+
this.malloyConfig,
|
|
668
|
+
sm,
|
|
669
|
+
{ buildManifest },
|
|
415
670
|
);
|
|
671
|
+
// Validate renderer tags here too (loadViaWorker does it for the
|
|
672
|
+
// create path). Reload keeps per-model placeholders rather than
|
|
673
|
+
// aborting the whole package, so a render-tag error is recorded as
|
|
674
|
+
// this model's compilationError instead of thrown.
|
|
675
|
+
try {
|
|
676
|
+
await model.validateRenderTags();
|
|
677
|
+
nextModels.set(sm.modelPath, model);
|
|
678
|
+
} catch (renderErr) {
|
|
679
|
+
const err =
|
|
680
|
+
renderErr instanceof Error
|
|
681
|
+
? renderErr
|
|
682
|
+
: new Error(String(renderErr));
|
|
683
|
+
logger.warn("Render-tag validation failed during reload", {
|
|
684
|
+
packageName: this.packageName,
|
|
685
|
+
modelPath: sm.modelPath,
|
|
686
|
+
error: err.message,
|
|
687
|
+
});
|
|
688
|
+
nextModels.set(
|
|
689
|
+
sm.modelPath,
|
|
690
|
+
Model.fromCompilationError(
|
|
691
|
+
this.packageName,
|
|
692
|
+
sm.modelPath,
|
|
693
|
+
sm.modelType,
|
|
694
|
+
err,
|
|
695
|
+
),
|
|
696
|
+
);
|
|
697
|
+
}
|
|
416
698
|
}
|
|
417
699
|
}
|
|
418
700
|
this.models = nextModels;
|
|
701
|
+
// A reload re-reads publisher.json in the worker; pick up any change to
|
|
702
|
+
// the explore set and query-boundary mode so listModels()/the gate
|
|
703
|
+
// reflect edited explores without a full Package.create.
|
|
704
|
+
this.packageMetadata.explores = outcome.packageMetadata.explores;
|
|
705
|
+
this.packageMetadata.queryableSources =
|
|
706
|
+
outcome.packageMetadata.queryableSources;
|
|
707
|
+
this.packageMetadata.manifestLocation =
|
|
708
|
+
outcome.packageMetadata.manifestLocation ?? null;
|
|
709
|
+
this.applyDiscoveryPolicyToModels();
|
|
710
|
+
this.applyQueryBoundaryToModels();
|
|
711
|
+
// Remember what we just bound so /compile can route identically and
|
|
712
|
+
// /status can report the binding. An empty map reverts to live (unbound).
|
|
713
|
+
this.recordManifestBinding(buildManifest);
|
|
714
|
+
// Re-run the fail-safe warning against the refreshed model set: an edit
|
|
715
|
+
// to publisher.json that introduces a bad entry should surface in the
|
|
716
|
+
// logs on reload too, not only at initial load (loadViaWorker).
|
|
717
|
+
const invalidMsg = this.formatInvalidExplores();
|
|
718
|
+
if (invalidMsg) {
|
|
719
|
+
logger.warn(`Package ${this.packageName} has invalid explores`, {
|
|
720
|
+
packageName: this.packageName,
|
|
721
|
+
detail: invalidMsg,
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
this.logEmptyDiscoveryWarnings();
|
|
419
725
|
}
|
|
420
726
|
|
|
421
727
|
public async getModelFileText(modelPath: string): Promise<string> {
|
|
@@ -427,22 +733,27 @@ export class Package {
|
|
|
427
733
|
}
|
|
428
734
|
|
|
429
735
|
public async listModels(): Promise<ApiModel[]> {
|
|
736
|
+
// When `explores` is declared in publisher.json, only those models
|
|
737
|
+
// form the public surface; every other .malloy file still compiles for
|
|
738
|
+
// import/join resolution but is hidden from the listing. Absent/empty →
|
|
739
|
+
// every model is listed (backward-compatible default). Notebooks are
|
|
740
|
+
// unaffected (see listNotebooks) — they are always public.
|
|
741
|
+
const exploreSet = this.exploreSet();
|
|
430
742
|
const values = await Promise.all(
|
|
431
743
|
Array.from(this.models.keys())
|
|
432
744
|
.filter((modelPath) => {
|
|
433
|
-
|
|
745
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) return false;
|
|
746
|
+
return exploreSet ? exploreSet.has(modelPath) : true;
|
|
434
747
|
})
|
|
435
748
|
.map(async (modelPath) => {
|
|
436
749
|
let error: string | undefined;
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
modelError
|
|
443
|
-
|
|
444
|
-
: undefined;
|
|
445
|
-
}
|
|
750
|
+
try {
|
|
751
|
+
await this.models.get(modelPath)?.getModel();
|
|
752
|
+
} catch (modelError) {
|
|
753
|
+
error =
|
|
754
|
+
modelError instanceof Error
|
|
755
|
+
? modelError.message
|
|
756
|
+
: undefined;
|
|
446
757
|
}
|
|
447
758
|
return {
|
|
448
759
|
environmentName: this.environmentName,
|
|
@@ -462,10 +773,7 @@ export class Package {
|
|
|
462
773
|
return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
|
|
463
774
|
})
|
|
464
775
|
.map(async (modelPath) => {
|
|
465
|
-
|
|
466
|
-
if (ENABLE_LIST_MODEL_COMPILATION) {
|
|
467
|
-
error = this.models.get(modelPath)?.getNotebookError();
|
|
468
|
-
}
|
|
776
|
+
const error = this.models.get(modelPath)?.getNotebookError();
|
|
469
777
|
return {
|
|
470
778
|
environmentName: this.environmentName,
|
|
471
779
|
packageName: this.packageName,
|
|
@@ -632,5 +940,7 @@ export class Package {
|
|
|
632
940
|
|
|
633
941
|
public setPackageMetadata(packageMetadata: ApiPackage) {
|
|
634
942
|
this.packageMetadata = packageMetadata;
|
|
943
|
+
this.applyDiscoveryPolicyToModels();
|
|
944
|
+
this.applyQueryBoundaryToModels();
|
|
635
945
|
}
|
|
636
946
|
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import * as fs from "fs/promises";
|
|
3
|
+
import * as os from "os";
|
|
4
|
+
import * as path from "path";
|
|
5
|
+
|
|
6
|
+
import { BadRequestError } from "../errors";
|
|
7
|
+
import { Environment } from "./environment";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Regression tests for explores-validation ROLLBACK, guarding against the two
|
|
11
|
+
* data-loss bugs a code review found:
|
|
12
|
+
*
|
|
13
|
+
* 1. No-location create rollback must NOT delete a pre-existing user
|
|
14
|
+
* directory. The controller now `unloadPackage`s (evict from memory, keep
|
|
15
|
+
* files) instead of `deletePackage` (which renames + rm's the tree).
|
|
16
|
+
*
|
|
17
|
+
* 2. A location update/reinstall with invalid explores must roll back to the
|
|
18
|
+
* PREVIOUS tree rather than swapping the rejected one in and 400-ing after.
|
|
19
|
+
* Validation now happens inside `installPackage`'s swap window.
|
|
20
|
+
*
|
|
21
|
+
* Both run against a real `Environment` + real `Package.create` over temp dirs.
|
|
22
|
+
*/
|
|
23
|
+
describe("explores-validation rollback (real Environment)", () => {
|
|
24
|
+
let rootDir: string;
|
|
25
|
+
let envPath: string;
|
|
26
|
+
|
|
27
|
+
const GOOD_MODEL = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
|
|
28
|
+
const BAD_MODEL = `source: twos is duckdb.sql("SELECT 2 as y")\n`;
|
|
29
|
+
|
|
30
|
+
async function writePackageDir(
|
|
31
|
+
dir: string,
|
|
32
|
+
manifest: Record<string, unknown>,
|
|
33
|
+
model: string,
|
|
34
|
+
): Promise<void> {
|
|
35
|
+
await fs.mkdir(dir, { recursive: true });
|
|
36
|
+
await fs.writeFile(
|
|
37
|
+
path.join(dir, "publisher.json"),
|
|
38
|
+
JSON.stringify({
|
|
39
|
+
name: "pkg",
|
|
40
|
+
description: "rollback fixture",
|
|
41
|
+
...manifest,
|
|
42
|
+
}),
|
|
43
|
+
);
|
|
44
|
+
await fs.writeFile(path.join(dir, "model.malloy"), model);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function copyDir(src: string, dst: string): Promise<void> {
|
|
48
|
+
await fs.mkdir(dst, { recursive: true });
|
|
49
|
+
await fs.cp(src, dst, { recursive: true });
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
beforeEach(async () => {
|
|
53
|
+
rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-rollback-"));
|
|
54
|
+
envPath = path.join(rootDir, "env");
|
|
55
|
+
await fs.mkdir(envPath, { recursive: true });
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
afterEach(async () => {
|
|
59
|
+
await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it("Fix 1: unloadPackage evicts from memory but keeps the on-disk directory", async () => {
|
|
63
|
+
// Mimics the no-location create path: a pre-existing package directory is
|
|
64
|
+
// registered, then rolled back. unloadPackage must not touch the files.
|
|
65
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
66
|
+
const pkgDir = path.join(envPath, "pkg");
|
|
67
|
+
await writePackageDir(pkgDir, {}, GOOD_MODEL);
|
|
68
|
+
|
|
69
|
+
await env.addPackage("pkg");
|
|
70
|
+
expect(env.getPackageStatus("pkg")).toBeDefined();
|
|
71
|
+
|
|
72
|
+
await env.unloadPackage("pkg");
|
|
73
|
+
|
|
74
|
+
// Evicted from memory...
|
|
75
|
+
expect(env.getPackageStatus("pkg")).toBeUndefined();
|
|
76
|
+
// ...but the user's files are untouched.
|
|
77
|
+
const modelText = await fs.readFile(
|
|
78
|
+
path.join(pkgDir, "model.malloy"),
|
|
79
|
+
"utf-8",
|
|
80
|
+
);
|
|
81
|
+
expect(modelText).toBe(GOOD_MODEL);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it("Fix 2: a rejected location reinstall restores the previous tree", async () => {
|
|
85
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
86
|
+
|
|
87
|
+
const goodFixture = path.join(rootDir, "good");
|
|
88
|
+
const badFixture = path.join(rootDir, "bad");
|
|
89
|
+
await writePackageDir(goodFixture, {}, GOOD_MODEL);
|
|
90
|
+
await writePackageDir(
|
|
91
|
+
badFixture,
|
|
92
|
+
{ explores: ["missing.malloy"] },
|
|
93
|
+
BAD_MODEL,
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
// First install: the good tree is served.
|
|
97
|
+
await env.installPackage("pkg", (stagingPath) =>
|
|
98
|
+
copyDir(goodFixture, stagingPath),
|
|
99
|
+
);
|
|
100
|
+
expect(await env.getModelFileText("pkg", "model.malloy")).toBe(
|
|
101
|
+
GOOD_MODEL,
|
|
102
|
+
);
|
|
103
|
+
|
|
104
|
+
// Reinstall with an invalid-explores tree, validated inside the swap.
|
|
105
|
+
await expect(
|
|
106
|
+
env.installPackage(
|
|
107
|
+
"pkg",
|
|
108
|
+
(stagingPath) => copyDir(badFixture, stagingPath),
|
|
109
|
+
(pkg) => pkg.formatInvalidExplores(),
|
|
110
|
+
),
|
|
111
|
+
).rejects.toBeInstanceOf(BadRequestError);
|
|
112
|
+
|
|
113
|
+
// The previous (good) tree is still the one on disk — the rejected tree
|
|
114
|
+
// was rolled back, not swapped in.
|
|
115
|
+
expect(await env.getModelFileText("pkg", "model.malloy")).toBe(
|
|
116
|
+
GOOD_MODEL,
|
|
117
|
+
);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("a valid location reinstall still succeeds (no false rollback)", async () => {
|
|
121
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
122
|
+
const goodFixture = path.join(rootDir, "good");
|
|
123
|
+
const nextFixture = path.join(rootDir, "next");
|
|
124
|
+
await writePackageDir(goodFixture, {}, GOOD_MODEL);
|
|
125
|
+
await writePackageDir(
|
|
126
|
+
nextFixture,
|
|
127
|
+
{ explores: ["model.malloy"] },
|
|
128
|
+
BAD_MODEL,
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
await env.installPackage("pkg", (stagingPath) =>
|
|
132
|
+
copyDir(goodFixture, stagingPath),
|
|
133
|
+
);
|
|
134
|
+
await env.installPackage(
|
|
135
|
+
"pkg",
|
|
136
|
+
(stagingPath) => copyDir(nextFixture, stagingPath),
|
|
137
|
+
(pkg) => pkg.formatInvalidExplores(),
|
|
138
|
+
);
|
|
139
|
+
expect(await env.getModelFileText("pkg", "model.malloy")).toBe(BAD_MODEL);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("updatePackage normalizes a ./-prefixed body explores (no false 400, persists normalized)", async () => {
|
|
143
|
+
// API-body explores must go through the same normalization the worker
|
|
144
|
+
// applies to on-disk explores, so `./model.malloy` validates and persists
|
|
145
|
+
// as `model.malloy` rather than being rejected with a misleading 404.
|
|
146
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
147
|
+
const pkgDir = path.join(envPath, "pkg");
|
|
148
|
+
await writePackageDir(pkgDir, {}, GOOD_MODEL);
|
|
149
|
+
await env.addPackage("pkg");
|
|
150
|
+
|
|
151
|
+
const updated = await env.updatePackage("pkg", {
|
|
152
|
+
name: "pkg",
|
|
153
|
+
explores: ["./model.malloy"],
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
// Accepted (no BadRequestError) and stored in normalized form...
|
|
157
|
+
expect(updated.explores).toEqual(["model.malloy"]);
|
|
158
|
+
// ...both in memory and on disk.
|
|
159
|
+
const manifest = JSON.parse(
|
|
160
|
+
await fs.readFile(path.join(pkgDir, "publisher.json"), "utf-8"),
|
|
161
|
+
);
|
|
162
|
+
expect(manifest.explores).toEqual(["model.malloy"]);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it("compileSource rejects a notebook path; a model path still compiles", async () => {
|
|
166
|
+
// /compile appends source to the target MODEL's content for context; a
|
|
167
|
+
// notebook isn't a valid target and must be rejected up front (not left to
|
|
168
|
+
// a confusing downstream parse error).
|
|
169
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
170
|
+
const pkgDir = path.join(envPath, "pkg");
|
|
171
|
+
await writePackageDir(pkgDir, {}, GOOD_MODEL);
|
|
172
|
+
await fs.writeFile(
|
|
173
|
+
path.join(pkgDir, "report.malloynb"),
|
|
174
|
+
`>>>markdown\n# Report\n`,
|
|
175
|
+
);
|
|
176
|
+
await env.addPackage("pkg");
|
|
177
|
+
|
|
178
|
+
await expect(
|
|
179
|
+
env.compileSource("pkg", "report.malloynb", "run: ones"),
|
|
180
|
+
).rejects.toBeInstanceOf(BadRequestError);
|
|
181
|
+
|
|
182
|
+
// The .malloy model still compiles ad-hoc source (no regression).
|
|
183
|
+
const { problems } = await env.compileSource(
|
|
184
|
+
"pkg",
|
|
185
|
+
"model.malloy",
|
|
186
|
+
"run: ones -> { select: x }",
|
|
187
|
+
);
|
|
188
|
+
expect(problems.some((p) => p.severity === "error")).toBe(false);
|
|
189
|
+
});
|
|
190
|
+
});
|