@malloy-publisher/server 0.0.227 → 0.0.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +8 -8
- package/dist/app/api-doc.yaml +84 -1
- package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
- package/dist/app/assets/{HomePage-CXguJsXS.js → HomePage-C71GOfVW.js} +1 -1
- package/dist/app/assets/{LightMode-ZsshUznu.js → LightMode-CrgCAwLe.js} +1 -1
- package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
- package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
- package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
- package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
- package/dist/app/assets/{RouteError-Chn7lL96.js → RouteError-Cymbp47a.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
- package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
- package/dist/app/assets/{core-vVgoO8IR.es-BD_THWs_.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
- package/dist/app/assets/{index-gEWxu09x.js → index-CM2qhQCI.js} +1 -1
- package/dist/app/assets/{index-D6YtyiJ0.js → index-CcuuST2X.js} +1 -1
- package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
- package/dist/app/assets/{index-BioohWQj.js → index-DlWCXghy.js} +1 -1
- package/dist/app/assets/index-DxArlgRD.js +529 -0
- package/dist/app/index.html +1 -1
- package/dist/default-publisher.config.json +7 -7
- package/dist/instrumentation.mjs +1 -1
- package/dist/package_load_worker.mjs +1 -1
- package/dist/runtime/publisher.js +5 -0
- package/dist/server.mjs +7675 -899
- package/package.json +4 -2
- package/publisher.config.example.bigquery.json +7 -7
- package/publisher.config.example.duckdb.json +7 -7
- package/publisher.config.json +7 -11
- package/src/config.spec.ts +89 -3
- package/src/config.ts +102 -1
- package/src/controller/materialization.controller.spec.ts +9 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/controller/package.controller.ts +62 -31
- package/src/default-publisher.config.json +7 -7
- package/src/materialization_metrics.ts +33 -0
- package/src/mcp/handler_utils.spec.ts +108 -0
- package/src/mcp/handler_utils.ts +98 -4
- package/src/mcp/server.protocol.spec.ts +58 -0
- package/src/mcp/server.ts +29 -1
- package/src/mcp/skills/build_skills_bundle.ts +29 -9
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +31 -2
- package/src/mcp/tools/compile_tool.spec.ts +207 -0
- package/src/mcp/tools/compile_tool.ts +177 -0
- package/src/mcp/tools/execute_query_tool.spec.ts +143 -0
- package/src/mcp/tools/execute_query_tool.ts +37 -4
- package/src/mcp/tools/get_context_tool.ts +1 -1
- package/src/mcp/tools/reload_package_tool.spec.ts +232 -0
- package/src/mcp/tools/reload_package_tool.ts +158 -0
- package/src/query_param_utils.ts +11 -0
- package/src/runtime/publisher.js +5 -0
- package/src/server.ts +74 -14
- package/src/service/connection.spec.ts +8 -7
- package/src/service/cron_evaluator.spec.ts +79 -0
- package/src/service/cron_evaluator.ts +105 -0
- package/src/service/environment.ts +188 -19
- package/src/service/environment_store.spec.ts +372 -2
- package/src/service/environment_store.ts +149 -17
- package/src/service/environment_store_anchoring.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +3 -1
- package/src/service/manifest_loader.ts +9 -5
- package/src/service/materialization_scheduler.spec.ts +260 -0
- package/src/service/materialization_scheduler.ts +304 -0
- package/src/service/materialization_service.spec.ts +167 -4
- package/src/service/materialization_service.ts +121 -8
- package/src/service/materialization_test_fixtures.ts +10 -1
- package/src/service/model.spec.ts +92 -0
- package/src/service/model.ts +58 -7
- package/src/service/package.ts +121 -14
- package/src/service/package_quote_bound_tables.spec.ts +171 -0
- package/src/service/package_reload_safety.spec.ts +193 -0
- package/src/service/persistence_policy.spec.ts +31 -0
- package/src/service/quoting.spec.ts +57 -1
- package/src/service/quoting.ts +33 -0
- package/src/storage/DatabaseInterface.ts +15 -0
- package/src/storage/duckdb/DuckDBRepository.ts +17 -0
- package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
- package/src/storage/duckdb/MaterializationRepository.ts +56 -0
- package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
- package/tests/fixtures/query-givens/data/orders.csv +7 -0
- package/tests/fixtures/query-givens/model.malloy +34 -0
- package/tests/fixtures/query-givens/publisher.json +5 -0
- package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
- package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +22 -22
- package/tests/integration/query_givens/query_givens.integration.spec.ts +146 -0
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +121 -0
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +110 -0
- package/dist/app/assets/EnvironmentPage-DvOJ7L_b.js +0 -1
- package/dist/app/assets/MainPage-BIe0VwBa.js +0 -2
- package/dist/app/assets/MaterializationsPage-BuZ6UJVx.js +0 -1
- package/dist/app/assets/ModelPage-DsPf-s8B.js +0 -1
- package/dist/app/assets/PackagePage-CEVNAKZa.js +0 -1
- package/dist/app/assets/ThemeEditorPage-DWC_FdNU.js +0 -1
- package/dist/app/assets/WorkbookPage-CGrsFz8p.js +0 -1
- package/dist/app/assets/index-DNUZpnaa.js +0 -527
|
@@ -7,10 +7,17 @@ import { isPublisherConfigFrozen } from "../config";
|
|
|
7
7
|
import { TEMP_DIR_PATH } from "../constants";
|
|
8
8
|
import { BadRequestError } from "../errors";
|
|
9
9
|
import { Environment } from "./environment";
|
|
10
|
-
import { EnvironmentStore } from "./environment_store";
|
|
10
|
+
import { EnvironmentStore, resolvePackageLocation } from "./environment_store";
|
|
11
11
|
|
|
12
12
|
type MockData = Record<string, unknown>;
|
|
13
13
|
|
|
14
|
+
// Environments the mock database reports at boot. Empty by default, which makes
|
|
15
|
+
// initialize() fall back to the config file — the path every other test here
|
|
16
|
+
// exercises. Set it to reach the database-restore branch instead, which is the
|
|
17
|
+
// one a real restart against an existing publisher.db takes. Read at call time,
|
|
18
|
+
// so a test can assign it before constructing the store.
|
|
19
|
+
let mockDbEnvironments: unknown[] = [];
|
|
20
|
+
|
|
14
21
|
mock.module("../storage/StorageManager", () => {
|
|
15
22
|
return {
|
|
16
23
|
StorageManager: class MockStorageManager {
|
|
@@ -21,7 +28,8 @@ mock.module("../storage/StorageManager", () => {
|
|
|
21
28
|
getRepository() {
|
|
22
29
|
return {
|
|
23
30
|
// ===== PROJECT METHODS =====
|
|
24
|
-
listEnvironments: async (): Promise<unknown[]> =>
|
|
31
|
+
listEnvironments: async (): Promise<unknown[]> =>
|
|
32
|
+
mockDbEnvironments,
|
|
25
33
|
|
|
26
34
|
getEnvironmentById: async (
|
|
27
35
|
id: string,
|
|
@@ -194,6 +202,10 @@ describe("EnvironmentStore Service", () => {
|
|
|
194
202
|
rmSync(serverRootPath, { recursive: true, force: true });
|
|
195
203
|
}
|
|
196
204
|
mkdirSync(serverRootPath);
|
|
205
|
+
// Default every test back to the config-file boot path. bun runs all specs
|
|
206
|
+
// in one process, so a leaked value would silently switch later tests onto
|
|
207
|
+
// the database-restore branch.
|
|
208
|
+
mockDbEnvironments = [];
|
|
197
209
|
sandbox = sinon.createSandbox();
|
|
198
210
|
|
|
199
211
|
// Mock the configuration to prevent initialization errors
|
|
@@ -212,6 +224,7 @@ describe("EnvironmentStore Service", () => {
|
|
|
212
224
|
rmSync(serverRootPath, { recursive: true, force: true });
|
|
213
225
|
}
|
|
214
226
|
mkdirSync(serverRootPath);
|
|
227
|
+
mockDbEnvironments = [];
|
|
215
228
|
sandbox.restore();
|
|
216
229
|
});
|
|
217
230
|
|
|
@@ -410,6 +423,294 @@ describe("EnvironmentStore Service", () => {
|
|
|
410
423
|
await expect(
|
|
411
424
|
newEnvironmentStore.getEnvironment(invalidProjectName),
|
|
412
425
|
).rejects.toThrow();
|
|
426
|
+
|
|
427
|
+
// The skip is not fatal, so the server still serves. getStatus is the
|
|
428
|
+
// only place that says the environment is missing rather than never
|
|
429
|
+
// configured: without loadErrors the two are indistinguishable.
|
|
430
|
+
const status = await newEnvironmentStore.getStatus();
|
|
431
|
+
expect(status.operationalState).toBe("serving");
|
|
432
|
+
expect(status.environments.map((e) => e.name)).toEqual([
|
|
433
|
+
validProjectName,
|
|
434
|
+
]);
|
|
435
|
+
expect(status.loadErrors).toHaveLength(1);
|
|
436
|
+
expect(status.loadErrors?.[0]?.environment).toBe(invalidProjectName);
|
|
437
|
+
expect(status.loadErrors?.[0]?.package).toBeUndefined();
|
|
438
|
+
expect(status.loadErrors?.[0]?.message).toBeTruthy();
|
|
439
|
+
});
|
|
440
|
+
|
|
441
|
+
it("should omit loadErrors when every environment loads", async () => {
|
|
442
|
+
const projectPath = path.join(serverRootPath, projectName);
|
|
443
|
+
mkdirSync(projectPath, { recursive: true });
|
|
444
|
+
writeFileSync(
|
|
445
|
+
path.join(projectPath, "publisher.json"),
|
|
446
|
+
JSON.stringify({ name: projectName, description: "Test package" }),
|
|
447
|
+
);
|
|
448
|
+
writeFileSync(
|
|
449
|
+
path.join(serverRootPath, "publisher.config.json"),
|
|
450
|
+
JSON.stringify({
|
|
451
|
+
frozenConfig: false,
|
|
452
|
+
environments: [
|
|
453
|
+
{
|
|
454
|
+
name: projectName,
|
|
455
|
+
packages: [{ name: projectName, location: projectPath }],
|
|
456
|
+
connections: [],
|
|
457
|
+
},
|
|
458
|
+
],
|
|
459
|
+
}),
|
|
460
|
+
);
|
|
461
|
+
|
|
462
|
+
const newEnvironmentStore = new EnvironmentStore(serverRootPath);
|
|
463
|
+
await newEnvironmentStore.finishedInitialization;
|
|
464
|
+
|
|
465
|
+
// Absent, not an empty array: a healthy status is unchanged by this
|
|
466
|
+
// field's existence.
|
|
467
|
+
const status = await newEnvironmentStore.getStatus();
|
|
468
|
+
expect(status.operationalState).toBe("serving");
|
|
469
|
+
expect(status.loadErrors).toBeUndefined();
|
|
470
|
+
expect("loadErrors" in status).toBe(false);
|
|
471
|
+
});
|
|
472
|
+
|
|
473
|
+
it("should not report an environment that is serving even if its database sync fails", async () => {
|
|
474
|
+
// An environment reaches this.environments before addEnvironmentToDatabase
|
|
475
|
+
// runs, so a throw from that tail is caught by the same handler that
|
|
476
|
+
// records load failures. The environment is live and listed, so reporting
|
|
477
|
+
// it as a load failure would make the status contradict itself.
|
|
478
|
+
const projectPath = path.join(serverRootPath, projectName);
|
|
479
|
+
mkdirSync(projectPath, { recursive: true });
|
|
480
|
+
writeFileSync(
|
|
481
|
+
path.join(projectPath, "publisher.json"),
|
|
482
|
+
JSON.stringify({ name: projectName }),
|
|
483
|
+
);
|
|
484
|
+
writeFileSync(
|
|
485
|
+
path.join(serverRootPath, "publisher.config.json"),
|
|
486
|
+
JSON.stringify({
|
|
487
|
+
frozenConfig: false,
|
|
488
|
+
environments: [
|
|
489
|
+
{
|
|
490
|
+
name: projectName,
|
|
491
|
+
packages: [{ name: projectName, location: projectPath }],
|
|
492
|
+
connections: [],
|
|
493
|
+
},
|
|
494
|
+
],
|
|
495
|
+
}),
|
|
496
|
+
);
|
|
497
|
+
|
|
498
|
+
// Stub the prototype before constructing: the constructor starts
|
|
499
|
+
// initialize() immediately, so stubbing the instance would race it.
|
|
500
|
+
sandbox
|
|
501
|
+
.stub(EnvironmentStore.prototype, "addEnvironmentToDatabase")
|
|
502
|
+
.rejects(new Error("database write failed"));
|
|
503
|
+
|
|
504
|
+
const newEnvironmentStore = new EnvironmentStore(serverRootPath);
|
|
505
|
+
await newEnvironmentStore.finishedInitialization;
|
|
506
|
+
|
|
507
|
+
const status = await newEnvironmentStore.getStatus();
|
|
508
|
+
expect(status.operationalState).toBe("serving");
|
|
509
|
+
// Still serving it, so it must not also be reported as not loaded.
|
|
510
|
+
expect(status.environments.map((e) => e.name)).toEqual([projectName]);
|
|
511
|
+
expect(status.loadErrors).toBeUndefined();
|
|
512
|
+
});
|
|
513
|
+
|
|
514
|
+
it("should report an environment that failed to restore from the database", async () => {
|
|
515
|
+
// The database-restore branch, not the config branch: initialize() takes
|
|
516
|
+
// this one whenever INITIALIZE_STORAGE is not "true" and the database
|
|
517
|
+
// already holds environments, which is every restart of a server with a
|
|
518
|
+
// persisted publisher.db. It has its own catch, so a failure recorded only
|
|
519
|
+
// on the config path would be dropped on the path production actually
|
|
520
|
+
// takes.
|
|
521
|
+
const envName = "restored-env";
|
|
522
|
+
const envPath = path.join(serverRootPath, envName);
|
|
523
|
+
|
|
524
|
+
// Exists, so the "files missing" branch is skipped, but is a file rather
|
|
525
|
+
// than a directory, so Environment.create throws. Stands in for any stored
|
|
526
|
+
// path that goes bad between restarts.
|
|
527
|
+
writeFileSync(envPath, "not a directory");
|
|
528
|
+
|
|
529
|
+
writeFileSync(
|
|
530
|
+
path.join(serverRootPath, "publisher.config.json"),
|
|
531
|
+
JSON.stringify({
|
|
532
|
+
frozenConfig: false,
|
|
533
|
+
environments: [
|
|
534
|
+
{
|
|
535
|
+
name: envName,
|
|
536
|
+
packages: [{ name: envName, location: envPath }],
|
|
537
|
+
connections: [],
|
|
538
|
+
},
|
|
539
|
+
],
|
|
540
|
+
}),
|
|
541
|
+
);
|
|
542
|
+
|
|
543
|
+
mockDbEnvironments = [
|
|
544
|
+
{
|
|
545
|
+
id: "restored-env-id",
|
|
546
|
+
name: envName,
|
|
547
|
+
path: envPath,
|
|
548
|
+
metadata: {},
|
|
549
|
+
},
|
|
550
|
+
];
|
|
551
|
+
|
|
552
|
+
const newEnvironmentStore = new EnvironmentStore(serverRootPath);
|
|
553
|
+
await newEnvironmentStore.finishedInitialization;
|
|
554
|
+
|
|
555
|
+
const status = await newEnvironmentStore.getStatus();
|
|
556
|
+
expect(status.operationalState).toBe("serving");
|
|
557
|
+
expect(status.environments.map((e) => e.name)).toEqual([]);
|
|
558
|
+
expect(status.loadErrors).toHaveLength(1);
|
|
559
|
+
expect(status.loadErrors?.[0]?.environment).toBe(envName);
|
|
560
|
+
expect(status.loadErrors?.[0]?.message).toBeTruthy();
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
it("should report a package that failed to load while its siblings serve", async () => {
|
|
564
|
+
const goodPackageName = "good-package";
|
|
565
|
+
const badPackageName = "bad-package";
|
|
566
|
+
const goodPackagePath = path.join(serverRootPath, goodPackageName);
|
|
567
|
+
const badPackagePath = path.join(serverRootPath, badPackageName);
|
|
568
|
+
|
|
569
|
+
mkdirSync(goodPackagePath, { recursive: true });
|
|
570
|
+
writeFileSync(
|
|
571
|
+
path.join(goodPackagePath, "publisher.json"),
|
|
572
|
+
JSON.stringify({ name: goodPackageName }),
|
|
573
|
+
);
|
|
574
|
+
// The directory exists, so the environment mounts; the package inside it
|
|
575
|
+
// has no manifest, so only that package fails. This is the second, and
|
|
576
|
+
// quieter, failure mode: the environment is present and serving, with a
|
|
577
|
+
// package missing from it.
|
|
578
|
+
mkdirSync(badPackagePath, { recursive: true });
|
|
579
|
+
|
|
580
|
+
writeFileSync(
|
|
581
|
+
path.join(serverRootPath, "publisher.config.json"),
|
|
582
|
+
JSON.stringify({
|
|
583
|
+
frozenConfig: false,
|
|
584
|
+
environments: [
|
|
585
|
+
{
|
|
586
|
+
name: projectName,
|
|
587
|
+
packages: [
|
|
588
|
+
{ name: goodPackageName, location: goodPackagePath },
|
|
589
|
+
{ name: badPackageName, location: badPackagePath },
|
|
590
|
+
],
|
|
591
|
+
connections: [],
|
|
592
|
+
},
|
|
593
|
+
],
|
|
594
|
+
}),
|
|
595
|
+
);
|
|
596
|
+
|
|
597
|
+
const newEnvironmentStore = new EnvironmentStore(serverRootPath);
|
|
598
|
+
await newEnvironmentStore.finishedInitialization;
|
|
599
|
+
|
|
600
|
+
const environment = await newEnvironmentStore.getEnvironment(projectName);
|
|
601
|
+
const packages = await environment.listPackages();
|
|
602
|
+
expect(packages.map((p) => p.name)).toEqual([goodPackageName]);
|
|
603
|
+
|
|
604
|
+
const status = await newEnvironmentStore.getStatus();
|
|
605
|
+
expect(status.operationalState).toBe("serving");
|
|
606
|
+
expect(status.loadErrors).toHaveLength(1);
|
|
607
|
+
expect(status.loadErrors?.[0]?.environment).toBe(projectName);
|
|
608
|
+
expect(status.loadErrors?.[0]?.package).toBe(badPackageName);
|
|
609
|
+
expect(status.loadErrors?.[0]?.message).toBeTruthy();
|
|
610
|
+
});
|
|
611
|
+
|
|
612
|
+
it("should stop reporting a failed package once it is deleted", async () => {
|
|
613
|
+
const goodPackageName = "good-package";
|
|
614
|
+
const badPackageName = "bad-package";
|
|
615
|
+
const goodPackagePath = path.join(serverRootPath, goodPackageName);
|
|
616
|
+
const badPackagePath = path.join(serverRootPath, badPackageName);
|
|
617
|
+
|
|
618
|
+
mkdirSync(goodPackagePath, { recursive: true });
|
|
619
|
+
writeFileSync(
|
|
620
|
+
path.join(goodPackagePath, "publisher.json"),
|
|
621
|
+
JSON.stringify({ name: goodPackageName }),
|
|
622
|
+
);
|
|
623
|
+
mkdirSync(badPackagePath, { recursive: true });
|
|
624
|
+
|
|
625
|
+
writeFileSync(
|
|
626
|
+
path.join(serverRootPath, "publisher.config.json"),
|
|
627
|
+
JSON.stringify({
|
|
628
|
+
frozenConfig: false,
|
|
629
|
+
environments: [
|
|
630
|
+
{
|
|
631
|
+
name: projectName,
|
|
632
|
+
packages: [
|
|
633
|
+
{ name: goodPackageName, location: goodPackagePath },
|
|
634
|
+
{ name: badPackageName, location: badPackagePath },
|
|
635
|
+
],
|
|
636
|
+
connections: [],
|
|
637
|
+
},
|
|
638
|
+
],
|
|
639
|
+
}),
|
|
640
|
+
);
|
|
641
|
+
|
|
642
|
+
const newEnvironmentStore = new EnvironmentStore(serverRootPath);
|
|
643
|
+
await newEnvironmentStore.finishedInitialization;
|
|
644
|
+
|
|
645
|
+
const environment = await newEnvironmentStore.getEnvironment(projectName);
|
|
646
|
+
expect((await newEnvironmentStore.getStatus()).loadErrors).toHaveLength(
|
|
647
|
+
1,
|
|
648
|
+
);
|
|
649
|
+
|
|
650
|
+
// A package that failed to load was evicted from `packages`, so the
|
|
651
|
+
// delete takes deletePackage's early return. The failure entry has to be
|
|
652
|
+
// cleared anyway: the caller goes on to drop the package's config row,
|
|
653
|
+
// and a loadError naming a package that is no longer configured sends
|
|
654
|
+
// whoever reads /status hunting for something that isn't there.
|
|
655
|
+
await environment.deletePackage(badPackageName);
|
|
656
|
+
|
|
657
|
+
const status = await newEnvironmentStore.getStatus();
|
|
658
|
+
expect(status.operationalState).toBe("serving");
|
|
659
|
+
expect(status.loadErrors).toBeUndefined();
|
|
660
|
+
expect((await environment.listPackages()).map((p) => p.name)).toEqual([
|
|
661
|
+
goodPackageName,
|
|
662
|
+
]);
|
|
663
|
+
});
|
|
664
|
+
|
|
665
|
+
it("should stop reporting a failed package once it loads", async () => {
|
|
666
|
+
const badPackageName = "bad-package";
|
|
667
|
+
const badPackagePath = path.join(serverRootPath, badPackageName);
|
|
668
|
+
|
|
669
|
+
// No manifest, so it fails to load and lands in loadErrors.
|
|
670
|
+
mkdirSync(badPackagePath, { recursive: true });
|
|
671
|
+
|
|
672
|
+
writeFileSync(
|
|
673
|
+
path.join(serverRootPath, "publisher.config.json"),
|
|
674
|
+
JSON.stringify({
|
|
675
|
+
frozenConfig: false,
|
|
676
|
+
environments: [
|
|
677
|
+
{
|
|
678
|
+
name: projectName,
|
|
679
|
+
packages: [
|
|
680
|
+
{ name: badPackageName, location: badPackagePath },
|
|
681
|
+
],
|
|
682
|
+
connections: [],
|
|
683
|
+
},
|
|
684
|
+
],
|
|
685
|
+
}),
|
|
686
|
+
);
|
|
687
|
+
|
|
688
|
+
const newEnvironmentStore = new EnvironmentStore(serverRootPath);
|
|
689
|
+
await newEnvironmentStore.finishedInitialization;
|
|
690
|
+
|
|
691
|
+
const environment = await newEnvironmentStore.getEnvironment(projectName);
|
|
692
|
+
expect((await newEnvironmentStore.getStatus()).loadErrors).toHaveLength(
|
|
693
|
+
1,
|
|
694
|
+
);
|
|
695
|
+
|
|
696
|
+
// Fix the package where the environment actually reads it, then re-add
|
|
697
|
+
// it. A package that is serving must not still be reported as failed.
|
|
698
|
+
writeFileSync(
|
|
699
|
+
path.join(
|
|
700
|
+
environment.metadata.location as string,
|
|
701
|
+
badPackageName,
|
|
702
|
+
"publisher.json",
|
|
703
|
+
),
|
|
704
|
+
JSON.stringify({ name: badPackageName }),
|
|
705
|
+
);
|
|
706
|
+
await environment.addPackage(badPackageName);
|
|
707
|
+
|
|
708
|
+
const status = await newEnvironmentStore.getStatus();
|
|
709
|
+
expect(status.operationalState).toBe("serving");
|
|
710
|
+
expect(status.loadErrors).toBeUndefined();
|
|
711
|
+
expect((await environment.listPackages()).map((p) => p.name)).toEqual([
|
|
712
|
+
badPackageName,
|
|
713
|
+
]);
|
|
413
714
|
});
|
|
414
715
|
|
|
415
716
|
it("should handle project updates", async () => {
|
|
@@ -1042,3 +1343,72 @@ describe("EnvironmentStore path-injection guards", () => {
|
|
|
1042
1343
|
);
|
|
1043
1344
|
});
|
|
1044
1345
|
});
|
|
1346
|
+
|
|
1347
|
+
describe("resolvePackageLocation", () => {
|
|
1348
|
+
// Built with `path`, not literals: the helper joins with the platform
|
|
1349
|
+
// separator and these specs run on windows-latest too (cross-platform-tests).
|
|
1350
|
+
const HOME = path.join(path.sep, "home", "tester");
|
|
1351
|
+
const CONFIG_DIR = path.join(path.sep, "etc", "publisher");
|
|
1352
|
+
|
|
1353
|
+
it("expands ~/ against the home directory", () => {
|
|
1354
|
+
expect(
|
|
1355
|
+
resolvePackageLocation("~/my-packages/sales", CONFIG_DIR, HOME),
|
|
1356
|
+
).toBe(path.join(HOME, "my-packages", "sales"));
|
|
1357
|
+
});
|
|
1358
|
+
|
|
1359
|
+
it("does not anchor an expanded ~/ path at the config dir", () => {
|
|
1360
|
+
// Regression: `~/` satisfies isLocalPath but not path.isAbsolute, so an
|
|
1361
|
+
// unexpanded tilde used to be joined onto the anchor, yielding
|
|
1362
|
+
// `<anchor>/~/my-packages/sales`.
|
|
1363
|
+
const resolved = resolvePackageLocation("~/x", CONFIG_DIR, HOME);
|
|
1364
|
+
expect(resolved).not.toContain("~");
|
|
1365
|
+
expect(resolved.startsWith(CONFIG_DIR)).toBe(false);
|
|
1366
|
+
});
|
|
1367
|
+
|
|
1368
|
+
it("returns an absolute location untouched", () => {
|
|
1369
|
+
const absolute = path.join(path.sep, "srv", "packages", "sales");
|
|
1370
|
+
expect(resolvePackageLocation(absolute, CONFIG_DIR, HOME)).toBe(absolute);
|
|
1371
|
+
});
|
|
1372
|
+
|
|
1373
|
+
it("anchors a relative location at the config dir, not the cwd", () => {
|
|
1374
|
+
expect(resolvePackageLocation("./sales", CONFIG_DIR, HOME)).toBe(
|
|
1375
|
+
path.join(CONFIG_DIR, "sales"),
|
|
1376
|
+
);
|
|
1377
|
+
});
|
|
1378
|
+
|
|
1379
|
+
it("anchors a ../ location at the config dir", () => {
|
|
1380
|
+
expect(
|
|
1381
|
+
resolvePackageLocation("../examples/storefront", CONFIG_DIR, HOME),
|
|
1382
|
+
).toBe(path.join(CONFIG_DIR, "..", "examples", "storefront"));
|
|
1383
|
+
});
|
|
1384
|
+
|
|
1385
|
+
it("throws when ~/ cannot be expanded because no home directory is set", () => {
|
|
1386
|
+
// An empty home must fail loudly: `path.join("", "x")` would otherwise
|
|
1387
|
+
// yield a relative "x" that silently anchors under the config dir.
|
|
1388
|
+
expect(() => resolvePackageLocation("~/x", CONFIG_DIR, "")).toThrow(
|
|
1389
|
+
/home directory is not set/,
|
|
1390
|
+
);
|
|
1391
|
+
});
|
|
1392
|
+
|
|
1393
|
+
it("never consults the home directory for non-tilde locations", () => {
|
|
1394
|
+
// The same empty home that makes a ~/ location throw must be irrelevant
|
|
1395
|
+
// to ./ and absolute locations; home is resolved lazily, tilde-only.
|
|
1396
|
+
expect(resolvePackageLocation("./sales", CONFIG_DIR, "")).toBe(
|
|
1397
|
+
path.join(CONFIG_DIR, "sales"),
|
|
1398
|
+
);
|
|
1399
|
+
const absolute = path.join(path.sep, "srv", "packages", "sales");
|
|
1400
|
+
expect(resolvePackageLocation(absolute, CONFIG_DIR, "")).toBe(absolute);
|
|
1401
|
+
});
|
|
1402
|
+
|
|
1403
|
+
it("expands only the POSIX ~/ prefix; bare ~ and ~user are not home paths", () => {
|
|
1404
|
+
// Neither form satisfies isLocalPath, so in practice they are rejected
|
|
1405
|
+
// upstream as non-local locations; pin here that the resolver itself
|
|
1406
|
+
// never treats them as home-relative either.
|
|
1407
|
+
expect(resolvePackageLocation("~", CONFIG_DIR, HOME)).toBe(
|
|
1408
|
+
path.join(CONFIG_DIR, "~"),
|
|
1409
|
+
);
|
|
1410
|
+
expect(resolvePackageLocation("~user/foo", CONFIG_DIR, HOME)).toBe(
|
|
1411
|
+
path.join(CONFIG_DIR, "~user", "foo"),
|
|
1412
|
+
);
|
|
1413
|
+
});
|
|
1414
|
+
});
|
|
@@ -4,12 +4,14 @@ import { Mutex } from "async-mutex";
|
|
|
4
4
|
import crypto from "crypto";
|
|
5
5
|
import extract from "extract-zip";
|
|
6
6
|
import * as fs from "fs";
|
|
7
|
+
import * as os from "os";
|
|
7
8
|
import * as path from "path";
|
|
8
9
|
import simpleGit from "simple-git";
|
|
9
10
|
import { Writable } from "stream";
|
|
10
11
|
import { components } from "../api";
|
|
11
12
|
import {
|
|
12
13
|
getProcessedPublisherConfig,
|
|
14
|
+
getPublisherConfigDir,
|
|
13
15
|
isPublisherConfigFrozen,
|
|
14
16
|
ProcessedEnvironment,
|
|
15
17
|
ProcessedPublisherConfig,
|
|
@@ -27,6 +29,7 @@ import {
|
|
|
27
29
|
} from "../errors";
|
|
28
30
|
import { getOperationalState, markNotReady, markReady } from "../health";
|
|
29
31
|
import { formatDuration, logger } from "../logger";
|
|
32
|
+
import { redactPgSecrets } from "../pg_helpers";
|
|
30
33
|
import {
|
|
31
34
|
assertSafeEnvironmentPath,
|
|
32
35
|
assertSafePackageName,
|
|
@@ -37,6 +40,9 @@ import { StorageConfig, StorageManager } from "../storage/StorageManager";
|
|
|
37
40
|
import { Environment, PackageStatus } from "./environment";
|
|
38
41
|
import type { PackageMemoryGovernor } from "./package_memory_governor";
|
|
39
42
|
type ApiEnvironment = components["schemas"]["Environment"];
|
|
43
|
+
type LoadError = NonNullable<
|
|
44
|
+
components["schemas"]["ServerStatus"]["loadErrors"]
|
|
45
|
+
>[number];
|
|
40
46
|
|
|
41
47
|
const AZURE_SUPPORTED_SCHEMES = ["https://", "http://", "abfss://", "az://"];
|
|
42
48
|
const AZURE_DATA_EXTENSIONS = [
|
|
@@ -115,9 +121,49 @@ async function clearMountTarget(targetPath: string): Promise<void> {
|
|
|
115
121
|
}
|
|
116
122
|
}
|
|
117
123
|
|
|
124
|
+
/**
|
|
125
|
+
* Absolute on-disk path for a local package `location`.
|
|
126
|
+
*
|
|
127
|
+
* `~/` (POSIX form only) expands to the home directory. Anything still
|
|
128
|
+
* relative resolves against `anchorDir`, the directory holding the active
|
|
129
|
+
* config, so a config and the packages it points at can be moved or
|
|
130
|
+
* committed together. Absolute locations are returned untouched.
|
|
131
|
+
*/
|
|
132
|
+
export function resolvePackageLocation(
|
|
133
|
+
location: string,
|
|
134
|
+
anchorDir: string,
|
|
135
|
+
homeDir?: string,
|
|
136
|
+
): string {
|
|
137
|
+
let expanded = location;
|
|
138
|
+
if (location.startsWith("~/")) {
|
|
139
|
+
// Resolved lazily, inside the tilde branch only: os.homedir() can throw
|
|
140
|
+
// where HOME is unset and there is no passwd entry for the uid (e.g. a
|
|
141
|
+
// distroless container with runAsUser), and a `./sales` or absolute
|
|
142
|
+
// location must never fail on that.
|
|
143
|
+
const home = homeDir ?? os.homedir();
|
|
144
|
+
if (!home) {
|
|
145
|
+
throw new Error(
|
|
146
|
+
`Cannot expand "~" in location "${location}": home directory is not set`,
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
expanded = path.join(home, location.slice(2));
|
|
150
|
+
}
|
|
151
|
+
return path.isAbsolute(expanded) ? expanded : path.join(anchorDir, expanded);
|
|
152
|
+
}
|
|
153
|
+
|
|
118
154
|
export class EnvironmentStore {
|
|
119
155
|
public serverRootPath: string;
|
|
120
156
|
private environments: Map<string, Environment> = new Map();
|
|
157
|
+
/**
|
|
158
|
+
* Environments that were configured but did not load, keyed by name.
|
|
159
|
+
*
|
|
160
|
+
* A load failure is not fatal: the environment is skipped and the server
|
|
161
|
+
* still reports `serving`. Nothing else remembers it, because the throw
|
|
162
|
+
* happens before the environment reaches `this.environments` or the
|
|
163
|
+
* database, so this is the only record that a configured environment is
|
|
164
|
+
* missing rather than never asked for. Surfaced on `getStatus`.
|
|
165
|
+
*/
|
|
166
|
+
private failedEnvironments = new Map<string, string>();
|
|
121
167
|
private environmentMutexes = new Map<string, Mutex>();
|
|
122
168
|
public publisherConfigIsFrozen: boolean;
|
|
123
169
|
public finishedInitialization: Promise<void>;
|
|
@@ -197,6 +243,15 @@ export class EnvironmentStore {
|
|
|
197
243
|
}
|
|
198
244
|
}
|
|
199
245
|
|
|
246
|
+
/**
|
|
247
|
+
* Snapshot of the environments currently held in memory. Used by the
|
|
248
|
+
* standalone materialization scheduler to sweep loaded packages without
|
|
249
|
+
* touching the database or triggering loads.
|
|
250
|
+
*/
|
|
251
|
+
public getLoadedEnvironments(): Environment[] {
|
|
252
|
+
return [...this.environments.values()];
|
|
253
|
+
}
|
|
254
|
+
|
|
200
255
|
private async addConfiguredEnvironment(environment: ProcessedEnvironment) {
|
|
201
256
|
try {
|
|
202
257
|
await this.addEnvironment(
|
|
@@ -209,11 +264,21 @@ export class EnvironmentStore {
|
|
|
209
264
|
true,
|
|
210
265
|
);
|
|
211
266
|
} catch (error) {
|
|
212
|
-
this.
|
|
267
|
+
this.recordEnvironmentInitializationFailure(environment.name, error);
|
|
213
268
|
}
|
|
214
269
|
}
|
|
215
270
|
|
|
216
|
-
|
|
271
|
+
/**
|
|
272
|
+
* Log a skipped environment AND remember why, for getStatus to report.
|
|
273
|
+
*
|
|
274
|
+
* Both boot paths funnel through here: the config path
|
|
275
|
+
* ({@link addConfiguredEnvironment}) and the database-restore path taken by
|
|
276
|
+
* every restart against an existing publisher.db. Keeping the record here
|
|
277
|
+
* rather than in the callers is deliberate: recording it in only one of them
|
|
278
|
+
* would leave the other silently dropping failures, which is the bug this is
|
|
279
|
+
* meant to fix.
|
|
280
|
+
*/
|
|
281
|
+
private recordEnvironmentInitializationFailure(
|
|
217
282
|
environmentName: string | undefined,
|
|
218
283
|
error: unknown,
|
|
219
284
|
) {
|
|
@@ -222,6 +287,24 @@ export class EnvironmentStore {
|
|
|
222
287
|
`Error initializing environment${label}; skipping environment`,
|
|
223
288
|
this.extractErrorDataFromError(error),
|
|
224
289
|
);
|
|
290
|
+
// Only record an environment that is genuinely not there. An environment
|
|
291
|
+
// reaches `this.environments` before the rest of its setup (the database
|
|
292
|
+
// sync, its package listing) is done, and a throw from that tail lands
|
|
293
|
+
// here while the environment is live and serving. Reporting it would
|
|
294
|
+
// contradict `environments`, which still lists it, and a loadErrors entry
|
|
295
|
+
// with no `package` means the whole environment was skipped. It also could
|
|
296
|
+
// never be cleared: the clear below only runs when an environment is
|
|
297
|
+
// created, so a live one would carry the entry until restart.
|
|
298
|
+
if (environmentName && !this.environments.has(environmentName)) {
|
|
299
|
+
// Redacted because this is bound for an HTTP response body, not just a
|
|
300
|
+
// log line; see the same call in environment.ts.
|
|
301
|
+
this.failedEnvironments.set(
|
|
302
|
+
environmentName,
|
|
303
|
+
redactPgSecrets(
|
|
304
|
+
error instanceof Error ? error.message : String(error),
|
|
305
|
+
),
|
|
306
|
+
);
|
|
307
|
+
}
|
|
225
308
|
}
|
|
226
309
|
|
|
227
310
|
private async initialize() {
|
|
@@ -340,7 +423,7 @@ export class EnvironmentStore {
|
|
|
340
423
|
|
|
341
424
|
return environmentInstance.listPackages();
|
|
342
425
|
} catch (error) {
|
|
343
|
-
this.
|
|
426
|
+
this.recordEnvironmentInitializationFailure(
|
|
344
427
|
dbEnvironment.name,
|
|
345
428
|
error,
|
|
346
429
|
);
|
|
@@ -759,9 +842,16 @@ export class EnvironmentStore {
|
|
|
759
842
|
: baseState
|
|
760
843
|
) as components["schemas"]["ServerStatus"]["operationalState"];
|
|
761
844
|
|
|
762
|
-
const status
|
|
845
|
+
const status: {
|
|
846
|
+
timestamp: number;
|
|
847
|
+
environments: Array<components["schemas"]["Environment"]>;
|
|
848
|
+
initialized: boolean;
|
|
849
|
+
frozenConfig: boolean;
|
|
850
|
+
operationalState: components["schemas"]["ServerStatus"]["operationalState"];
|
|
851
|
+
loadErrors?: LoadError[];
|
|
852
|
+
} = {
|
|
763
853
|
timestamp: Date.now(),
|
|
764
|
-
environments: []
|
|
854
|
+
environments: [],
|
|
765
855
|
initialized: this.isInitialized,
|
|
766
856
|
frozenConfig: isPublisherConfigFrozen(this.serverRootPath),
|
|
767
857
|
operationalState,
|
|
@@ -803,6 +893,34 @@ export class EnvironmentStore {
|
|
|
803
893
|
}
|
|
804
894
|
}),
|
|
805
895
|
);
|
|
896
|
+
|
|
897
|
+
// Collected AFTER listEnvironments, not before: serialize() awaits
|
|
898
|
+
// listPackages(), which is what loads an environment's packages and so
|
|
899
|
+
// records their failures. Reading the maps at the top of this method risks
|
|
900
|
+
// reading them before the failures they report have been recorded.
|
|
901
|
+
//
|
|
902
|
+
// A read-time overlay over live state, like "throttled" above, so there is
|
|
903
|
+
// no fourth stored status to keep in sync.
|
|
904
|
+
const loadErrors: LoadError[] = [];
|
|
905
|
+
for (const [environmentName, message] of this.failedEnvironments) {
|
|
906
|
+
loadErrors.push({ environment: environmentName, message });
|
|
907
|
+
}
|
|
908
|
+
for (const [environmentName, environment] of this.environments) {
|
|
909
|
+
for (const [packageName, message] of environment.getFailedPackages()) {
|
|
910
|
+
loadErrors.push({
|
|
911
|
+
environment: environmentName,
|
|
912
|
+
package: packageName,
|
|
913
|
+
message,
|
|
914
|
+
});
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
// Left unset when nothing failed, so the key is absent from the response
|
|
918
|
+
// and a healthy server's status body is byte-for-byte what it was before
|
|
919
|
+
// this field existed.
|
|
920
|
+
if (loadErrors.length > 0) {
|
|
921
|
+
status.loadErrors = loadErrors;
|
|
922
|
+
}
|
|
923
|
+
|
|
806
924
|
return status;
|
|
807
925
|
}
|
|
808
926
|
|
|
@@ -921,6 +1039,11 @@ export class EnvironmentStore {
|
|
|
921
1039
|
newEnvironment.metadata.location = absoluteEnvironmentPath;
|
|
922
1040
|
|
|
923
1041
|
this.environments.set(environmentName, newEnvironment);
|
|
1042
|
+
// It loaded, so any earlier failure for this name is stale. A boot
|
|
1043
|
+
// failure can be followed by a successful POST or lazy load of the same
|
|
1044
|
+
// environment, and a status that keeps reporting the old error would lie
|
|
1045
|
+
// in the other direction.
|
|
1046
|
+
this.failedEnvironments.delete(environmentName);
|
|
924
1047
|
|
|
925
1048
|
environment?.packages?.forEach((_package) => {
|
|
926
1049
|
if (_package.name) {
|
|
@@ -1110,6 +1233,21 @@ export class EnvironmentStore {
|
|
|
1110
1233
|
);
|
|
1111
1234
|
}
|
|
1112
1235
|
|
|
1236
|
+
/**
|
|
1237
|
+
* Absolute on-disk path for a local package `location`, anchored at the
|
|
1238
|
+
* directory holding the active config. That covers locations POSTed at
|
|
1239
|
+
* runtime too, which no config declares. When there is no config worth
|
|
1240
|
+
* anchoring to, `getPublisherConfigDir` returns null (it owns the cases) and
|
|
1241
|
+
* the server root is the only meaningful base, which is also what this did
|
|
1242
|
+
* before the anchor moved.
|
|
1243
|
+
*/
|
|
1244
|
+
private resolveLocalPath(location: string): string {
|
|
1245
|
+
return resolvePackageLocation(
|
|
1246
|
+
location,
|
|
1247
|
+
getPublisherConfigDir(this.serverRootPath) ?? this.serverRootPath,
|
|
1248
|
+
);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1113
1251
|
private isGitHubURL(location: string) {
|
|
1114
1252
|
return (
|
|
1115
1253
|
location.startsWith("https://github.com/") ||
|
|
@@ -1236,15 +1374,11 @@ export class EnvironmentStore {
|
|
|
1236
1374
|
} else {
|
|
1237
1375
|
// For non-GitHub locations, use package name
|
|
1238
1376
|
if (this.isLocalPath(_package.location)) {
|
|
1239
|
-
//
|
|
1240
|
-
//
|
|
1241
|
-
//
|
|
1242
|
-
//
|
|
1243
|
-
|
|
1244
|
-
// branch is unreachable.
|
|
1245
|
-
sourcePath = path.isAbsolute(_package.location)
|
|
1246
|
-
? _package.location
|
|
1247
|
-
: path.join(this.serverRootPath, _package.location);
|
|
1377
|
+
// Same resolution rule as `downloadOrMountLocation`.
|
|
1378
|
+
// Without this step the existing-source check below
|
|
1379
|
+
// falls through for any relative location, and the
|
|
1380
|
+
// in-place mount branch is unreachable.
|
|
1381
|
+
sourcePath = this.resolveLocalPath(_package.location);
|
|
1248
1382
|
} else {
|
|
1249
1383
|
sourcePath = safeJoinUnderRoot(
|
|
1250
1384
|
tempDownloadPath,
|
|
@@ -1456,9 +1590,7 @@ export class EnvironmentStore {
|
|
|
1456
1590
|
|
|
1457
1591
|
// Handle absolute and relative paths
|
|
1458
1592
|
if (this.isLocalPath(location)) {
|
|
1459
|
-
const packagePath: string =
|
|
1460
|
-
? location
|
|
1461
|
-
: path.join(this.serverRootPath, location);
|
|
1593
|
+
const packagePath: string = this.resolveLocalPath(location);
|
|
1462
1594
|
try {
|
|
1463
1595
|
logger.info(
|
|
1464
1596
|
`Mounting local directory at "${packagePath}" to "${targetPath}"`,
|