@checkstack/backend 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/package.json +5 -5
- package/src/index.ts +67 -22
- package/src/plugin-manager.ts +148 -27
- package/src/services/plugin-installers/github-installer.ts +4 -1
- package/src/services/plugin-installers/install-from-tarball.it.test.ts +122 -0
- package/src/services/plugin-installers/install-from-tarball.ts +109 -0
- package/src/services/plugin-installers/tarball-installer.ts +6 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,54 @@
|
|
|
1
1
|
# @checkstack/backend
|
|
2
2
|
|
|
3
|
+
## 0.17.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- @checkstack/api-docs-common@0.1.18
|
|
8
|
+
- @checkstack/auth-common@0.8.2
|
|
9
|
+
- @checkstack/backend-api@0.21.3
|
|
10
|
+
- @checkstack/cache-api@0.3.11
|
|
11
|
+
- @checkstack/common@0.14.1
|
|
12
|
+
- @checkstack/pluginmanager-common@0.2.7
|
|
13
|
+
- @checkstack/queue-api@0.3.11
|
|
14
|
+
- @checkstack/signal-backend@0.3.3
|
|
15
|
+
- @checkstack/signal-common@0.2.8
|
|
16
|
+
|
|
17
|
+
## 0.17.0
|
|
18
|
+
|
|
19
|
+
### Minor Changes
|
|
20
|
+
|
|
21
|
+
- 968c12f: Make installed (runtime) frontend plugins actually load, via Module Federation 2.0. Previously a packed external plugin's frontend could not run: the host only shared React/router with runtime plugins, and there was no working way to share the framework/UI singletons (hand-rolled import-map externalisation hit an unsolvable rolldown CJS-interop wall).
|
|
22
|
+
|
|
23
|
+
- **Host (`@checkstack/frontend`)** now uses `@module-federation/vite` as an MF host and loads runtime plugins through the MF runtime (`registerRemotes` + `loadRemote`) instead of a raw `import()`. The shared set (react, react-dom, react-router-dom, @tanstack/react-query, @checkstack/frontend-api) is owned by the host; plugins reuse those exact instances via the share scope. The old hand-rolled vendor build + import map are removed.
|
|
24
|
+
- **`@checkstack/ui`** is bundled per consumer (tree-shaken); its Theme / Toast / Performance React contexts are unified across the host and bundled-in-plugin copies via a registered (globalThis-keyed) context, so a plugin's `useTheme`/`useToast`/`usePerformance` resolve to the host's providers. The ONE exception is the Monaco / VS Code **CodeEditor**, now exposed as the `@checkstack/ui/code-editor` subpath and shared as an MF singleton: the host owns the single editor instance (and builds its `?worker&url` workers), and plugins reuse it. A plugin can now render `<CodeEditor>` (directly or via `ScriptTestPanel` / template/JSON fields) without bundling Monaco.
|
|
25
|
+
- **Scaffold + pack (`@checkstack/scripts`)** build frontend plugins as MF remotes (`vite build` with the federation plugin, exposing `./plugin`, manifest enabled, DTS disabled). The CodeEditor is shared with `import: false` so the plugin is a consume-only participant - it never bundles a local fallback of the editor, keeping the heavy `@codingame/*` / `monaco-languageclient` / `vscode` subtree out of the plugin entirely (so no `vscode` alias or ES-worker config is needed in the plugin build). `plugin-pack` builds frontend packages with `NODE_ENV=production` (the MF plugin skips the remote under `NODE_ENV=test`) and ships only `dist/`. The scaffolded route now declares a `nav` entry so it appears in the sidebar.
|
|
26
|
+
- **Backend (`@checkstack/backend`)** serves a plugin's MF assets under its (possibly scoped) package name (`/assets/plugins/@scope/name/*`), with correct content types, and the SPA catch-all defers those paths so the federation manifest/remoteEntry are not shadowed by `index.html`.
|
|
27
|
+
|
|
28
|
+
Verified end-to-end by the external-plugin install E2E (scaffold → pack → install via the Plugin Manager UI → frontend + backend + co-loaded core plugins all work).
|
|
29
|
+
|
|
30
|
+
### Patch Changes
|
|
31
|
+
|
|
32
|
+
- e434d62: Fix the runtime (installed) plugin path so an external plugin uploaded via the Plugin Manager actually installs and its backend loads. Five distinct defects, surfaced by a new full install E2E:
|
|
33
|
+
|
|
34
|
+
- **Plugin Manager access denied for admins.** The Plugin Manager's core access rules were registered _after_ `loadPlugins`, so the auth full-sync never wrote them to the DB; and the hand-rolled `upload-tarball` route checked `accessRules.includes(rule)` without honoring the admin `"*"` wildcard. Rules now register before `loadPlugins`, and the route honors `"*"` (matching `openapi-router.ts`).
|
|
35
|
+
- **Bundle installs 404'd on intra-bundle deps.** A bundle's siblings were installed one tarball at a time, so a sibling that depends on another sibling failed to resolve against the registry. `installBundleFromArtifacts` now installs the whole bundle via a throwaway manifest using `file:` deps + `overrides`, resolving siblings locally and merging the result into the shared runtime dir.
|
|
36
|
+
- **Primary artifact was the outer bundle archive.** The tarball/github installers stored the outer `bundle.json` archive as the primary's artifact instead of the primary's own package tarball; they now store the inner package tarball.
|
|
37
|
+
- **Non-backend siblings loaded as backend plugins.** The install broadcast tried to load `common`/`frontend` siblings as backend plugins ("does not export a valid BackendPlugin"). Only `type: "backend"` packages now register as backend plugins (mirroring fresh-instance bootstrap).
|
|
38
|
+
- **Runtime backend never migrated or got a scoped DB.** `loadSinglePlugin` now runs the plugin's Drizzle migrations into its isolated schema and injects the plugin-scoped `database` into `init`, matching the full-system loader.
|
|
39
|
+
|
|
40
|
+
Note: the installed _frontend_ half of a runtime plugin remains a known gap (the host only shares React/router with runtime plugins, and `plugin-pack` does not build frontends); tracked separately for a follow-up.
|
|
41
|
+
|
|
42
|
+
- @checkstack/api-docs-common@0.1.18
|
|
43
|
+
- @checkstack/auth-common@0.8.2
|
|
44
|
+
- @checkstack/backend-api@0.21.2
|
|
45
|
+
- @checkstack/cache-api@0.3.11
|
|
46
|
+
- @checkstack/common@0.14.1
|
|
47
|
+
- @checkstack/pluginmanager-common@0.2.7
|
|
48
|
+
- @checkstack/queue-api@0.3.11
|
|
49
|
+
- @checkstack/signal-backend@0.3.2
|
|
50
|
+
- @checkstack/signal-common@0.2.8
|
|
51
|
+
|
|
3
52
|
## 0.16.2
|
|
4
53
|
|
|
5
54
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@checkstack/backend",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.1",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"checkstack": {
|
|
6
6
|
"type": "backend"
|
|
@@ -16,12 +16,12 @@
|
|
|
16
16
|
"dependencies": {
|
|
17
17
|
"@checkstack/api-docs-common": "0.1.18",
|
|
18
18
|
"@checkstack/auth-common": "0.8.2",
|
|
19
|
-
"@checkstack/backend-api": "0.21.
|
|
19
|
+
"@checkstack/backend-api": "0.21.3",
|
|
20
20
|
"@checkstack/common": "0.14.1",
|
|
21
21
|
"@checkstack/drizzle-helper": "0.0.5",
|
|
22
22
|
"@checkstack/cache-api": "0.3.11",
|
|
23
23
|
"@checkstack/queue-api": "0.3.11",
|
|
24
|
-
"@checkstack/signal-backend": "0.3.
|
|
24
|
+
"@checkstack/signal-backend": "0.3.3",
|
|
25
25
|
"@checkstack/signal-common": "0.2.8",
|
|
26
26
|
"@checkstack/pluginmanager-common": "0.2.7",
|
|
27
27
|
"@hono/zod-validator": "^0.7.6",
|
|
@@ -45,8 +45,8 @@
|
|
|
45
45
|
"@types/bun": "latest",
|
|
46
46
|
"@types/semver": "^7.5.0",
|
|
47
47
|
"@checkstack/tsconfig": "0.0.7",
|
|
48
|
-
"@checkstack/scripts": "0.
|
|
49
|
-
"@checkstack/test-utils-backend": "0.1.
|
|
48
|
+
"@checkstack/scripts": "0.6.0",
|
|
49
|
+
"@checkstack/test-utils-backend": "0.1.37",
|
|
50
50
|
"drizzle-kit": "^0.31.10"
|
|
51
51
|
}
|
|
52
52
|
}
|
package/src/index.ts
CHANGED
|
@@ -395,7 +395,11 @@ if (frontendDistPath && fs.existsSync(frontendDistPath)) {
|
|
|
395
395
|
// handlers (oRPC RPC + OpenAPI REST mounts).
|
|
396
396
|
const apiPath =
|
|
397
397
|
c.req.path.startsWith("/api/") || c.req.path.startsWith("/rest/");
|
|
398
|
-
|
|
398
|
+
// Runtime frontend-plugin assets are served by the `/assets/plugins/*`
|
|
399
|
+
// route registered later during init. Defer to it here, otherwise this SPA
|
|
400
|
+
// fallback would return index.html for a plugin's mf-manifest.json /
|
|
401
|
+
// remoteEntry.js and the Module Federation runtime would fail (#RUNTIME-003).
|
|
402
|
+
if (apiPath || c.req.path.startsWith("/assets/plugins/")) {
|
|
399
403
|
return next();
|
|
400
404
|
}
|
|
401
405
|
|
|
@@ -522,7 +526,17 @@ const init = async () => {
|
|
|
522
526
|
"accessRules" in user ? user.accessRules : []
|
|
523
527
|
) as string[];
|
|
524
528
|
const anonymous = await authService.getAnonymousAccessRules();
|
|
525
|
-
|
|
529
|
+
// `enrichUser` collapses the admin role to the `"*"` wildcard (it never
|
|
530
|
+
// materialises every individual rule id), so an admin's `accessRules` is
|
|
531
|
+
// just `["*"]`. Honour that wildcard the same way the autoAuthMiddleware
|
|
532
|
+
// and oRPC procedures do (see auth-backend router.ts) - a bare
|
|
533
|
+
// `includes(requiredAccess)` would 403 every admin on this hand-rolled
|
|
534
|
+
// multipart route.
|
|
535
|
+
const hasAccess =
|
|
536
|
+
accessRules.includes("*") ||
|
|
537
|
+
accessRules.includes(requiredAccess) ||
|
|
538
|
+
anonymous.includes(requiredAccess);
|
|
539
|
+
if (!hasAccess) {
|
|
526
540
|
return c.json({ error: "Access denied" }, 403);
|
|
527
541
|
}
|
|
528
542
|
|
|
@@ -595,28 +609,51 @@ const init = async () => {
|
|
|
595
609
|
});
|
|
596
610
|
});
|
|
597
611
|
|
|
598
|
-
// Serve static assets for runtime frontend plugins
|
|
599
|
-
//
|
|
600
|
-
// e.g. /assets/plugins/
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
612
|
+
// Serve static assets for runtime frontend plugins (built Module Federation
|
|
613
|
+
// remotes). The plugin is served under its package name, which may be SCOPED
|
|
614
|
+
// (e.g. /assets/plugins/@scope/name/mf-manifest.json) — so a single-segment
|
|
615
|
+
// `:pluginName` route can't capture it. Use a catch-all and split the package
|
|
616
|
+
// name (two segments when it starts with `@`) from the asset path.
|
|
617
|
+
// e.g. /assets/plugins/@acme/widget-frontend/mf-manifest.json ->
|
|
618
|
+
// runtime_plugins/node_modules/@acme/widget-frontend/dist/mf-manifest.json
|
|
619
|
+
const ASSET_CONTENT_TYPES: Record<string, string> = {
|
|
620
|
+
".js": "text/javascript",
|
|
621
|
+
".mjs": "text/javascript",
|
|
622
|
+
".css": "text/css",
|
|
623
|
+
".json": "application/json",
|
|
624
|
+
".map": "application/json",
|
|
625
|
+
".wasm": "application/wasm",
|
|
626
|
+
};
|
|
627
|
+
app.use("/assets/plugins/*", async (c, next) => {
|
|
628
|
+
const rest = c.req.path.split("/assets/plugins/")[1] ?? "";
|
|
629
|
+
const segments = rest.split("/").filter(Boolean);
|
|
630
|
+
const scoped = rest.startsWith("@");
|
|
631
|
+
const pluginName = scoped
|
|
632
|
+
? segments.slice(0, 2).join("/")
|
|
633
|
+
: segments[0];
|
|
634
|
+
const assetPath = scoped
|
|
635
|
+
? segments.slice(2).join("/")
|
|
636
|
+
: segments.slice(1).join("/");
|
|
637
|
+
// Reject path traversal and empty lookups.
|
|
638
|
+
if (!pluginName || !assetPath || assetPath.includes("..")) {
|
|
639
|
+
return next();
|
|
640
|
+
}
|
|
641
|
+
|
|
604
642
|
const results = await db
|
|
605
643
|
.select()
|
|
606
644
|
.from(plugins)
|
|
607
645
|
.where(eq(plugins.name, pluginName));
|
|
608
646
|
const plugin = results[0];
|
|
609
|
-
|
|
610
|
-
// Only serve assets for frontend plugins
|
|
611
647
|
if (!plugin || plugin.type !== "frontend") {
|
|
612
648
|
return next();
|
|
613
649
|
}
|
|
614
650
|
|
|
615
|
-
// We assume plugins are built into 'dist' folder
|
|
616
|
-
const assetPath = c.req.path.split(`/assets/plugins/${pluginName}/`)[1];
|
|
617
651
|
const filePath = path.join(plugin.path, "dist", assetPath);
|
|
618
|
-
|
|
619
652
|
if (fs.existsSync(filePath)) {
|
|
653
|
+
const type =
|
|
654
|
+
ASSET_CONTENT_TYPES[path.extname(filePath)] ??
|
|
655
|
+
"application/octet-stream";
|
|
656
|
+
c.header("Content-Type", type);
|
|
620
657
|
return c.body(fs.readFileSync(filePath));
|
|
621
658
|
}
|
|
622
659
|
return next();
|
|
@@ -763,6 +800,20 @@ const init = async () => {
|
|
|
763
800
|
}
|
|
764
801
|
}
|
|
765
802
|
|
|
803
|
+
// Register the plugin-manager's core access rules + metadata BEFORE
|
|
804
|
+
// loadPlugins. The auth-backend's full access-rule sync to the DB runs in
|
|
805
|
+
// `afterPluginsReady`, which fires *inside* loadPlugins and reads
|
|
806
|
+
// `pluginManager.getAllAccessRules()` at that moment. Registering these
|
|
807
|
+
// core rules after loadPlugins (where the router is wired up below) would
|
|
808
|
+
// miss that sync entirely, so the admin role would never be granted
|
|
809
|
+
// `pluginmanager.plugin.manage` and the install endpoints would 403 even
|
|
810
|
+
// for operators. The ids land prefixed as e.g. `pluginmanager.plugin.manage`.
|
|
811
|
+
pluginManager.registerCoreAccessRules(
|
|
812
|
+
pluginManagerMetadata.pluginId,
|
|
813
|
+
pluginManagerAccessRules,
|
|
814
|
+
);
|
|
815
|
+
pluginManager.registerCorePluginMetadata(pluginManagerMetadata);
|
|
816
|
+
|
|
766
817
|
await pluginManager.loadPlugins(app, manualPlugins, {
|
|
767
818
|
skipDiscovery: !!devPluginPath,
|
|
768
819
|
manualPluginPaths,
|
|
@@ -785,15 +836,9 @@ const init = async () => {
|
|
|
785
836
|
}
|
|
786
837
|
|
|
787
838
|
// 4.5. Register the plugin-manager admin router (core router, not a regular
|
|
788
|
-
// plugin).
|
|
789
|
-
//
|
|
790
|
-
//
|
|
791
|
-
// ids land as e.g. `pluginmanager.plugin.manage`.
|
|
792
|
-
pluginManager.registerCoreAccessRules(
|
|
793
|
-
pluginManagerMetadata.pluginId,
|
|
794
|
-
pluginManagerAccessRules,
|
|
795
|
-
);
|
|
796
|
-
pluginManager.registerCorePluginMetadata(pluginManagerMetadata);
|
|
839
|
+
// plugin). Its access rules + metadata were registered before loadPlugins
|
|
840
|
+
// above so the auth-backend full sync picks them up; here we only wire the
|
|
841
|
+
// router now that plugin services are available.
|
|
797
842
|
const pluginManagerRouter = createPluginManagerRouter({
|
|
798
843
|
db,
|
|
799
844
|
pluginManager,
|
package/src/plugin-manager.ts
CHANGED
|
@@ -24,6 +24,11 @@ import { createExtensionPointManager } from "./plugin-manager/extension-points";
|
|
|
24
24
|
import { loadPlugins as loadPluginsImpl } from "./plugin-manager/plugin-loader";
|
|
25
25
|
import { rootLogger } from "./logger";
|
|
26
26
|
import type { PluginEventRecorder } from "./services/plugin-event-recorder";
|
|
27
|
+
import { installBundleFromArtifacts } from "./services/plugin-installers/install-from-tarball";
|
|
28
|
+
import { getPluginSchemaName } from "@checkstack/drizzle-helper";
|
|
29
|
+
import { stripPublicSchemaFromMigrations } from "./utils/strip-public-schema";
|
|
30
|
+
import { runPluginMigrations } from "./utils/run-plugin-migrations";
|
|
31
|
+
import { createScopedDb } from "./utils/scoped-db";
|
|
27
32
|
|
|
28
33
|
export interface DeregisterOptions {
|
|
29
34
|
deleteSchema: boolean;
|
|
@@ -70,6 +75,14 @@ export class PluginManager {
|
|
|
70
75
|
// and writes to here when handling broadcast install hooks.
|
|
71
76
|
private runtimeDir: string | undefined;
|
|
72
77
|
|
|
78
|
+
// In-process dedupe for bundle installs: the install broadcast fans out one
|
|
79
|
+
// `pluginInstallationRequested` event PER package, so all siblings of a
|
|
80
|
+
// bundle race to install the same set of tarballs into `runtimeDir`. Keyed
|
|
81
|
+
// by bundleId, the first handler runs the single co-install and the rest
|
|
82
|
+
// await the same promise (cross-pod isolation is inherent — each pod has its
|
|
83
|
+
// own filesystem and its own map).
|
|
84
|
+
private bundleInstallLocks = new Map<string, Promise<void>>();
|
|
85
|
+
|
|
73
86
|
// Resolves once `/api/:pluginId/*` is registered on the root router and
|
|
74
87
|
// Phase 2 (per-plugin init) is starting. The HTTP server awaits this
|
|
75
88
|
// promise to know when it is safe to stop gating incoming requests.
|
|
@@ -507,7 +520,9 @@ export class PluginManager {
|
|
|
507
520
|
|
|
508
521
|
// Fast path: monorepo-local plugin, just load by package name / path.
|
|
509
522
|
if (!row.isUninstallable) {
|
|
510
|
-
|
|
523
|
+
if (row.type === "backend") {
|
|
524
|
+
await this.loadSinglePlugin(pluginId, row.path);
|
|
525
|
+
}
|
|
511
526
|
return;
|
|
512
527
|
}
|
|
513
528
|
|
|
@@ -519,41 +534,123 @@ export class PluginManager {
|
|
|
519
534
|
|
|
520
535
|
const pkgDir = path.join(this.runtimeDir, "node_modules", pluginId);
|
|
521
536
|
if (!fs.existsSync(path.join(pkgDir, "package.json"))) {
|
|
522
|
-
// Module not installed yet — pull from artifact store and install.
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
537
|
+
// Module not installed yet — pull from the artifact store and install.
|
|
538
|
+
if (row.bundleId) {
|
|
539
|
+
// Multi-package bundle: install ALL siblings in one `bun install` so a
|
|
540
|
+
// sibling that depends on another sibling resolves from the bundled
|
|
541
|
+
// tarballs instead of 404ing against a registry (the siblings are
|
|
542
|
+
// shipped inside the bundle and are usually unpublished). Deduped per
|
|
543
|
+
// bundleId so the parallel per-package broadcast handlers cooperate.
|
|
544
|
+
await this.ensureBundleInstalled(row.bundleId);
|
|
545
|
+
} else {
|
|
546
|
+
const artifactStore = await this.registry.get(
|
|
547
|
+
coreServices.pluginArtifactStore,
|
|
548
|
+
{ pluginId: "core" },
|
|
549
|
+
);
|
|
550
|
+
const artifact = await artifactStore.fetch({
|
|
551
|
+
pluginName: pluginId,
|
|
552
|
+
version: row.version,
|
|
553
|
+
});
|
|
554
|
+
if (!artifact) {
|
|
555
|
+
throw new Error(
|
|
556
|
+
`No tarball found in plugin_artifacts for ${pluginId}@${row.version}. ` +
|
|
557
|
+
`The plugin row exists but its artifact is missing — re-install from the original source.`,
|
|
558
|
+
);
|
|
559
|
+
}
|
|
560
|
+
const allowInstallScripts =
|
|
561
|
+
(row.metadata as { checkstack?: { allowInstallScripts?: boolean } })
|
|
562
|
+
?.checkstack?.allowInstallScripts === true;
|
|
563
|
+
|
|
564
|
+
const installerRegistry = await this.registry.get(
|
|
565
|
+
coreServices.pluginInstallerRegistry,
|
|
566
|
+
{ pluginId: "core" },
|
|
567
|
+
);
|
|
568
|
+
// Any installer's installFromArtifact does the same thing (delegates
|
|
569
|
+
// to the shared install-from-tarball helper) — pick npm.
|
|
570
|
+
await installerRegistry
|
|
571
|
+
.forSource("npm")
|
|
572
|
+
.installFromArtifact({
|
|
573
|
+
tarball: artifact.tarball,
|
|
574
|
+
pluginName: pluginId,
|
|
575
|
+
allowInstallScripts,
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// Only BACKEND packages register as backend plugins. A bundle's `common`
|
|
581
|
+
// (a plain library) and `frontend` (served to the browser via
|
|
582
|
+
// /api/plugins) siblings still need to be installed on disk — done above —
|
|
583
|
+
// but they export no `BackendPlugin`, so loading them here would throw.
|
|
584
|
+
// This mirrors the fresh-instance bootstrap, which installs every package
|
|
585
|
+
// but only imports `type = 'backend'` rows.
|
|
586
|
+
if (row.type === "backend") {
|
|
587
|
+
await this.loadSinglePlugin(pluginId, pkgDir);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/**
|
|
592
|
+
* Install every package of a bundle into `runtimeDir` in a single
|
|
593
|
+
* `bun install`, deduped per bundleId so the N parallel per-package install
|
|
594
|
+
* broadcasts run it exactly once. Resolves when the bundle is on disk.
|
|
595
|
+
*/
|
|
596
|
+
private async ensureBundleInstalled(bundleId: string): Promise<void> {
|
|
597
|
+
const inflight = this.bundleInstallLocks.get(bundleId);
|
|
598
|
+
if (inflight) return inflight;
|
|
599
|
+
const promise = this.installBundleNow(bundleId).finally(() => {
|
|
600
|
+
this.bundleInstallLocks.delete(bundleId);
|
|
601
|
+
});
|
|
602
|
+
this.bundleInstallLocks.set(bundleId, promise);
|
|
603
|
+
return promise;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
private async installBundleNow(bundleId: string): Promise<void> {
|
|
607
|
+
if (!this.runtimeDir) {
|
|
608
|
+
throw new Error(
|
|
609
|
+
`Runtime plugin dir not configured — call setRuntimeDir before hydrating runtime plugins.`,
|
|
526
610
|
);
|
|
611
|
+
}
|
|
612
|
+
const { plugins: pluginsTable } = await import("./schema");
|
|
613
|
+
const { eq } = await import("drizzle-orm");
|
|
614
|
+
|
|
615
|
+
const siblings = await db
|
|
616
|
+
.select()
|
|
617
|
+
.from(pluginsTable)
|
|
618
|
+
.where(eq(pluginsTable.bundleId, bundleId));
|
|
619
|
+
if (siblings.length === 0) {
|
|
620
|
+
throw new Error(`Bundle '${bundleId}' has no rows in 'plugins' table`);
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
const artifactStore = await this.registry.get(
|
|
624
|
+
coreServices.pluginArtifactStore,
|
|
625
|
+
{ pluginId: "core" },
|
|
626
|
+
);
|
|
627
|
+
const packages: Array<{ tarball: Uint8Array; pluginName: string }> = [];
|
|
628
|
+
for (const sib of siblings) {
|
|
527
629
|
const artifact = await artifactStore.fetch({
|
|
528
|
-
pluginName:
|
|
529
|
-
version:
|
|
630
|
+
pluginName: sib.name,
|
|
631
|
+
version: sib.version,
|
|
530
632
|
});
|
|
531
633
|
if (!artifact) {
|
|
532
634
|
throw new Error(
|
|
533
|
-
`No tarball found in plugin_artifacts for ${
|
|
534
|
-
`
|
|
635
|
+
`No tarball found in plugin_artifacts for ${sib.name}@${sib.version} ` +
|
|
636
|
+
`(bundle ${bundleId}). Re-install from the original source.`,
|
|
535
637
|
);
|
|
536
638
|
}
|
|
537
|
-
|
|
538
|
-
(row.metadata as { checkstack?: { allowInstallScripts?: boolean } })
|
|
539
|
-
?.checkstack?.allowInstallScripts === true;
|
|
540
|
-
|
|
541
|
-
const installerRegistry = await this.registry.get(
|
|
542
|
-
coreServices.pluginInstallerRegistry,
|
|
543
|
-
{ pluginId: "core" },
|
|
544
|
-
);
|
|
545
|
-
// Any installer's installFromArtifact does the same thing (delegates
|
|
546
|
-
// to the shared install-from-tarball helper) — pick npm.
|
|
547
|
-
await installerRegistry
|
|
548
|
-
.forSource("npm")
|
|
549
|
-
.installFromArtifact({
|
|
550
|
-
tarball: artifact.tarball,
|
|
551
|
-
pluginName: pluginId,
|
|
552
|
-
allowInstallScripts,
|
|
553
|
-
});
|
|
639
|
+
packages.push({ tarball: artifact.tarball, pluginName: sib.name });
|
|
554
640
|
}
|
|
555
641
|
|
|
556
|
-
|
|
642
|
+
// `--ignore-scripts` is all-or-nothing for one command; gate on the
|
|
643
|
+
// primary package the operator chose to trust.
|
|
644
|
+
const primary = siblings.find((s) => s.isPrimary) ?? siblings[0];
|
|
645
|
+
const allowInstallScripts =
|
|
646
|
+
(primary.metadata as { checkstack?: { allowInstallScripts?: boolean } })
|
|
647
|
+
?.checkstack?.allowInstallScripts === true;
|
|
648
|
+
|
|
649
|
+
await installBundleFromArtifacts({
|
|
650
|
+
packages,
|
|
651
|
+
allowInstallScripts,
|
|
652
|
+
runtimeDir: this.runtimeDir,
|
|
653
|
+
});
|
|
557
654
|
}
|
|
558
655
|
|
|
559
656
|
/**
|
|
@@ -617,6 +714,16 @@ export class PluginManager {
|
|
|
617
714
|
backendPlugin.metadata
|
|
618
715
|
);
|
|
619
716
|
}
|
|
717
|
+
// Inject the plugin-scoped database when a schema is declared —
|
|
718
|
+
// mirrors the full-system loader (plugin-loader Phase 2). Without
|
|
719
|
+
// this the `database` init arg is undefined and the plugin's
|
|
720
|
+
// service throws on its first query.
|
|
721
|
+
if (args.schema) {
|
|
722
|
+
resolvedDeps["database"] = createScopedDb(
|
|
723
|
+
db,
|
|
724
|
+
getPluginSchemaName(metaPluginId),
|
|
725
|
+
);
|
|
726
|
+
}
|
|
620
727
|
await args.init(resolvedDeps as never);
|
|
621
728
|
},
|
|
622
729
|
});
|
|
@@ -667,6 +774,20 @@ export class PluginManager {
|
|
|
667
774
|
},
|
|
668
775
|
});
|
|
669
776
|
|
|
777
|
+
// 2.5. Run this plugin's Drizzle migrations into its isolated schema
|
|
778
|
+
// before init, so its tables exist when the service issues its first
|
|
779
|
+
// query. Mirrors the full-system loader; a runtime-installed plugin
|
|
780
|
+
// ships its `drizzle/` folder inside the package.
|
|
781
|
+
const migrationsFolder = path.join(pluginPath, "drizzle");
|
|
782
|
+
if (fs.existsSync(migrationsFolder)) {
|
|
783
|
+
stripPublicSchemaFromMigrations(migrationsFolder);
|
|
784
|
+
await runPluginMigrations({
|
|
785
|
+
pool: adminPool,
|
|
786
|
+
migrationsFolder,
|
|
787
|
+
migrationsSchema: getPluginSchemaName(metaPluginId),
|
|
788
|
+
});
|
|
789
|
+
}
|
|
790
|
+
|
|
670
791
|
// 3. Initialize plugin (Phase 2)
|
|
671
792
|
for (const pending of pendingInits) {
|
|
672
793
|
await pending.init();
|
|
@@ -182,7 +182,10 @@ export class GithubPluginInstaller implements PluginInstaller {
|
|
|
182
182
|
`Bundle manifest in '${asset.name}' names primary '${bundle.manifest.primary}' but no matching sibling tarball was found.`,
|
|
183
183
|
);
|
|
184
184
|
}
|
|
185
|
-
|
|
185
|
+
// Persist/install the primary's INNER package tarball (a plain
|
|
186
|
+
// `package/package.json` tarball), not the outer bundle archive `buf`;
|
|
187
|
+
// the bundle is exposed via `bundle` for sibling resolution only.
|
|
188
|
+
return { tarball: primary.tarball, packageJson: primary.packageJson, bundle };
|
|
186
189
|
}
|
|
187
190
|
|
|
188
191
|
let packageJson;
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration test for `installBundleFromArtifacts`: a bundle whose primary
|
|
3
|
+
* depends on a sibling must install WITHOUT reaching a registry — the sibling
|
|
4
|
+
* ships inside the bundle and is (deliberately) not published anywhere.
|
|
5
|
+
*
|
|
6
|
+
* The packages use a `@cstest/*` scope that exists on no registry, and the
|
|
7
|
+
* registry env is pointed at an unroutable address, so a successful install
|
|
8
|
+
* can only mean bun resolved the intra-bundle dependency from the co-provided
|
|
9
|
+
* tarball. Before the bundle-aware co-install fix, each sibling was installed
|
|
10
|
+
* via its own `bun install <one.tgz>`, which 404s on the sibling dep.
|
|
11
|
+
*
|
|
12
|
+
* Shells out to a real `bun install`, so it's gated behind `CHECKSTACK_IT=1`
|
|
13
|
+
* (the same gate as the other integration tests) to keep the default unit
|
|
14
|
+
* lane fast and network-free.
|
|
15
|
+
*/
|
|
16
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import os from "node:os";
|
|
19
|
+
import fs from "node:fs/promises";
|
|
20
|
+
import { installBundleFromArtifacts } from "./install-from-tarball";
|
|
21
|
+
|
|
22
|
+
async function shellTar(args: {
|
|
23
|
+
cwd: string;
|
|
24
|
+
tarPath: string;
|
|
25
|
+
entries: string[];
|
|
26
|
+
}): Promise<void> {
|
|
27
|
+
const proc = Bun.spawn(["tar", "-czf", args.tarPath, ...args.entries], {
|
|
28
|
+
cwd: args.cwd,
|
|
29
|
+
stdout: "pipe",
|
|
30
|
+
stderr: "pipe",
|
|
31
|
+
});
|
|
32
|
+
const exitCode = await proc.exited;
|
|
33
|
+
if (exitCode !== 0) {
|
|
34
|
+
const stderr = await new Response(proc.stderr).text();
|
|
35
|
+
throw new Error(`tar exited with status ${exitCode}: ${stderr}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function buildTarball(pkgJson: Record<string, unknown>): Promise<Uint8Array> {
|
|
40
|
+
const stage = await fs.mkdtemp(path.join(os.tmpdir(), "bundle-install-test-"));
|
|
41
|
+
try {
|
|
42
|
+
const pkgDir = path.join(stage, "package");
|
|
43
|
+
await fs.mkdir(pkgDir, { recursive: true });
|
|
44
|
+
await fs.writeFile(
|
|
45
|
+
path.join(pkgDir, "package.json"),
|
|
46
|
+
JSON.stringify(pkgJson, undefined, 2),
|
|
47
|
+
);
|
|
48
|
+
await fs.writeFile(path.join(pkgDir, "index.js"), "module.exports = {};");
|
|
49
|
+
const tarPath = path.join(stage, "out.tgz");
|
|
50
|
+
await shellTar({ cwd: stage, tarPath, entries: ["package"] });
|
|
51
|
+
return new Uint8Array(await fs.readFile(tarPath));
|
|
52
|
+
} finally {
|
|
53
|
+
await fs.rm(stage, { recursive: true, force: true });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
describe.skipIf(!process.env.CHECKSTACK_IT)(
|
|
58
|
+
"installBundleFromArtifacts (real bun install)",
|
|
59
|
+
() => {
|
|
60
|
+
let runtimeDir: string;
|
|
61
|
+
let savedRegistry: string | undefined;
|
|
62
|
+
|
|
63
|
+
beforeAll(async () => {
|
|
64
|
+
runtimeDir = await fs.mkdtemp(path.join(os.tmpdir(), "bundle-runtime-"));
|
|
65
|
+
// Force any registry round-trip to fail fast, so a passing install
|
|
66
|
+
// proves the sibling dep resolved from the co-provided tarball.
|
|
67
|
+
savedRegistry = process.env.BUN_CONFIG_REGISTRY;
|
|
68
|
+
process.env.BUN_CONFIG_REGISTRY = "http://127.0.0.1:1/";
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
afterAll(async () => {
|
|
72
|
+
if (savedRegistry === undefined) delete process.env.BUN_CONFIG_REGISTRY;
|
|
73
|
+
else process.env.BUN_CONFIG_REGISTRY = savedRegistry;
|
|
74
|
+
await fs.rm(runtimeDir, { recursive: true, force: true });
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("resolves an intra-bundle sibling dependency without a registry", async () => {
|
|
78
|
+
const common = await buildTarball({
|
|
79
|
+
name: "@cstest/widget-common",
|
|
80
|
+
version: "0.0.1",
|
|
81
|
+
description: "test common",
|
|
82
|
+
author: "test",
|
|
83
|
+
license: "Elastic-2.0",
|
|
84
|
+
checkstack: { type: "common", pluginId: "widget" },
|
|
85
|
+
});
|
|
86
|
+
// The primary DEPENDS on the (unpublished) sibling — the exact shape
|
|
87
|
+
// that 404s when installed one-tarball-at-a-time.
|
|
88
|
+
const backend = await buildTarball({
|
|
89
|
+
name: "@cstest/widget-backend",
|
|
90
|
+
version: "0.0.1",
|
|
91
|
+
description: "test backend",
|
|
92
|
+
author: "test",
|
|
93
|
+
license: "Elastic-2.0",
|
|
94
|
+
dependencies: { "@cstest/widget-common": "^0.0.1" },
|
|
95
|
+
checkstack: { type: "backend", pluginId: "widget" },
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
const installed = await installBundleFromArtifacts({
|
|
99
|
+
packages: [
|
|
100
|
+
{ tarball: common, pluginName: "@cstest/widget-common" },
|
|
101
|
+
{ tarball: backend, pluginName: "@cstest/widget-backend" },
|
|
102
|
+
],
|
|
103
|
+
runtimeDir,
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
expect(installed.map((i) => i.name).sort()).toEqual([
|
|
107
|
+
"@cstest/widget-backend",
|
|
108
|
+
"@cstest/widget-common",
|
|
109
|
+
]);
|
|
110
|
+
// Both packages physically landed under the runtime dir's node_modules.
|
|
111
|
+
for (const name of ["@cstest/widget-common", "@cstest/widget-backend"]) {
|
|
112
|
+
const pkgJsonPath = path.join(
|
|
113
|
+
runtimeDir,
|
|
114
|
+
"node_modules",
|
|
115
|
+
name,
|
|
116
|
+
"package.json",
|
|
117
|
+
);
|
|
118
|
+
expect(await fs.exists(pkgJsonPath)).toBe(true);
|
|
119
|
+
}
|
|
120
|
+
}, 60_000);
|
|
121
|
+
},
|
|
122
|
+
);
|
|
@@ -67,3 +67,112 @@ export async function installFromArtifact({
|
|
|
67
67
|
await fs.rm(tmpDir, { recursive: true, force: true });
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Install every package of a multi-package bundle so that intra-bundle
|
|
73
|
+
* dependencies (a sibling depending on `@scope/other-sibling`) resolve from
|
|
74
|
+
* the co-shipped tarballs instead of being fetched from a registry — those
|
|
75
|
+
* siblings live inside the bundle and are typically not published anywhere.
|
|
76
|
+
*
|
|
77
|
+
* `bun install a.tgz b.tgz` does NOT make one CLI-provided tarball satisfy
|
|
78
|
+
* another's named dependency range; bun still resolves `@scope/sibling@^x`
|
|
79
|
+
* against the registry and 404s. The reliable mechanism is a throwaway
|
|
80
|
+
* manifest that lists every sibling as a `file:` dependency AND pins every
|
|
81
|
+
* sibling name in `overrides` to that same `file:` tarball, so any sibling's
|
|
82
|
+
* range is forced to the local tarball. External deps (`@checkstack/*` and
|
|
83
|
+
* their transitive deps) still resolve from the registry as normal.
|
|
84
|
+
*
|
|
85
|
+
* We install into a temp root (a manifest-driven install would PRUNE the
|
|
86
|
+
* shared runtime dir down to just this bundle), then merge the resulting
|
|
87
|
+
* `node_modules` into `runtimeDir/node_modules` additively — bun's default
|
|
88
|
+
* layout is hoisted real directories (no symlinks), so a recursive copy is a
|
|
89
|
+
* safe per-entry merge that leaves previously-installed plugins intact.
|
|
90
|
+
*
|
|
91
|
+
* `allowInstallScripts` is bundle-wide: `--ignore-scripts` is all-or-nothing
|
|
92
|
+
* for one command, so callers pass the primary package's opt-in (the package
|
|
93
|
+
* the operator chose to trust).
|
|
94
|
+
*/
|
|
95
|
+
export async function installBundleFromArtifacts({
|
|
96
|
+
packages,
|
|
97
|
+
allowInstallScripts,
|
|
98
|
+
runtimeDir,
|
|
99
|
+
}: {
|
|
100
|
+
packages: Array<{ tarball: Uint8Array; pluginName: string }>;
|
|
101
|
+
allowInstallScripts?: boolean;
|
|
102
|
+
runtimeDir: string;
|
|
103
|
+
}): Promise<InstalledArtifact[]> {
|
|
104
|
+
if (packages.length === 0) {
|
|
105
|
+
throw new Error("installBundleFromArtifacts: no packages to install");
|
|
106
|
+
}
|
|
107
|
+
await fs.mkdir(runtimeDir, { recursive: true });
|
|
108
|
+
|
|
109
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "checkstack-bundle-"));
|
|
110
|
+
try {
|
|
111
|
+
const stageDir = path.join(tmpDir, "stage");
|
|
112
|
+
const rootDir = path.join(tmpDir, "root");
|
|
113
|
+
await fs.mkdir(stageDir, { recursive: true });
|
|
114
|
+
await fs.mkdir(rootDir, { recursive: true });
|
|
115
|
+
|
|
116
|
+
// Stage each tarball and validate its name up front.
|
|
117
|
+
const dependencies: Record<string, string> = {};
|
|
118
|
+
const overrides: Record<string, string> = {};
|
|
119
|
+
for (const [i, pkg] of packages.entries()) {
|
|
120
|
+
const expected = await extractPackageJson(pkg.tarball);
|
|
121
|
+
if (expected.name !== pkg.pluginName) {
|
|
122
|
+
throw new Error(
|
|
123
|
+
`Tarball name mismatch: expected '${pkg.pluginName}', got '${expected.name}'`,
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
const tarPath = path.join(stageDir, `pkg-${i}.tgz`);
|
|
127
|
+
await fs.writeFile(tarPath, pkg.tarball);
|
|
128
|
+
const fileSpecifier = `file:${tarPath}`;
|
|
129
|
+
dependencies[expected.name] = fileSpecifier;
|
|
130
|
+
overrides[expected.name] = fileSpecifier;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
await fs.writeFile(
|
|
134
|
+
path.join(rootDir, "package.json"),
|
|
135
|
+
JSON.stringify(
|
|
136
|
+
{ name: "_checkstack-bundle-install", version: "0.0.0", dependencies, overrides },
|
|
137
|
+
undefined,
|
|
138
|
+
2,
|
|
139
|
+
),
|
|
140
|
+
);
|
|
141
|
+
|
|
142
|
+
const ignoreScripts = allowInstallScripts ? "" : "--ignore-scripts";
|
|
143
|
+
const cmd =
|
|
144
|
+
`bun install --cwd "${rootDir}" --no-save ${ignoreScripts}`.trim();
|
|
145
|
+
rootLogger.info(`📦 Running (bundle): ${cmd}`);
|
|
146
|
+
const { stderr } = await execAsync(cmd);
|
|
147
|
+
if (stderr) rootLogger.debug(stderr);
|
|
148
|
+
|
|
149
|
+
// Merge the resolved tree into the shared runtime dir, per top-level
|
|
150
|
+
// entry, so existing plugins' node_modules entries are preserved.
|
|
151
|
+
const srcModules = path.join(rootDir, "node_modules");
|
|
152
|
+
const destModules = path.join(runtimeDir, "node_modules");
|
|
153
|
+
await fs.mkdir(destModules, { recursive: true });
|
|
154
|
+
for (const entry of await fs.readdir(srcModules)) {
|
|
155
|
+
await fs.cp(
|
|
156
|
+
path.join(srcModules, entry),
|
|
157
|
+
path.join(destModules, entry),
|
|
158
|
+
{ recursive: true, force: true },
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const installed: InstalledArtifact[] = [];
|
|
163
|
+
for (const pkg of packages) {
|
|
164
|
+
const pkgDir = path.join(destModules, pkg.pluginName);
|
|
165
|
+
const installedPkgJson = JSON.parse(
|
|
166
|
+
await fs.readFile(path.join(pkgDir, "package.json"), "utf8"),
|
|
167
|
+
);
|
|
168
|
+
installed.push({
|
|
169
|
+
name: installedPkgJson.name,
|
|
170
|
+
path: pkgDir,
|
|
171
|
+
version: installedPkgJson.version,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
return installed;
|
|
175
|
+
} finally {
|
|
176
|
+
await fs.rm(tmpDir, { recursive: true, force: true });
|
|
177
|
+
}
|
|
178
|
+
}
|
|
@@ -41,7 +41,6 @@ export class TarballPluginInstaller implements PluginInstaller {
|
|
|
41
41
|
|
|
42
42
|
const bundle = await tryExtractBundle(stored.tarball);
|
|
43
43
|
if (bundle) {
|
|
44
|
-
// For bundles, the outer tarball's "package.json" is the primary's.
|
|
45
44
|
const primary = bundle.siblings.find(
|
|
46
45
|
(s) => s.packageJson.name === bundle.manifest.primary,
|
|
47
46
|
);
|
|
@@ -51,8 +50,13 @@ export class TarballPluginInstaller implements PluginInstaller {
|
|
|
51
50
|
`Bundle manifest names primary '${bundle.manifest.primary}' but no matching sibling tarball was found in the uploaded archive.`,
|
|
52
51
|
);
|
|
53
52
|
}
|
|
53
|
+
// `tarball` is the primary's INNER package tarball (not the outer bundle
|
|
54
|
+
// archive): it's what gets persisted as the primary's artifact and what
|
|
55
|
+
// the runtime installs, so it must be a plain `package/package.json`
|
|
56
|
+
// tarball. The outer bundle is exposed via `bundle` for sibling
|
|
57
|
+
// resolution only.
|
|
54
58
|
return {
|
|
55
|
-
tarball:
|
|
59
|
+
tarball: primary.tarball,
|
|
56
60
|
packageJson: primary.packageJson,
|
|
57
61
|
bundle,
|
|
58
62
|
};
|