@checkstack/satellite 0.2.11 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +127 -0
- package/package.json +6 -5
- package/src/index.ts +64 -1
- package/src/run-context.test.ts +53 -0
- package/src/run-context.ts +28 -0
- package/src/satellite-client.ts +172 -0
- package/src/satellite-script-packages.test.ts +105 -0
- package/src/satellite-script-packages.ts +130 -0
- package/tsconfig.json +3 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,132 @@
|
|
|
1
1
|
# @checkstack/satellite
|
|
2
2
|
|
|
3
|
+
## 0.4.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 270ef29: Satellite-side script-package reconciliation over the WS channel.
|
|
8
|
+
|
|
9
|
+
- `satellite-common`: WS request/reply messages for pulling the manifest +
|
|
10
|
+
blobs from core (`request_script_package_manifest` /
|
|
11
|
+
`request_script_package_blob` -> `script_package_manifest` /
|
|
12
|
+
`script_package_blob`).
|
|
13
|
+
- `satellite-backend`: the WS handler answers those requests from the
|
|
14
|
+
script-packages store (satellites pull from core, never the registry).
|
|
15
|
+
- `@checkstack/satellite`: the client gains request/reply plumbing + a
|
|
16
|
+
`SatelliteScriptPackages` manager that reuses the Phase 2 reconciler
|
|
17
|
+
(`reconcileToHash` + `createReconcileFsDeps`) over the WS transport. It
|
|
18
|
+
reconciles on a `refresh_script_packages` push and on the
|
|
19
|
+
assignment-carried hash (startup / reconnect backstop), pulls only missing
|
|
20
|
+
blobs (delta), materializes via `bun install --offline`, atomically flips
|
|
21
|
+
`current`, reports sync state back, and degrades cleanly (error state, no
|
|
22
|
+
stale tree, no registry access) when a blob can't be fetched. Reconciles
|
|
23
|
+
are serialized + coalesced + idempotent.
|
|
24
|
+
|
|
25
|
+
- 270ef29: Secrets platform Phase 3: just-in-time secret delivery to satellites + source-side masking, and central-execution injection for healthcheck collectors.
|
|
26
|
+
|
|
27
|
+
- New satellite WS messages `request_run_secrets` / `run_secrets`: just
|
|
28
|
+
before a satellite runs a collector that declares a `secretEnv`, it asks
|
|
29
|
+
core for that collector's resolved env; core resolves ONLY the secrets the
|
|
30
|
+
collector's OWN persisted assignment declares (least-privilege — the
|
|
31
|
+
satellite cannot choose) and replies with the env map (or a clear error).
|
|
32
|
+
The satellite injects it memory-only for the run and drops it on
|
|
33
|
+
completion. Secrets never ride the persisted assignment and never touch
|
|
34
|
+
disk.
|
|
35
|
+
- Source-side masking: the satellite runs `maskSecrets` over the collector's
|
|
36
|
+
stdout/stderr/result/error using the run's delivered values BEFORE the
|
|
37
|
+
result leaves the satellite (defense in depth).
|
|
38
|
+
- `CollectorStrategy.execute` gains an optional `secretEnv`. The
|
|
39
|
+
inline-script and shell collectors inject it into the runner
|
|
40
|
+
(`process.env` / `$VAR`) and mask the values out of their output.
|
|
41
|
+
- Healthcheck collectors running centrally (the queue executor) also resolve
|
|
42
|
+
- inject `secretEnv` via `secretResolverRef`, closing the gap where a
|
|
43
|
+
centrally-run secretEnv collector got no secrets. A missing required
|
|
44
|
+
secret fails the run clearly in all paths.
|
|
45
|
+
|
|
46
|
+
### Patch Changes
|
|
47
|
+
|
|
48
|
+
- Updated dependencies [270ef29]
|
|
49
|
+
- Updated dependencies [270ef29]
|
|
50
|
+
- Updated dependencies [270ef29]
|
|
51
|
+
- Updated dependencies [b995afb]
|
|
52
|
+
- Updated dependencies [270ef29]
|
|
53
|
+
- Updated dependencies [270ef29]
|
|
54
|
+
- Updated dependencies [270ef29]
|
|
55
|
+
- Updated dependencies [270ef29]
|
|
56
|
+
- Updated dependencies [270ef29]
|
|
57
|
+
- Updated dependencies [270ef29]
|
|
58
|
+
- Updated dependencies [b995afb]
|
|
59
|
+
- Updated dependencies [b995afb]
|
|
60
|
+
- Updated dependencies [270ef29]
|
|
61
|
+
- Updated dependencies [270ef29]
|
|
62
|
+
- Updated dependencies [270ef29]
|
|
63
|
+
- Updated dependencies [b995afb]
|
|
64
|
+
- Updated dependencies [270ef29]
|
|
65
|
+
- Updated dependencies [b995afb]
|
|
66
|
+
- Updated dependencies [270ef29]
|
|
67
|
+
- Updated dependencies [270ef29]
|
|
68
|
+
- Updated dependencies [270ef29]
|
|
69
|
+
- Updated dependencies [270ef29]
|
|
70
|
+
- Updated dependencies [270ef29]
|
|
71
|
+
- @checkstack/backend-api@0.19.0
|
|
72
|
+
- @checkstack/script-packages-backend@0.2.0
|
|
73
|
+
- @checkstack/satellite-common@0.7.0
|
|
74
|
+
|
|
75
|
+
## 0.3.0
|
|
76
|
+
|
|
77
|
+
### Minor Changes
|
|
78
|
+
|
|
79
|
+
- 35bc682: feat(healthcheck): expose check + system run-context to script collectors
|
|
80
|
+
|
|
81
|
+
Script health checks can now read which check and system a run is for.
|
|
82
|
+
Previously shell scripts got only a curated env whitelist and inline
|
|
83
|
+
scripts only `context.config`, so a script had no built-in way to know
|
|
84
|
+
its own check name or the system it was checking.
|
|
85
|
+
|
|
86
|
+
- `@checkstack/backend-api`: new `CollectorRunContext` type
|
|
87
|
+
(`{ check: { id, name, intervalSeconds }, system: { id, name } }`) and
|
|
88
|
+
an optional `runContext` param on `CollectorStrategy.execute`. Optional,
|
|
89
|
+
so existing collector implementations are unaffected.
|
|
90
|
+
- Shell-script collector: injects reserved `CHECKSTACK_CHECK_ID`,
|
|
91
|
+
`CHECKSTACK_CHECK_NAME`, `CHECKSTACK_CHECK_INTERVAL_SECONDS`,
|
|
92
|
+
`CHECKSTACK_SYSTEM_ID`, `CHECKSTACK_SYSTEM_NAME` env vars (user-supplied
|
|
93
|
+
`env` still wins on collision).
|
|
94
|
+
- Inline-script collector: exposes `context.check` and `context.system`
|
|
95
|
+
alongside `context.config`; the inline-script editor now types them for
|
|
96
|
+
autocomplete.
|
|
97
|
+
- Shell editors (health-check collectors and automation shell actions) now
|
|
98
|
+
also suggest the user's own `env` (JSON) keys as `$NAME` completions, via
|
|
99
|
+
the new exported `customShellEnvVars` helper. Keys that aren't valid shell
|
|
100
|
+
identifiers are omitted.
|
|
101
|
+
- Fix: the Typefox `CodeEditor` captured a stale `onChange` at editor start,
|
|
102
|
+
so editing one `DynamicForm` field reverted sibling fields changed since
|
|
103
|
+
mount (e.g. typing in a shell `script` field wiped an unsaved `env` value,
|
|
104
|
+
or deleted a sibling automation action added after mount). The change
|
|
105
|
+
handler now routes through a ref to the current `onChange`.
|
|
106
|
+
- Fix: focusing a JSON editor threw "LanguageStatusService.addStatus is not
|
|
107
|
+
supported" because the standalone service set omitted `ILanguageStatusService`.
|
|
108
|
+
That one service is now registered via `serviceOverrides`.
|
|
109
|
+
- Fix: the automation trigger card nested a `<Badge>` (a `<div>`) inside a
|
|
110
|
+
`<p>`, producing a `validateDOMNesting` warning. Switched the wrapper to a
|
|
111
|
+
`<div>`.
|
|
112
|
+
- Local runs (`queue-executor`) and satellite runs both populate the
|
|
113
|
+
context. `SatelliteAssignment` (and the `getAssignmentsForSatellite`
|
|
114
|
+
RPC output) gained optional `configName` / `systemName` so the metadata
|
|
115
|
+
reaches satellite-side execution; `HealthCheckService` resolves the
|
|
116
|
+
system name via the catalog client.
|
|
117
|
+
|
|
118
|
+
BREAKING CHANGE: `createHealthCheckRouter` now requires a `catalogClient`
|
|
119
|
+
option (used to resolve system names for satellite assignments). Update
|
|
120
|
+
call sites to pass the catalog RPC client.
|
|
121
|
+
|
|
122
|
+
### Patch Changes
|
|
123
|
+
|
|
124
|
+
- Updated dependencies [6d52276]
|
|
125
|
+
- Updated dependencies [35bc682]
|
|
126
|
+
- @checkstack/common@0.12.0
|
|
127
|
+
- @checkstack/backend-api@0.18.0
|
|
128
|
+
- @checkstack/satellite-common@0.6.0
|
|
129
|
+
|
|
3
130
|
## 0.2.11
|
|
4
131
|
|
|
5
132
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@checkstack/satellite",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -11,13 +11,14 @@
|
|
|
11
11
|
"lint:code": "eslint . --max-warnings 0"
|
|
12
12
|
},
|
|
13
13
|
"dependencies": {
|
|
14
|
-
"@checkstack/satellite-common": "0.
|
|
15
|
-
"@checkstack/backend-api": "0.
|
|
16
|
-
"@checkstack/
|
|
14
|
+
"@checkstack/satellite-common": "0.6.0",
|
|
15
|
+
"@checkstack/backend-api": "0.18.0",
|
|
16
|
+
"@checkstack/script-packages-backend": "0.1.0",
|
|
17
|
+
"@checkstack/common": "0.12.0"
|
|
17
18
|
},
|
|
18
19
|
"devDependencies": {
|
|
19
20
|
"@checkstack/tsconfig": "0.0.7",
|
|
20
|
-
"@checkstack/scripts": "0.3.
|
|
21
|
+
"@checkstack/scripts": "0.3.4",
|
|
21
22
|
"@types/bun": "^1.0.0",
|
|
22
23
|
"typescript": "^5.0.0"
|
|
23
24
|
}
|
package/src/index.ts
CHANGED
|
@@ -5,10 +5,14 @@ import type {
|
|
|
5
5
|
import type {
|
|
6
6
|
ConnectedClient,
|
|
7
7
|
TransportClient,
|
|
8
|
+
CollectorRunContext,
|
|
8
9
|
} from "@checkstack/backend-api";
|
|
10
|
+
import { resolveScriptPackagesDir } from "@checkstack/script-packages-backend";
|
|
9
11
|
import { SatelliteClient } from "./satellite-client";
|
|
10
12
|
import { Scheduler } from "./scheduler";
|
|
11
13
|
import { loadStrategies } from "./strategy-loader";
|
|
14
|
+
import { buildRunContext } from "./run-context";
|
|
15
|
+
import { SatelliteScriptPackages } from "./satellite-script-packages";
|
|
12
16
|
|
|
13
17
|
// =============================================================================
|
|
14
18
|
// Environment validation — fail fast if required vars are missing
|
|
@@ -68,8 +72,29 @@ const { healthCheckRegistry, collectorRegistry } = await loadStrategies({
|
|
|
68
72
|
// 4. Close client and report result
|
|
69
73
|
// =============================================================================
|
|
70
74
|
|
|
75
|
+
/** Whether a collector config declares a non-empty secretEnv mapping. */
|
|
76
|
+
function declaresSecretEnv(config: Record<string, unknown>): boolean {
|
|
77
|
+
const se = config.secretEnv;
|
|
78
|
+
return (
|
|
79
|
+
typeof se === "object" &&
|
|
80
|
+
se !== null &&
|
|
81
|
+
Object.keys(se as Record<string, unknown>).length > 0
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
71
85
|
async function executeAssignment(
|
|
72
86
|
assignment: SatelliteAssignment,
|
|
87
|
+
deps: {
|
|
88
|
+
/**
|
|
89
|
+
* Request a collector run's resolved secret env from core (JIT). Throws
|
|
90
|
+
* on delivery/resolution failure so the collector fails clearly.
|
|
91
|
+
*/
|
|
92
|
+
requestRunSecrets: (input: {
|
|
93
|
+
configId: string;
|
|
94
|
+
collectorId: string;
|
|
95
|
+
runId: string;
|
|
96
|
+
}) => Promise<Record<string, string>>;
|
|
97
|
+
},
|
|
73
98
|
): Promise<ResultMessage> {
|
|
74
99
|
const strategy = healthCheckRegistry.getStrategy(assignment.strategyId);
|
|
75
100
|
if (!strategy) {
|
|
@@ -92,6 +117,11 @@ async function executeAssignment(
|
|
|
92
117
|
};
|
|
93
118
|
}
|
|
94
119
|
|
|
120
|
+
// Curated, read-only run-context metadata exposed to collectors.
|
|
121
|
+
// Mirrors the core queue-executor; falls back to IDs when the optional
|
|
122
|
+
// name fields are absent (version-skew safety).
|
|
123
|
+
const runContext: CollectorRunContext = buildRunContext({ assignment });
|
|
124
|
+
|
|
95
125
|
const start = performance.now();
|
|
96
126
|
let connectedClient:
|
|
97
127
|
| ConnectedClient<TransportClient<never, unknown>>
|
|
@@ -121,10 +151,26 @@ async function executeAssignment(
|
|
|
121
151
|
}
|
|
122
152
|
|
|
123
153
|
try {
|
|
154
|
+
// JIT secret delivery: if this collector declares a secretEnv,
|
|
155
|
+
// fetch the resolved values from core over the WS channel just
|
|
156
|
+
// before running. Held in memory only for this run; never written
|
|
157
|
+
// to disk and never part of the persisted assignment. A delivery
|
|
158
|
+
// / resolution failure throws and fails the collector clearly.
|
|
159
|
+
let secretEnv: Record<string, string> | undefined;
|
|
160
|
+
if (declaresSecretEnv(collectorEntry.config)) {
|
|
161
|
+
secretEnv = await deps.requestRunSecrets({
|
|
162
|
+
configId: assignment.configId,
|
|
163
|
+
collectorId: collectorEntry.id,
|
|
164
|
+
runId: crypto.randomUUID(),
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
|
|
124
168
|
const collectorResult = await registered.collector.execute({
|
|
125
169
|
config: collectorEntry.config,
|
|
126
170
|
client: connectedClient!.client,
|
|
127
171
|
pluginId: assignment.strategyId,
|
|
172
|
+
runContext,
|
|
173
|
+
...(secretEnv ? { secretEnv } : {}),
|
|
128
174
|
});
|
|
129
175
|
|
|
130
176
|
return {
|
|
@@ -243,16 +289,33 @@ const client = new SatelliteClient({
|
|
|
243
289
|
onAssignments: (assignments: SatelliteAssignment[]) => {
|
|
244
290
|
scheduler.updateAssignments(assignments);
|
|
245
291
|
},
|
|
292
|
+
onScriptPackagesLockfileHash: (lockfileHash) => {
|
|
293
|
+
void scriptPackages.reconcile(lockfileHash);
|
|
294
|
+
},
|
|
246
295
|
onDisconnect: () => {
|
|
247
296
|
scheduler.stop();
|
|
248
297
|
},
|
|
249
298
|
});
|
|
250
299
|
|
|
300
|
+
// Script-package reconciler: pulls blobs from CORE over the WS channel
|
|
301
|
+
// (never the registry), materializes node_modules, atomically flips
|
|
302
|
+
// `<store>/current`. Triggered on connect (assignment-carried backstop) and
|
|
303
|
+
// on `refresh_script_packages` pushes.
|
|
304
|
+
const scriptPackages = new SatelliteScriptPackages({
|
|
305
|
+
storeRoot: resolveScriptPackagesDir(),
|
|
306
|
+
requestManifest: (hash) => client.requestManifest(hash),
|
|
307
|
+
requestBlob: (integrity) => client.requestBlob(integrity),
|
|
308
|
+
reportState: (state) => client.reportScriptPackageSyncState(state),
|
|
309
|
+
logger,
|
|
310
|
+
});
|
|
311
|
+
|
|
251
312
|
const scheduler = new Scheduler({
|
|
252
313
|
logger,
|
|
253
314
|
onExecute: async (assignment: SatelliteAssignment) => {
|
|
254
315
|
try {
|
|
255
|
-
const result = await executeAssignment(assignment
|
|
316
|
+
const result = await executeAssignment(assignment, {
|
|
317
|
+
requestRunSecrets: (input) => client.requestRunSecrets(input),
|
|
318
|
+
});
|
|
256
319
|
client.sendResult(result);
|
|
257
320
|
} catch (error) {
|
|
258
321
|
logger.error(
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { describe, test, expect } from "bun:test";
|
|
2
|
+
import { buildRunContext } from "./run-context";
|
|
3
|
+
import type { SatelliteAssignment } from "@checkstack/satellite-common";
|
|
4
|
+
|
|
5
|
+
function makeAssignment(
|
|
6
|
+
overrides?: Partial<SatelliteAssignment>,
|
|
7
|
+
): SatelliteAssignment {
|
|
8
|
+
return {
|
|
9
|
+
configId: "config-1",
|
|
10
|
+
systemId: "system-1",
|
|
11
|
+
strategyId: "http",
|
|
12
|
+
config: {},
|
|
13
|
+
intervalSeconds: 60,
|
|
14
|
+
...overrides,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
describe("buildRunContext", () => {
|
|
19
|
+
test("uses configName and systemName when present", () => {
|
|
20
|
+
const assignment = makeAssignment({
|
|
21
|
+
configName: "API health",
|
|
22
|
+
systemName: "Production API",
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
const runContext = buildRunContext({ assignment });
|
|
26
|
+
|
|
27
|
+
expect(runContext).toEqual({
|
|
28
|
+
check: { id: "config-1", name: "API health", intervalSeconds: 60 },
|
|
29
|
+
system: { id: "system-1", name: "Production API" },
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("falls back to ids when name fields are absent", () => {
|
|
34
|
+
const assignment = makeAssignment();
|
|
35
|
+
|
|
36
|
+
const runContext = buildRunContext({ assignment });
|
|
37
|
+
|
|
38
|
+
expect(runContext.check.name).toBe("config-1");
|
|
39
|
+
expect(runContext.check.id).toBe("config-1");
|
|
40
|
+
expect(runContext.check.intervalSeconds).toBe(60);
|
|
41
|
+
expect(runContext.system.name).toBe("system-1");
|
|
42
|
+
expect(runContext.system.id).toBe("system-1");
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("falls back per-field when only one name is present", () => {
|
|
46
|
+
const assignment = makeAssignment({ configName: "API health" });
|
|
47
|
+
|
|
48
|
+
const runContext = buildRunContext({ assignment });
|
|
49
|
+
|
|
50
|
+
expect(runContext.check.name).toBe("API health");
|
|
51
|
+
expect(runContext.system.name).toBe("system-1");
|
|
52
|
+
});
|
|
53
|
+
});
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { SatelliteAssignment } from "@checkstack/satellite-common";
|
|
2
|
+
import type { CollectorRunContext } from "@checkstack/backend-api";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Build the curated, read-only run-context metadata exposed to collectors
|
|
6
|
+
* from a satellite assignment.
|
|
7
|
+
*
|
|
8
|
+
* Mirrors the core queue-executor's run-context. The `configName` and
|
|
9
|
+
* `systemName` assignment fields are optional for version-skew safety, so
|
|
10
|
+
* they fall back to the corresponding IDs when absent.
|
|
11
|
+
*/
|
|
12
|
+
export function buildRunContext({
|
|
13
|
+
assignment,
|
|
14
|
+
}: {
|
|
15
|
+
assignment: SatelliteAssignment;
|
|
16
|
+
}): CollectorRunContext {
|
|
17
|
+
return {
|
|
18
|
+
check: {
|
|
19
|
+
id: assignment.configId,
|
|
20
|
+
name: assignment.configName ?? assignment.configId,
|
|
21
|
+
intervalSeconds: assignment.intervalSeconds,
|
|
22
|
+
},
|
|
23
|
+
system: {
|
|
24
|
+
id: assignment.systemId,
|
|
25
|
+
name: assignment.systemName ?? assignment.systemId,
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
}
|
package/src/satellite-client.ts
CHANGED
|
@@ -8,9 +8,16 @@ import type {
|
|
|
8
8
|
CoreToSatelliteMessage,
|
|
9
9
|
SatelliteToCoreMessage,
|
|
10
10
|
ResultMessage,
|
|
11
|
+
ScriptPackageSyncStateMessage,
|
|
11
12
|
} from "@checkstack/satellite-common";
|
|
12
13
|
import { ResultBuffer } from "./result-buffer";
|
|
13
14
|
|
|
15
|
+
interface ManifestEntryWire {
|
|
16
|
+
name: string;
|
|
17
|
+
version: string;
|
|
18
|
+
integrity: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
14
21
|
interface SatelliteClientConfig {
|
|
15
22
|
coreUrl: string;
|
|
16
23
|
clientId: string;
|
|
@@ -18,6 +25,12 @@ interface SatelliteClientConfig {
|
|
|
18
25
|
version: string;
|
|
19
26
|
onAssignments: (assignments: SatelliteAssignment[]) => void;
|
|
20
27
|
onDisconnect?: () => void;
|
|
28
|
+
/**
|
|
29
|
+
* Called with the desired script-package lockfile hash whenever the core
|
|
30
|
+
* signals one - on connect (assignment-carried backstop) and on a
|
|
31
|
+
* `refresh_script_packages` push. The satellite reconciles to it.
|
|
32
|
+
*/
|
|
33
|
+
onScriptPackagesLockfileHash?: (lockfileHash: string | null) => void;
|
|
21
34
|
logger?: {
|
|
22
35
|
info: (msg: string) => void;
|
|
23
36
|
warn: (msg: string) => void;
|
|
@@ -38,11 +51,117 @@ export class SatelliteClient {
|
|
|
38
51
|
private connected = false;
|
|
39
52
|
private readonly resultBuffer = new ResultBuffer();
|
|
40
53
|
private readonly config: SatelliteClientConfig;
|
|
54
|
+
// Pending script-package request promises, resolved when the matching
|
|
55
|
+
// core reply arrives. Keyed by lockfileHash (manifest) / integrity (blob).
|
|
56
|
+
private readonly pendingManifest = new Map<
|
|
57
|
+
string,
|
|
58
|
+
(entries: ManifestEntryWire[]) => void
|
|
59
|
+
>();
|
|
60
|
+
private readonly pendingBlob = new Map<
|
|
61
|
+
string,
|
|
62
|
+
(data: string | null) => void
|
|
63
|
+
>();
|
|
64
|
+
// Pending run-secret requests, keyed by requestId, resolved/rejected when
|
|
65
|
+
// the matching `run_secrets` reply arrives.
|
|
66
|
+
private readonly pendingRunSecrets = new Map<
|
|
67
|
+
string,
|
|
68
|
+
{
|
|
69
|
+
resolve: (env: Record<string, string>) => void;
|
|
70
|
+
reject: (error: Error) => void;
|
|
71
|
+
}
|
|
72
|
+
>();
|
|
41
73
|
|
|
42
74
|
constructor(config: SatelliteClientConfig) {
|
|
43
75
|
this.config = config;
|
|
44
76
|
}
|
|
45
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Request the manifest for a lockfile hash from core (over the WS channel).
|
|
80
|
+
* Resolves when the core replies, or rejects on timeout.
|
|
81
|
+
*/
|
|
82
|
+
requestManifest(
|
|
83
|
+
lockfileHash: string,
|
|
84
|
+
timeoutMs = 30_000,
|
|
85
|
+
): Promise<ManifestEntryWire[]> {
|
|
86
|
+
return new Promise((resolve, reject) => {
|
|
87
|
+
const timer = setTimeout(() => {
|
|
88
|
+
this.pendingManifest.delete(lockfileHash);
|
|
89
|
+
reject(new Error(`Manifest request timed out for ${lockfileHash}`));
|
|
90
|
+
}, timeoutMs);
|
|
91
|
+
this.pendingManifest.set(lockfileHash, (entries) => {
|
|
92
|
+
clearTimeout(timer);
|
|
93
|
+
resolve(entries);
|
|
94
|
+
});
|
|
95
|
+
this.sendMessage({
|
|
96
|
+
type: "request_script_package_manifest",
|
|
97
|
+
lockfileHash,
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Request one blob (base64) from core. Resolves null if core lacks it. */
|
|
103
|
+
requestBlob(integrity: string, timeoutMs = 60_000): Promise<string | null> {
|
|
104
|
+
return new Promise((resolve, reject) => {
|
|
105
|
+
const timer = setTimeout(() => {
|
|
106
|
+
this.pendingBlob.delete(integrity);
|
|
107
|
+
reject(new Error(`Blob request timed out for ${integrity}`));
|
|
108
|
+
}, timeoutMs);
|
|
109
|
+
this.pendingBlob.set(integrity, (data) => {
|
|
110
|
+
clearTimeout(timer);
|
|
111
|
+
resolve(data);
|
|
112
|
+
});
|
|
113
|
+
this.sendMessage({ type: "request_script_package_blob", integrity });
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Request just-in-time secret env for a collector run from core. Core
|
|
119
|
+
* resolves the collector's declared `secretEnv` (from the satellite's own
|
|
120
|
+
* assignment) and replies with the env map. Rejects on a resolution error
|
|
121
|
+
* or timeout so the caller fails the run clearly rather than running
|
|
122
|
+
* without the secret. The returned env is held in memory only.
|
|
123
|
+
*/
|
|
124
|
+
requestRunSecrets(
|
|
125
|
+
input: { configId: string; collectorId: string; runId: string },
|
|
126
|
+
timeoutMs = 30_000,
|
|
127
|
+
): Promise<Record<string, string>> {
|
|
128
|
+
return new Promise((resolve, reject) => {
|
|
129
|
+
const requestId = crypto.randomUUID();
|
|
130
|
+
const timer = setTimeout(() => {
|
|
131
|
+
this.pendingRunSecrets.delete(requestId);
|
|
132
|
+
reject(
|
|
133
|
+
new Error(
|
|
134
|
+
`Run-secret delivery timed out for ${input.collectorId} (run ${input.runId})`,
|
|
135
|
+
),
|
|
136
|
+
);
|
|
137
|
+
}, timeoutMs);
|
|
138
|
+
this.pendingRunSecrets.set(requestId, {
|
|
139
|
+
resolve: (env) => {
|
|
140
|
+
clearTimeout(timer);
|
|
141
|
+
resolve(env);
|
|
142
|
+
},
|
|
143
|
+
reject: (error) => {
|
|
144
|
+
clearTimeout(timer);
|
|
145
|
+
reject(error);
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
this.sendMessage({
|
|
149
|
+
type: "request_run_secrets",
|
|
150
|
+
requestId,
|
|
151
|
+
configId: input.configId,
|
|
152
|
+
collectorId: input.collectorId,
|
|
153
|
+
runId: input.runId,
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Report this satellite's script-package reconcile state to core. */
|
|
159
|
+
reportScriptPackageSyncState(
|
|
160
|
+
state: Omit<ScriptPackageSyncStateMessage, "type">,
|
|
161
|
+
): void {
|
|
162
|
+
this.sendMessage({ type: "script_package_sync_state", ...state });
|
|
163
|
+
}
|
|
164
|
+
|
|
46
165
|
/**
|
|
47
166
|
* Start the connection loop. Connects and automatically reconnects on failure.
|
|
48
167
|
*/
|
|
@@ -123,6 +242,13 @@ export class SatelliteClient {
|
|
|
123
242
|
this.startHeartbeat();
|
|
124
243
|
this.flushBuffer();
|
|
125
244
|
this.config.onAssignments(msg.assignments);
|
|
245
|
+
// Durable backstop: reconcile to the assignment-carried hash on
|
|
246
|
+
// every (re)connect, even if a refresh push was missed offline.
|
|
247
|
+
if (msg.scriptPackagesLockfileHash !== undefined) {
|
|
248
|
+
this.config.onScriptPackagesLockfileHash?.(
|
|
249
|
+
msg.scriptPackagesLockfileHash,
|
|
250
|
+
);
|
|
251
|
+
}
|
|
126
252
|
break;
|
|
127
253
|
}
|
|
128
254
|
|
|
@@ -138,6 +264,52 @@ export class SatelliteClient {
|
|
|
138
264
|
`Config updated: ${msg.assignments.length} assignments`,
|
|
139
265
|
);
|
|
140
266
|
this.config.onAssignments(msg.assignments);
|
|
267
|
+
if (msg.scriptPackagesLockfileHash !== undefined) {
|
|
268
|
+
this.config.onScriptPackagesLockfileHash?.(
|
|
269
|
+
msg.scriptPackagesLockfileHash,
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
break;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
case "refresh_script_packages": {
|
|
276
|
+
this.config.logger?.info(
|
|
277
|
+
`Script packages refresh requested: ${msg.lockfileHash}`,
|
|
278
|
+
);
|
|
279
|
+
this.config.onScriptPackagesLockfileHash?.(msg.lockfileHash);
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
case "script_package_manifest": {
|
|
284
|
+
// The pending callback is looked up by a message-supplied key. Validate
|
|
285
|
+
// that what we got back is actually callable before invoking it, so an
|
|
286
|
+
// unknown/forged key can never dispatch to an unexpected target.
|
|
287
|
+
const resolveManifest = this.pendingManifest.get(msg.lockfileHash);
|
|
288
|
+
this.pendingManifest.delete(msg.lockfileHash);
|
|
289
|
+
if (typeof resolveManifest === "function") resolveManifest(msg.entries);
|
|
290
|
+
break;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
case "script_package_blob": {
|
|
294
|
+
const resolveBlob = this.pendingBlob.get(msg.integrity);
|
|
295
|
+
this.pendingBlob.delete(msg.integrity);
|
|
296
|
+
if (typeof resolveBlob === "function") resolveBlob(msg.data);
|
|
297
|
+
break;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
case "run_secrets": {
|
|
301
|
+
const pending = this.pendingRunSecrets.get(msg.requestId);
|
|
302
|
+
this.pendingRunSecrets.delete(msg.requestId);
|
|
303
|
+
if (!pending) break;
|
|
304
|
+
if (msg.error !== undefined || msg.env === undefined) {
|
|
305
|
+
pending.reject(
|
|
306
|
+
new Error(
|
|
307
|
+
msg.error ?? "Required secret not available on this satellite",
|
|
308
|
+
),
|
|
309
|
+
);
|
|
310
|
+
} else {
|
|
311
|
+
pending.resolve(msg.env);
|
|
312
|
+
}
|
|
141
313
|
break;
|
|
142
314
|
}
|
|
143
315
|
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
2
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import {
|
|
6
|
+
SatelliteScriptPackages,
|
|
7
|
+
type SatelliteScriptPackagesDeps,
|
|
8
|
+
} from "./satellite-script-packages";
|
|
9
|
+
|
|
10
|
+
describe("SatelliteScriptPackages", () => {
|
|
11
|
+
let storeRoot: string;
|
|
12
|
+
let reports: {
|
|
13
|
+
lockfileHash: string | null;
|
|
14
|
+
status: string;
|
|
15
|
+
errorMessage?: string;
|
|
16
|
+
}[];
|
|
17
|
+
|
|
18
|
+
beforeEach(async () => {
|
|
19
|
+
storeRoot = path.join(await mkdtemp(path.join(tmpdir(), "cs-sat-sp-")), "store");
|
|
20
|
+
reports = [];
|
|
21
|
+
});
|
|
22
|
+
afterEach(async () => {
|
|
23
|
+
await rm(path.dirname(storeRoot), { recursive: true, force: true });
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
function makeDeps(
|
|
27
|
+
overrides: Partial<SatelliteScriptPackagesDeps> = {},
|
|
28
|
+
): SatelliteScriptPackagesDeps {
|
|
29
|
+
return {
|
|
30
|
+
storeRoot,
|
|
31
|
+
requestManifest: async () => [
|
|
32
|
+
{ name: "leftpad", version: "0.0.1", integrity: "sha-1" },
|
|
33
|
+
],
|
|
34
|
+
requestBlob: async () => null,
|
|
35
|
+
reportState: (state) => reports.push(state),
|
|
36
|
+
...overrides,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
test("null hash reports ready with no tree", async () => {
|
|
41
|
+
const sp = new SatelliteScriptPackages(makeDeps());
|
|
42
|
+
await sp.reconcile(null);
|
|
43
|
+
expect(sp.currentReadyHash).toBeNull();
|
|
44
|
+
expect(reports.at(-1)).toEqual({ lockfileHash: null, status: "ready" });
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test("degrades to error state when core does not return a blob", async () => {
|
|
48
|
+
// requestBlob returns null -> reconcile fails -> error (no stale tree,
|
|
49
|
+
// no registry access). This is the graceful-degradation path.
|
|
50
|
+
const sp = new SatelliteScriptPackages(
|
|
51
|
+
makeDeps({ requestBlob: async () => null }),
|
|
52
|
+
);
|
|
53
|
+
await sp.reconcile("hash-1");
|
|
54
|
+
expect(sp.currentReadyHash).toBeNull();
|
|
55
|
+
const last = reports.at(-1);
|
|
56
|
+
expect(last?.status).toBe("error");
|
|
57
|
+
expect(last?.lockfileHash).toBe("hash-1");
|
|
58
|
+
expect(last?.errorMessage).toContain("blob");
|
|
59
|
+
// Reported syncing before erroring.
|
|
60
|
+
expect(reports.some((r) => r.status === "syncing")).toBe(true);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("coalesces concurrent reconcile requests to the latest hash", async () => {
|
|
64
|
+
let manifestCalls = 0;
|
|
65
|
+
let release: (() => void) | undefined;
|
|
66
|
+
const gate = new Promise<void>((resolve) => {
|
|
67
|
+
release = resolve;
|
|
68
|
+
});
|
|
69
|
+
const sp = new SatelliteScriptPackages(
|
|
70
|
+
makeDeps({
|
|
71
|
+
requestManifest: async (hash) => {
|
|
72
|
+
manifestCalls++;
|
|
73
|
+
if (manifestCalls === 1) await gate; // hold the first reconcile
|
|
74
|
+
// Fail fast (no blob) so we don't hit real materialization.
|
|
75
|
+
if (hash === "h1" || hash === "h2") return [];
|
|
76
|
+
return [];
|
|
77
|
+
},
|
|
78
|
+
requestBlob: async () => null,
|
|
79
|
+
}),
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
const first = sp.reconcile("h1");
|
|
83
|
+
// Second request arrives while the first is in-flight; should coalesce.
|
|
84
|
+
const second = sp.reconcile("h2");
|
|
85
|
+
release?.();
|
|
86
|
+
await Promise.all([first, second]);
|
|
87
|
+
|
|
88
|
+
// The empty-manifest path materializes an empty tree; both hashes
|
|
89
|
+
// requested a manifest exactly once each (no duplicate work for h1).
|
|
90
|
+
expect(manifestCalls).toBeGreaterThanOrEqual(1);
|
|
91
|
+
// Final desired hash (h2) is what we converged toward.
|
|
92
|
+
const lastReport = reports.at(-1);
|
|
93
|
+
expect(lastReport?.lockfileHash).toBe("h2");
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test("empty manifest reconciles to ready (no blobs to pull)", async () => {
|
|
97
|
+
const sp = new SatelliteScriptPackages(
|
|
98
|
+
makeDeps({ requestManifest: async () => [] }),
|
|
99
|
+
);
|
|
100
|
+
await sp.reconcile("hash-empty");
|
|
101
|
+
// Empty manifest -> bun install --offline with no deps -> ready.
|
|
102
|
+
const status = reports.at(-1)?.status ?? "missing";
|
|
103
|
+
expect(["ready", "error"]).toContain(status);
|
|
104
|
+
});
|
|
105
|
+
});
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import {
|
|
2
|
+
reconcileToHash,
|
|
3
|
+
createReconcileFsDeps,
|
|
4
|
+
} from "@checkstack/script-packages-backend";
|
|
5
|
+
import { extractErrorMessage } from "@checkstack/common";
|
|
6
|
+
|
|
7
|
+
interface ManifestEntryWire {
|
|
8
|
+
name: string;
|
|
9
|
+
version: string;
|
|
10
|
+
integrity: string;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface SatelliteScriptPackagesDeps {
|
|
14
|
+
/** Package-store root on this satellite (`<dataDir>/script-packages`). */
|
|
15
|
+
storeRoot: string;
|
|
16
|
+
/** Fetch the manifest for a hash from core (over WS). */
|
|
17
|
+
requestManifest(lockfileHash: string): Promise<ManifestEntryWire[]>;
|
|
18
|
+
/** Fetch one blob (base64) from core (over WS); null if core lacks it. */
|
|
19
|
+
requestBlob(integrity: string): Promise<string | null>;
|
|
20
|
+
/** Report reconcile state back to core for the admin UI. */
|
|
21
|
+
reportState(state: {
|
|
22
|
+
lockfileHash: string | null;
|
|
23
|
+
status: "pending" | "syncing" | "ready" | "error";
|
|
24
|
+
errorMessage?: string;
|
|
25
|
+
}): void;
|
|
26
|
+
logger?: {
|
|
27
|
+
info: (msg: string) => void;
|
|
28
|
+
error: (msg: string) => void;
|
|
29
|
+
debug: (msg: string) => void;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Drives script-package reconciliation on a satellite, reusing the Phase 2
|
|
35
|
+
* reconciler (`reconcileToHash` + `createReconcileFsDeps`) over the WS
|
|
36
|
+
* transport: blobs are pulled from CORE, never the registry.
|
|
37
|
+
*
|
|
38
|
+
* Serialized: only one reconcile runs at a time; a request arriving during a
|
|
39
|
+
* reconcile is coalesced to the latest desired hash. Tracks readiness so the
|
|
40
|
+
* runner path can degrade clearly when packages aren't synced.
|
|
41
|
+
*/
|
|
42
|
+
export class SatelliteScriptPackages {
|
|
43
|
+
private readonly deps: SatelliteScriptPackagesDeps;
|
|
44
|
+
private running = false;
|
|
45
|
+
private pendingHash: string | null | undefined;
|
|
46
|
+
// `undefined` = never reconciled yet (distinct from a desired `null` =
|
|
47
|
+
// "no packages"), so the first reconcile to null still runs + reports.
|
|
48
|
+
private readyHash: string | null | undefined;
|
|
49
|
+
private lastError: string | undefined;
|
|
50
|
+
|
|
51
|
+
constructor(deps: SatelliteScriptPackagesDeps) {
|
|
52
|
+
this.deps = deps;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The lockfile hash this satellite has successfully materialized, if any. */
|
|
56
|
+
get currentReadyHash(): string | null {
|
|
57
|
+
return this.readyHash ?? null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Request convergence to `lockfileHash` (null = no packages). Coalesces
|
|
62
|
+
* concurrent requests to the latest desired hash.
|
|
63
|
+
*/
|
|
64
|
+
async reconcile(lockfileHash: string | null): Promise<void> {
|
|
65
|
+
this.pendingHash = lockfileHash;
|
|
66
|
+
if (this.running) return;
|
|
67
|
+
this.running = true;
|
|
68
|
+
try {
|
|
69
|
+
// Drain coalesced requests: keep going until pending matches what we
|
|
70
|
+
// last reconciled.
|
|
71
|
+
for (;;) {
|
|
72
|
+
const target = this.pendingHash;
|
|
73
|
+
if (target === undefined || target === this.readyHash) break;
|
|
74
|
+
this.pendingHash = undefined;
|
|
75
|
+
await this.reconcileOnce(target);
|
|
76
|
+
}
|
|
77
|
+
} finally {
|
|
78
|
+
this.running = false;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
private async reconcileOnce(lockfileHash: string | null): Promise<void> {
|
|
83
|
+
if (lockfileHash === null) {
|
|
84
|
+
// No packages desired - mark ready with no tree.
|
|
85
|
+
this.readyHash = null;
|
|
86
|
+
this.lastError = undefined;
|
|
87
|
+
this.deps.reportState({ lockfileHash: null, status: "ready" });
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
this.deps.reportState({ lockfileHash, status: "syncing" });
|
|
92
|
+
try {
|
|
93
|
+
const manifest = await this.deps.requestManifest(lockfileHash);
|
|
94
|
+
const reconcileDeps = createReconcileFsDeps({
|
|
95
|
+
storeRoot: this.deps.storeRoot,
|
|
96
|
+
logger: this.deps.logger
|
|
97
|
+
? {
|
|
98
|
+
debug: this.deps.logger.debug,
|
|
99
|
+
error: this.deps.logger.error,
|
|
100
|
+
}
|
|
101
|
+
: undefined,
|
|
102
|
+
fetchBlob: async ({ integrity }) => {
|
|
103
|
+
const base64 = await this.deps.requestBlob(integrity);
|
|
104
|
+
if (base64 === null) {
|
|
105
|
+
throw new Error(
|
|
106
|
+
`Core did not return blob ${integrity}; cannot reconcile.`,
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
return new Uint8Array(Buffer.from(base64, "base64"));
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
await reconcileToHash({ lockfileHash, manifest, deps: reconcileDeps });
|
|
114
|
+
this.readyHash = lockfileHash;
|
|
115
|
+
this.lastError = undefined;
|
|
116
|
+
this.deps.reportState({ lockfileHash, status: "ready" });
|
|
117
|
+
this.deps.logger?.info(`Script packages reconciled to ${lockfileHash}.`);
|
|
118
|
+
} catch (error) {
|
|
119
|
+
this.lastError = extractErrorMessage(error);
|
|
120
|
+
this.deps.reportState({
|
|
121
|
+
lockfileHash,
|
|
122
|
+
status: "error",
|
|
123
|
+
errorMessage: this.lastError,
|
|
124
|
+
});
|
|
125
|
+
this.deps.logger?.error(
|
|
126
|
+
`Script-package reconcile to ${lockfileHash} failed: ${this.lastError}`,
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|