@checkstack/script-packages-backend 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +273 -0
- package/drizzle/0000_flashy_squadron_supreme.sql +63 -0
- package/drizzle/0001_flawless_drax.sql +15 -0
- package/drizzle/meta/0000_snapshot.json +395 -0
- package/drizzle/meta/0001_snapshot.json +491 -0
- package/drizzle/meta/_journal.json +20 -0
- package/drizzle.config.ts +7 -0
- package/package.json +32 -0
- package/src/atomic-symlink.test.ts +47 -0
- package/src/atomic-symlink.ts +66 -0
- package/src/blob-gc-runner.test.ts +120 -0
- package/src/blob-gc-runner.ts +139 -0
- package/src/blob-gc.test.ts +182 -0
- package/src/blob-gc.ts +161 -0
- package/src/blob-hash.test.ts +70 -0
- package/src/blob-hash.ts +56 -0
- package/src/blob-store-registry.test.ts +78 -0
- package/src/blob-store-registry.ts +75 -0
- package/src/blob-store.ts +51 -0
- package/src/cache-archive.test.ts +164 -0
- package/src/cache-archive.ts +192 -0
- package/src/cache-layout.ts +64 -0
- package/src/data-dir.test.ts +41 -0
- package/src/data-dir.ts +42 -0
- package/src/e2e-install-reconcile.test.ts +121 -0
- package/src/hooks.ts +20 -0
- package/src/index.ts +594 -0
- package/src/install-controller.test.ts +257 -0
- package/src/install-controller.ts +144 -0
- package/src/install-service.test.ts +104 -0
- package/src/install-service.ts +116 -0
- package/src/install-state-store.ts +131 -0
- package/src/lockfile.test.ts +60 -0
- package/src/lockfile.ts +0 -0
- package/src/npmrc.test.ts +48 -0
- package/src/npmrc.ts +42 -0
- package/src/package-types.test.ts +293 -0
- package/src/package-types.ts +408 -0
- package/src/parse-bun-lock.test.ts +62 -0
- package/src/parse-bun-lock.ts +59 -0
- package/src/reconcile-diff.test.ts +41 -0
- package/src/reconcile-diff.ts +26 -0
- package/src/reconcile-fs.ts +199 -0
- package/src/reconciler.test.ts +289 -0
- package/src/reconciler.ts +81 -0
- package/src/registry-client.test.ts +314 -0
- package/src/registry-client.ts +0 -0
- package/src/registry-request-config.ts +63 -0
- package/src/registry-token.test.ts +124 -0
- package/src/registry-token.ts +104 -0
- package/src/resolution-root.test.ts +82 -0
- package/src/resolution-root.ts +127 -0
- package/src/resolver.test.ts +133 -0
- package/src/resolver.ts +132 -0
- package/src/router.ts +273 -0
- package/src/schema.ts +166 -0
- package/src/size-cap.test.ts +32 -0
- package/src/size-cap.ts +40 -0
- package/src/storage-migration.test.ts +318 -0
- package/src/storage-migration.ts +213 -0
- package/src/stores.ts +533 -0
- package/src/tree-gc.test.ts +184 -0
- package/src/tree-gc.ts +160 -0
- package/src/tree-retirement.ts +81 -0
- package/src/type-acquisition-route.ts +178 -0
- package/tsconfig.json +23 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
# @checkstack/script-packages-backend
|
|
2
|
+
|
|
3
|
+
## 0.2.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 270ef29: Fix several correctness defects around distributed coordination and stored-data handling.
|
|
8
|
+
|
|
9
|
+
- Dwell `for:` timers now fire via an atomic `DELETE ... RETURNING` claim, so two pods (or the stalled sweeper vs the queue consumer) can no longer both fire the same dwell.
|
|
10
|
+
- Postgres session-level advisory locks now keep connection affinity. A shared `AdvisoryLockService` (backed by a dedicated pooled client) replaces the previous acquire/release-on-different-connection pattern that leaked locks. Used by the script-packages installer election, the automation run resume + stalled sweeper, and (via a new transaction-scoped `withXactLock`) incident dedup.
|
|
11
|
+
- A storage migration that crashed mid-flight is now resumed on startup under the installer-election lock, instead of permanently wedging installs.
|
|
12
|
+
- Distributed script-package blobs carry a `blobSha256` and are verified before extraction (the SRI `integrity` hashes the npm tarball, not the transported archive). Backward-safe: entries without the field skip verification until a re-install regenerates the manifest.
|
|
13
|
+
- Archive extraction rejects zip-slip paths (absolute or `..` entries) before writing anything.
|
|
14
|
+
- `incident.create` with `dedupe_open_for_system` serializes its check-then-create per system, so concurrent triggers for the same system can't both open a duplicate incident.
|
|
15
|
+
- Seeded auto-incident filter expressions JSON-encode interpolated ids so a quote/backslash can't corrupt the expression.
|
|
16
|
+
- Stored jsonb snapshots (dwell `actorSnapshot`, wait-lock `waitConfig`) are validated with zod on load and degrade safely instead of flowing through as the wrong type.
|
|
17
|
+
|
|
18
|
+
- 270ef29: Core-side satellite script-package distribution.
|
|
19
|
+
|
|
20
|
+
- `satellite-backend`: the WS handler now carries the desired script-package
|
|
21
|
+
lockfile hash in `authenticated` / `config_updated` payloads (the durable
|
|
22
|
+
backstop), exposes `pushRefreshScriptPackagesToAll` (wired to the
|
|
23
|
+
`script-packages.changed` broadcast hook in `mode: "broadcast"`, so each
|
|
24
|
+
core instance fans the refresh out to its own connected satellites), and
|
|
25
|
+
persists `script_package_sync_state` reports from satellites.
|
|
26
|
+
- `script-packages-*`: new `reportSatelliteSyncState` RPC + store method so
|
|
27
|
+
satellite-backend can record per-satellite reconcile state for the admin
|
|
28
|
+
UI. Satellites pull blobs from core via the existing `getManifest` /
|
|
29
|
+
`downloadBlob` endpoints, never the registry.
|
|
30
|
+
|
|
31
|
+
- 270ef29: Activate npm packages in script execution: thread the managed
|
|
32
|
+
`resolutionRoot` into every user-script call site so an allowlisted package
|
|
33
|
+
can actually be `import`ed.
|
|
34
|
+
|
|
35
|
+
- `@checkstack/backend-api`: the ESM runner now always writes a per-run
|
|
36
|
+
`bunfig.toml` with `[install] auto = "disable"` and runs with that dir as
|
|
37
|
+
CWD. Without this Bun silently auto-installs any imported package from the
|
|
38
|
+
registry (verified), defeating the allowlist; with it, imports resolve
|
|
39
|
+
only against the reconciled `current/node_modules` (when a `resolutionRoot`
|
|
40
|
+
is set) and otherwise fail fast.
|
|
41
|
+
- `@checkstack/script-packages-backend`: `resolveResolutionRoot` /
|
|
42
|
+
`resolveResolutionRootFromStore` / `resolveResolutionRootForHost` decide a
|
|
43
|
+
host's resolution-root status (`none` / `ready` / `notReady`) from the
|
|
44
|
+
local `<store>/current`.
|
|
45
|
+
- `run_script` (integration-script-backend), the inline-script collector
|
|
46
|
+
(healthcheck-script-backend, core + satellite), and the in-UI `testScript`
|
|
47
|
+
/ `testCollectorScript` endpoints all resolve the root per run and pass it
|
|
48
|
+
to the runner; `run_script` surfaces a clear "npm packages not ready"
|
|
49
|
+
error when configured-but-unsynced. Shell paths are unaffected (no module
|
|
50
|
+
resolution).
|
|
51
|
+
|
|
52
|
+
An opt-in end-to-end test (`CHECKSTACK_E2E_NETWORK=1`) proves an allowlisted
|
|
53
|
+
package imports successfully through the real `run_script` action execute
|
|
54
|
+
path, with non-network degradation tests running always.
|
|
55
|
+
|
|
56
|
+
BREAKING CHANGES: `@checkstack/backend-api`'s `defaultEsmScriptRunner` now
|
|
57
|
+
always disables Bun auto-install for the user subprocess. A script that
|
|
58
|
+
previously relied on Bun silently fetching an un-vendored package from the
|
|
59
|
+
registry at import time will now fail to resolve it. This is intentional -
|
|
60
|
+
package availability is governed by the admin allowlist - but any caller
|
|
61
|
+
depending on the old implicit auto-install behavior must add the package to
|
|
62
|
+
the allowlist instead. The new `EsmScriptRunOptions.resolutionRoot` field is
|
|
63
|
+
optional and additive (defaults to today's `os.tmpdir()` behavior when
|
|
64
|
+
unset), so the runner API itself is source-compatible.
|
|
65
|
+
|
|
66
|
+
- 270ef29: Add the script-packages backend package skeleton: Drizzle data model
|
|
67
|
+
(allowlist, registry config, install state, size cap, content-addressed
|
|
68
|
+
blob index, storage config, per-satellite sync state) + initial
|
|
69
|
+
migration, the `blobStoreExtensionPoint` + `BlobStore` interface and a
|
|
70
|
+
blob-store registry (with dual-backend read fallback for storage
|
|
71
|
+
migration), the `CHECKSTACK_DATA_DIR` package-store path resolver, and
|
|
72
|
+
the plugin registration that wires the extension point + access rules.
|
|
73
|
+
- b995afb: Fix two Script Packages bugs: empty-allowlist installs and read-only Advanced settings.
|
|
74
|
+
|
|
75
|
+
- Install now: clicking "Install now" with no enabled packages no longer fails with ENOENT and an `error` install state. With an empty dependency set `bun install` writes no `bun.lock`, so the central resolver previously threw reading it. The resolver now short-circuits an empty (or all-disabled) allowlist to an empty resolved set, ending the install in `ready` at 0.0 MB with the deterministic empty-lockfile hash. No subprocess or registry call is made in that case.
|
|
76
|
+
- Advanced settings: the registry URL, "ignore install scripts" toggle, write-only auth token, size guardrail thresholds, and active storage backend are now editable in the Script Packages settings page (previously read-only displays) and wired to the existing `setRegistryConfig` / `setSizeCapConfig` / `setStorageBackend` mutations. The auth-token field is write-only: a blank field leaves the stored token untouched, and a "Clear token" action removes it. The destructive blob-migration flow is unchanged.
|
|
77
|
+
|
|
78
|
+
No schema or RPC contract changes.
|
|
79
|
+
|
|
80
|
+
- 270ef29: Add garbage collection for script packages, reclaiming both shared blob storage and per-host disk.
|
|
81
|
+
|
|
82
|
+
Two things accumulated and were never reclaimed; both are now collected with provably-safe guards (referenced-set + grace + lock / active-run protection - when in doubt, retain).
|
|
83
|
+
|
|
84
|
+
- **Blob GC (Postgres / S3 reclamation).** A new `gcBlobs` RPC (manage-gated) plus a daily scheduled job prune content-addressed blobs no longer referenced by any retained lockfile manifest (the current desired hash plus the previous N, default 1, for rollback / in-flight reconciles). Candidates are only deleted after a grace window (default 24h, keyed on `created_at`), each blob's bytes are removed from its recorded backend before its index row, and the pass holds the installer-election advisory lock so it is mutually exclusive with installs and storage migrations. The settings UI surfaces last-run and total-reclaimed figures and a "Run cleanup now" action. The installer now records each successful manifest in a new `script_package_lockfile_history` table so the retained set is computable; GC state lives in `script_package_blob_gc_state`.
|
|
85
|
+
- **Tree GC (per-host disk).** After a successful symlink flip, each host (core pod or satellite) sweeps its `trees/<lockfileHash>/` dirs, deleting non-current trees older than a grace window (default 1h). `current`'s target is never deleted. Active-run safety uses a conservative mtime-keyed grace window chosen to exceed the longest run timeout, since runs are throwaway subprocesses with no robust cross-process refcount - a tree only becomes eligible once no live run could still be pinned to it.
|
|
86
|
+
|
|
87
|
+
Adds the `gcBlobs` / `getBlobGcState` RPCs, the `BlobGcSummary` / `BlobGcState` schemas, and two new Drizzle tables (`script_package_lockfile_history`, `script_package_blob_gc_state`).
|
|
88
|
+
|
|
89
|
+
- 270ef29: Harden the script-packages store against three confirmed defects:
|
|
90
|
+
|
|
91
|
+
- **Tree GC no longer deletes live trees.** The tree garbage collector keyed
|
|
92
|
+
its grace window on the materialized tree's dir mtime. A tree that had been
|
|
93
|
+
`current` for days carried an ancient mtime, so it became eligible for
|
|
94
|
+
deletion the instant it was superseded by a flip - and the post-flip sweep
|
|
95
|
+
would then delete a tree that an in-flight run (which snapshots its
|
|
96
|
+
resolution root at run start) was still pinned to. The flip now stamps a
|
|
97
|
+
`.retired-at` marker into the superseded tree, and the grace window is
|
|
98
|
+
measured from that retirement timestamp. A non-current tree with no marker
|
|
99
|
+
is retained (and lazily back-filled) so it ages out instead of leaking, and
|
|
100
|
+
is never deleted on a missing signal.
|
|
101
|
+
|
|
102
|
+
BREAKING CHANGE: the tree-GC grace window is now measured from a tree's
|
|
103
|
+
retirement time (when it stopped being `current`), not its dir mtime.
|
|
104
|
+
Existing non-current trees with no `.retired-at` marker are retained on the
|
|
105
|
+
first sweep and back-filled, then collected on a later sweep once the grace
|
|
106
|
+
window elapses from the back-filled time.
|
|
107
|
+
|
|
108
|
+
- **Installer no longer leaves a plaintext registry token on disk after a
|
|
109
|
+
failed resolve.** The central resolver wrote the auth-token-bearing
|
|
110
|
+
`.npmrc` into its scratch dir but only removed the scratch dir on the
|
|
111
|
+
success path; any failure between `bun install` and packing the cache
|
|
112
|
+
entries left the token on disk. Scratch-dir removal now runs in a `finally`
|
|
113
|
+
so the token is cleaned up on every exit path.
|
|
114
|
+
|
|
115
|
+
- **Tar extraction rejects symlink/hardlink entries.** Blob unpacking
|
|
116
|
+
validated entry names against zip-slip but not link targets, so a symlink
|
|
117
|
+
with a safe name but an escaping target (for example `-> /etc` or
|
|
118
|
+
`-> ../../..`) passed; a later regular-file entry could then be written
|
|
119
|
+
through it and escape the target directory. The listing pass now inspects
|
|
120
|
+
entry types (`tar -tzvf`) and rejects any non-regular, non-directory entry.
|
|
121
|
+
|
|
122
|
+
- 270ef29: Add the install/resolve service for script packages: deterministic
|
|
123
|
+
lockfile-manifest hashing (content-addressed, order-independent),
|
|
124
|
+
generated store `package.json` builder, `.npmrc` renderer (auth token
|
|
125
|
+
write-only, never logged), `bun.lock` parser (name/version/integrity
|
|
126
|
+
extraction), the elected-installer Postgres advisory lock (pattern copied
|
|
127
|
+
from automation-backend) + singleton install-state store, the
|
|
128
|
+
`performInstall` orchestration (resolve -> delta-publish blobs to the
|
|
129
|
+
active store -> record manifest/hash/size), and the admin-configurable
|
|
130
|
+
size-cap guardrail (warn 150MB / block 300MB).
|
|
131
|
+
|
|
132
|
+
Empirically verified: `bun install --offline` reconstructs `node_modules`
|
|
133
|
+
from a pre-seeded Bun cache with zero network access (the delta-sync model
|
|
134
|
+
the reconciler builds on). Hardlink-vs-copy is filesystem-dependent and
|
|
135
|
+
does not affect correctness.
|
|
136
|
+
|
|
137
|
+
- b995afb: Fix package IntelliSense in script editors: lazy Automatic Type Acquisition (ATA) with proper `@types/*` resolution.
|
|
138
|
+
|
|
139
|
+
Script editors (automation "Run Script (TypeScript)" and healthcheck collectors) now provide real autocomplete for installed npm packages. Importing a package whose types live in DefinitelyTyped - e.g. `import { debounce } from "lodash"` (lodash ships no own types; `@types/lodash` does) - now yields member completions. Previously no package completions appeared at all.
|
|
140
|
+
|
|
141
|
+
Root cause: the old rollup wrapped each package's raw, multi-file `.d.ts` (with `export =`, `export as namespace`, and triple-slash `/// <reference path>` chains) inside a single `declare module "<name>" { ... }`, which the TypeScript worker silently rejected, and it truncated large type sets (lodash is ~866 KB across ~700 files) at a 256 KB cap.
|
|
142
|
+
|
|
143
|
+
The fix registers the REAL declaration files at their `node_modules/...` virtual paths and lets TypeScript's own NodeJs + `@types` resolution do the work:
|
|
144
|
+
|
|
145
|
+
- `@checkstack/script-packages-backend`: replaced `rollupPackageTypes` with a tree-driven closure extractor (`resolvePackageTypeClosure`). Given a bare specifier, it resolves against the materialized tree - own types via `package.json` `types`/`typings`/`exports` (bundled-types packages like `zod`/`dayjs`), the `@types/<mangled>` companion when it exists (`lodash` -> `@types/lodash`, scoped `@babel/core` -> `@types/babel__core`), or both, or neither (graceful empty, never a throw). It follows `/// <reference path|types>` and relative imports, includes each package's `package.json`, leaves every file UNWRAPPED, and surfaces a `truncated` flag instead of silently capping. Served from a new raw, HTTP-cacheable route `GET /api/script-packages/types/:lockfileHash/:specifier` (`Cache-Control: private, max-age=1y, immutable`), auth-gated by `script-packages.read`.
|
|
146
|
+
- `@checkstack/script-packages-common`: **BREAKING** - replaced the `listPackageTypes` RPC procedure and `PackageTypesSchema { name, version, dts }` with `PackageTypeClosureSchema` (a `{ path, content }` file-map plus `hasOwnTypes`/`hasAtTypes`/`notFound`/`truncated`) served over the cacheable HTTP route. Added a shared `buildTypeAcquisitionPath`/`parseTypeAcquisitionPath` path contract.
|
|
147
|
+
- `@checkstack/ui`: `CodeEditor`/`TypefoxEditor` gained an injected `acquireTypes` resolver + `acquireResetKey`. On debounced buffer change it parses bare `import`/`require` specifiers (pure, unit-tested) and lazily fetches + registers each NEW package's closure via `addExtraLib` at `file:///node_modules/...`, deduped by a shared acquired-set that resets when the install hash changes. Compiler options set `moduleResolution: NodeJs`, `baseUrl: "file:///"`, and `typeRoots` so a bare import resolves to its `@types` companion. The `context` ambient global keeps working unchanged.
|
|
148
|
+
- `@checkstack/script-packages-frontend`: replaced the old `useScriptPackageTypes` (which concatenated the broken `dts`) with `useScriptPackageTypeAcquisition()`, returning the `acquireTypes` resolver (targets the cacheable route, zod-validates the response) and the current `lockfileHash` as `acquireResetKey`.
|
|
149
|
+
- `@checkstack/automation-frontend` / `@checkstack/healthcheck-frontend`: wired the resolver into the Run Script and collector editors.
|
|
150
|
+
|
|
151
|
+
State & scale: the type closure is derived on read from the materialized package tree (no new durable state). The editor's acquired-set is pod-local UI bookkeeping; the route is keyed by the cluster-wide `lockfileHash`, so the browser HTTP cache is correct across pods and only refetches after a new install changes the hash.
|
|
152
|
+
|
|
153
|
+
- 270ef29: Add the per-host script-package reconciler and the runner resolution root.
|
|
154
|
+
|
|
155
|
+
- `@checkstack/backend-api`: `EsmScriptRunOptions.resolutionRoot` - when
|
|
156
|
+
set, the per-run temp dir is created inside it so module resolution walks
|
|
157
|
+
up to `<resolutionRoot>/node_modules` and user scripts can `import`
|
|
158
|
+
managed npm packages. Defaults to today's `os.tmpdir()` behavior when
|
|
159
|
+
unset (backward-compatible; isolation unchanged - the subprocess still
|
|
160
|
+
only sees `SAFE_ENV_VARS`).
|
|
161
|
+
- `@checkstack/script-packages-backend`: content-addressed cache archive
|
|
162
|
+
(tar+gzip per package), pure delta diff (`computeMissingBlobs`), atomic
|
|
163
|
+
`current` symlink swap, the host reconciler (`reconcileToHash` -
|
|
164
|
+
idempotent: pull only missing blobs, materialize a versioned tree via
|
|
165
|
+
`bun install --offline`, atomically flip `current`), the concrete fs/Bun
|
|
166
|
+
adapter, the central install resolver, and the `script-packages.changed`
|
|
167
|
+
broadcast hook. An opt-in end-to-end test
|
|
168
|
+
(`CHECKSTACK_E2E_NETWORK=1`) proves resolve -> publish -> cold reconcile
|
|
169
|
+
(no registry) -> offline materialize -> import.
|
|
170
|
+
|
|
171
|
+
- b995afb: Add live, backend-proxied npm package-name autocomplete and version lookup to the Script Packages "Allowed packages" form.
|
|
172
|
+
|
|
173
|
+
The package-name field now searches the configured registry as you type (debounced). The version field suggests the package's published versions newest-first, defaulting to the registry's `latest` dist-tag while staying free-typeable for an exact manual pin. Picking a package now auto-fills its version (from the search hit, then upgraded to `latest`) instead of clearing the field, and the version dropdown stays open so a version is actually selectable.
|
|
174
|
+
|
|
175
|
+
This is fully backend-proxied so it reuses the SAME configured registry + auth the install path uses: the registry can be private with a server-side-only auth token (the client only ever sees `hasAuthToken`), and browsers can't reach arbitrary registries due to CORS.
|
|
176
|
+
|
|
177
|
+
- `@checkstack/script-packages-common`: two new `manage`-gated query procedures - `searchPackages` (input `{ text }`, output `{ items: [{ name, version?, description? }] }`) and `getPackageVersions` (input `{ name }`, output `{ versions, distTags? }`). Output `version`/`versions` are relaxed to plain strings so valid-but-unusual registry versions surface as suggestions; strict `PackageVersionSchema` validation still applies on `addPackage`.
|
|
178
|
+
- `@checkstack/script-packages-backend`: new `registry-client` (fetch-based, AbortController timeout, size cap, zod-validated registry responses, scoped-registry selection, tolerant semver-descending sort, a best-effort pod-local read cache that is never a source of truth, and errors that never leak the auth token). Search results use the registry's own relevance ranking from `-/v1/search`. The registry + token resolution used by the install path is factored into a shared `resolveRegistryRequestConfig` helper reused by both the new RPC handlers and the installer.
|
|
179
|
+
- `@checkstack/script-packages-frontend`: the package and version inputs become live comboboxes (Popover + Input) with inline pinned-version validation before "Add" is enabled. Selecting a suggestion routes through a dedicated `onSelect(hit)` callback (separate from manual-typing `onValueChange`) so a pick auto-fills the version instead of clearing it; the popover dismissable layer ignores interactions originating on the anchor input, fixing the version dropdown that previously opened then immediately closed. The version-autofill decision logic is extracted into a pure, unit-tested helper.
|
|
180
|
+
|
|
181
|
+
State & scale: the registry-client TTL cache is an explicitly non-authoritative, pod-local best-effort read cache (search/version lookups are non-authoritative reads); a cache miss on another pod simply re-fetches, so pod-local divergence is harmless. No new durable state of record is introduced.
|
|
182
|
+
|
|
183
|
+
- 270ef29: Wire up the script-packages RPC router, admin UI, and editor IntelliSense.
|
|
184
|
+
|
|
185
|
+
- `script-packages-backend`: the oRPC router implementing the full
|
|
186
|
+
contract (allowlist CRUD, registry config with encrypted write-only auth
|
|
187
|
+
token, `installNow` via the elected installer, size cap, storage backend
|
|
188
|
+
selection, install state, `getManifest` / `downloadBlob` for reconcilers,
|
|
189
|
+
and `listPackageTypes`), the `installNow` controller (election, size-cap
|
|
190
|
+
enforcement, `script-packages.changed` emit, blocked during migration),
|
|
191
|
+
the `.d.ts` rollup, the singleton config stores, and the full plugin
|
|
192
|
+
wiring (broadcast-hook reconcile + startup backstop).
|
|
193
|
+
- `script-packages-common`: admin route for the settings page.
|
|
194
|
+
- `script-packages-frontend`: the Settings -> Script Packages admin page
|
|
195
|
+
(allowlist, install state + size, registry/storage summary, satellite
|
|
196
|
+
sync) and the `useScriptPackageTypes()` hook.
|
|
197
|
+
- `automation-frontend` / `healthcheck-frontend`: merge installed-package
|
|
198
|
+
`.d.ts` into the script-editor `typeDefinitions` so `import` from an
|
|
199
|
+
allowlisted package autocompletes in every script field.
|
|
200
|
+
|
|
201
|
+
- 270ef29: Add storage-backend migration for script packages.
|
|
202
|
+
|
|
203
|
+
- `migrateStorage({ target })` copies every blob from the active backend to
|
|
204
|
+
the target, verifies each copy byte-for-byte (read back + SHA-256 compare),
|
|
205
|
+
flips the per-blob `backend` only after a verified copy, then atomically
|
|
206
|
+
switches the active backend. Resumable from a partial state (the work set
|
|
207
|
+
is re-derived from the index), aborts cleanly on an integrity mismatch
|
|
208
|
+
(active backend untouched), and supports optional source GC. Built on the
|
|
209
|
+
Phase 2 dual-backend read fallback, so reads keep working mid-migration.
|
|
210
|
+
- Migration and `installNow` are mutually exclusive via the installer
|
|
211
|
+
advisory lock; `setStorageBackend` is refused while a migration runs.
|
|
212
|
+
- New `listStorageBackends` RPC + admin UI: a storage-backend card with a
|
|
213
|
+
target selector, "Migrate" action, and live progress / completion / error
|
|
214
|
+
state.
|
|
215
|
+
|
|
216
|
+
- 270ef29: Secrets platform Phase 5: internal-secret consolidation (registry token) + connection-credential leak hardening.
|
|
217
|
+
|
|
218
|
+
- New `internalSecretsRef`: platform-internal secrets (not user-managed
|
|
219
|
+
named secrets) stored under a reserved `__internal__:` prefix, ALWAYS on
|
|
220
|
+
the local (always-writable, AES-GCM) backend so internal writes never
|
|
221
|
+
break when Vault is the active backend. Excluded from the user-facing
|
|
222
|
+
Secrets list.
|
|
223
|
+
- The script-package registry auth token is consolidated onto
|
|
224
|
+
`internalSecretsRef`. The `authSecretRef` column now holds a stable
|
|
225
|
+
marker; a one-time, idempotent, parity-verified migration moves legacy
|
|
226
|
+
inline ciphertext into the platform and only rewrites the column once the
|
|
227
|
+
platform copy reads back identically (legacy value never dropped early).
|
|
228
|
+
Resolution stays backward-compatible with legacy ciphertext.
|
|
229
|
+
- Integration: `createConnection` / `updateConnection` now return the
|
|
230
|
+
redacted connection preview instead of echoing the submitted credential
|
|
231
|
+
fields back in the response (leak hardening). Non-breaking — the frontend
|
|
232
|
+
refetches the redacted list and ignores the returned preview.
|
|
233
|
+
|
|
234
|
+
NOTE: integration connection-credential STORAGE is intentionally NOT
|
|
235
|
+
migrated onto the secrets platform. Connection creds are co-mingled
|
|
236
|
+
secret/non-secret config stored per-provider via `ConfigService` (which
|
|
237
|
+
already uses the same AES-GCM crypto + per-field redaction); splitting them
|
|
238
|
+
out would require per-provider schema-walking and a lossy migration across
|
|
239
|
+
live integrations for no real gain. The `ConnectionStore` API + storage are
|
|
240
|
+
unchanged.
|
|
241
|
+
|
|
242
|
+
### Patch Changes
|
|
243
|
+
|
|
244
|
+
- b995afb: Harden the script-package blob byte boundary against a raw `ArrayBuffer`.
|
|
245
|
+
|
|
246
|
+
Blob bytes can reach the content-hash and storage codecs as a raw `ArrayBuffer` (e.g. from an S3/HTTP transport's `arrayBuffer()`), and Node/Bun's `crypto.Hash.update()` rejects a bare `ArrayBuffer` ("The 'data' argument must be of type string or an instance of Buffer, TypedArray, or DataView. Received an instance of ArrayBuffer"), which would fail a real-package install with `status=error`. `blobSha256` / `verifyBlobSha256` (script-packages-backend) and `encodeBlob` (script-packages-store-postgres) now normalize `ArrayBuffer` to a `Uint8Array` view at the boundary before hashing/encoding. A view over the same bytes hashes and encodes identically, so existing content hashes and stored blobs are unchanged. Adds regression tests feeding an `ArrayBuffer` through both the hash and the Postgres codec.
|
|
247
|
+
|
|
248
|
+
- Updated dependencies [270ef29]
|
|
249
|
+
- Updated dependencies [270ef29]
|
|
250
|
+
- Updated dependencies [270ef29]
|
|
251
|
+
- Updated dependencies [b995afb]
|
|
252
|
+
- Updated dependencies [270ef29]
|
|
253
|
+
- Updated dependencies [270ef29]
|
|
254
|
+
- Updated dependencies [270ef29]
|
|
255
|
+
- Updated dependencies [270ef29]
|
|
256
|
+
- Updated dependencies [270ef29]
|
|
257
|
+
- Updated dependencies [b995afb]
|
|
258
|
+
- Updated dependencies [270ef29]
|
|
259
|
+
- Updated dependencies [b995afb]
|
|
260
|
+
- Updated dependencies [270ef29]
|
|
261
|
+
- Updated dependencies [270ef29]
|
|
262
|
+
- Updated dependencies [b995afb]
|
|
263
|
+
- Updated dependencies [270ef29]
|
|
264
|
+
- Updated dependencies [270ef29]
|
|
265
|
+
- Updated dependencies [270ef29]
|
|
266
|
+
- Updated dependencies [270ef29]
|
|
267
|
+
- Updated dependencies [270ef29]
|
|
268
|
+
- Updated dependencies [270ef29]
|
|
269
|
+
- Updated dependencies [b995afb]
|
|
270
|
+
- @checkstack/backend-api@0.19.0
|
|
271
|
+
- @checkstack/secrets-backend@0.1.0
|
|
272
|
+
- @checkstack/script-packages-common@0.2.0
|
|
273
|
+
- @checkstack/secrets-common@0.1.0
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
CREATE TABLE "script_package_blob" (
|
|
2
|
+
"integrity" text PRIMARY KEY NOT NULL,
|
|
3
|
+
"name" text NOT NULL,
|
|
4
|
+
"version" text NOT NULL,
|
|
5
|
+
"backend" text NOT NULL,
|
|
6
|
+
"size_bytes" bigint NOT NULL,
|
|
7
|
+
"created_at" timestamp DEFAULT now() NOT NULL
|
|
8
|
+
);
|
|
9
|
+
--> statement-breakpoint
|
|
10
|
+
CREATE TABLE "script_package_install_state" (
|
|
11
|
+
"id" text PRIMARY KEY DEFAULT 'singleton' NOT NULL,
|
|
12
|
+
"status" text DEFAULT 'idle' NOT NULL,
|
|
13
|
+
"lockfile_hash" text,
|
|
14
|
+
"manifest" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
|
15
|
+
"total_size_bytes" bigint DEFAULT 0 NOT NULL,
|
|
16
|
+
"last_installed_at" timestamp,
|
|
17
|
+
"error_message" text
|
|
18
|
+
);
|
|
19
|
+
--> statement-breakpoint
|
|
20
|
+
CREATE TABLE "script_package_registry_config" (
|
|
21
|
+
"id" text PRIMARY KEY DEFAULT 'singleton' NOT NULL,
|
|
22
|
+
"registry_url" text DEFAULT 'https://registry.npmjs.org/' NOT NULL,
|
|
23
|
+
"scoped_registries" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
|
24
|
+
"auth_secret_ref" text,
|
|
25
|
+
"ignore_scripts" boolean DEFAULT true NOT NULL,
|
|
26
|
+
"updated_at" timestamp DEFAULT now() NOT NULL
|
|
27
|
+
);
|
|
28
|
+
--> statement-breakpoint
|
|
29
|
+
CREATE TABLE "script_package_satellite_state" (
|
|
30
|
+
"satellite_id" text PRIMARY KEY NOT NULL,
|
|
31
|
+
"lockfile_hash" text,
|
|
32
|
+
"status" text DEFAULT 'pending' NOT NULL,
|
|
33
|
+
"error_message" text,
|
|
34
|
+
"synced_at" timestamp
|
|
35
|
+
);
|
|
36
|
+
--> statement-breakpoint
|
|
37
|
+
CREATE TABLE "script_package_size_cap" (
|
|
38
|
+
"id" text PRIMARY KEY DEFAULT 'singleton' NOT NULL,
|
|
39
|
+
"warn_bytes" bigint DEFAULT 157286400 NOT NULL,
|
|
40
|
+
"block_bytes" bigint DEFAULT 314572800 NOT NULL,
|
|
41
|
+
"updated_at" timestamp DEFAULT now() NOT NULL
|
|
42
|
+
);
|
|
43
|
+
--> statement-breakpoint
|
|
44
|
+
CREATE TABLE "script_package_storage_config" (
|
|
45
|
+
"id" text PRIMARY KEY DEFAULT 'singleton' NOT NULL,
|
|
46
|
+
"active_backend" text DEFAULT 'postgres' NOT NULL,
|
|
47
|
+
"migration_status" text DEFAULT 'idle' NOT NULL,
|
|
48
|
+
"migration_target" text,
|
|
49
|
+
"migrated_count" integer DEFAULT 0 NOT NULL,
|
|
50
|
+
"migration_error" text,
|
|
51
|
+
"updated_at" timestamp DEFAULT now() NOT NULL
|
|
52
|
+
);
|
|
53
|
+
--> statement-breakpoint
|
|
54
|
+
CREATE TABLE "script_packages" (
|
|
55
|
+
"name" text PRIMARY KEY NOT NULL,
|
|
56
|
+
"version" text NOT NULL,
|
|
57
|
+
"enabled" boolean DEFAULT true NOT NULL,
|
|
58
|
+
"added_by" text,
|
|
59
|
+
"added_at" timestamp DEFAULT now() NOT NULL,
|
|
60
|
+
"updated_at" timestamp DEFAULT now() NOT NULL
|
|
61
|
+
);
|
|
62
|
+
--> statement-breakpoint
|
|
63
|
+
CREATE INDEX "script_package_blob_backend_idx" ON "script_package_blob" USING btree ("backend");
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
CREATE TABLE "script_package_blob_gc_state" (
|
|
2
|
+
"id" text PRIMARY KEY DEFAULT 'singleton' NOT NULL,
|
|
3
|
+
"last_run_at" timestamp,
|
|
4
|
+
"last_deleted" integer DEFAULT 0 NOT NULL,
|
|
5
|
+
"last_bytes_reclaimed" bigint DEFAULT 0 NOT NULL,
|
|
6
|
+
"total_bytes_reclaimed" bigint DEFAULT 0 NOT NULL
|
|
7
|
+
);
|
|
8
|
+
--> statement-breakpoint
|
|
9
|
+
CREATE TABLE "script_package_lockfile_history" (
|
|
10
|
+
"lockfile_hash" text PRIMARY KEY NOT NULL,
|
|
11
|
+
"manifest" jsonb DEFAULT '[]'::jsonb NOT NULL,
|
|
12
|
+
"recorded_at" timestamp DEFAULT now() NOT NULL
|
|
13
|
+
);
|
|
14
|
+
--> statement-breakpoint
|
|
15
|
+
CREATE INDEX "script_package_lockfile_history_recorded_idx" ON "script_package_lockfile_history" USING btree ("recorded_at");
|