@opengeni/runtime 2.5.2 → 2.5.3-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-Q2VNH5FV.js → agent-RJQLKFIW.js} +2 -2
- package/dist/{chunk-LGLJ5PVD.js → chunk-46BFUGFL.js} +2622 -1368
- package/dist/chunk-46BFUGFL.js.map +1 -0
- package/dist/{chunk-CD22I76N.js → chunk-FHGLYLBL.js} +2 -2
- package/dist/{chunk-25B3EQRW.js → chunk-UNICCRW5.js} +4 -4
- package/dist/{chunk-GJC7H2FC.js → chunk-VUEWS4GI.js} +494 -95
- package/dist/chunk-VUEWS4GI.js.map +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +22 -4
- package/dist/{localSources-Z5IJIIRV.js → localSources-PVFCAPN2.js} +3 -3
- package/dist/mcp-operation-dispatch.d.ts +45 -0
- package/dist/mcp-operation-observation.d.ts +137 -0
- package/dist/operational-instructions.d.ts +1 -1
- package/dist/sandbox/archive-spool.d.ts +14 -0
- package/dist/sandbox/host-archive-spool.d.ts +19 -0
- package/dist/sandbox/index.d.ts +5 -1
- package/dist/sandbox/index.js +7 -3
- package/dist/sandbox/routing/backend-resolver.d.ts +1 -0
- package/dist/sandbox/selfhosted/file-transfer.d.ts +20 -0
- package/dist/sandbox/selfhosted/session.d.ts +7 -0
- package/dist/sandbox/workspace-archive.d.ts +7 -0
- package/dist/workspace-skills.d.ts +4 -0
- package/dist/workspace-tool-gateway.js +4 -4
- package/package.json +11 -11
- package/src/bundled_default_skills/document-parsing/SKILL.md +36 -0
- package/src/bundled_site_skills/opengeni-sites/SKILL.md +1 -1
- package/src/index.ts +231 -45
- package/src/lazy-tool-transport.ts +2 -0
- package/src/mcp-operation-dispatch.ts +144 -0
- package/src/mcp-operation-observation.ts +148 -0
- package/src/operational-instructions.ts +1 -1
- package/src/runtime-skills.ts +7 -2
- package/src/sandbox/archive-spool.ts +16 -0
- package/src/sandbox/host-archive-spool.ts +1040 -0
- package/src/sandbox/index.ts +292 -203
- package/src/sandbox/routing/backend-resolver.ts +2 -0
- package/src/sandbox/selfhosted/control-rpc.ts +22 -0
- package/src/sandbox/selfhosted/fault-rendering.ts +9 -0
- package/src/sandbox/selfhosted/file-transfer.ts +160 -0
- package/src/sandbox/selfhosted/op-testing.ts +3 -0
- package/src/sandbox/selfhosted/session.ts +194 -5
- package/src/sandbox/workspace-archive.ts +54 -1
- package/src/workspace-skills.ts +171 -31
- package/dist/chunk-GJC7H2FC.js.map +0 -1
- package/dist/chunk-LGLJ5PVD.js.map +0 -1
- /package/dist/{agent-Q2VNH5FV.js.map → agent-RJQLKFIW.js.map} +0 -0
- /package/dist/{chunk-CD22I76N.js.map → chunk-FHGLYLBL.js.map} +0 -0
- /package/dist/{chunk-25B3EQRW.js.map → chunk-UNICCRW5.js.map} +0 -0
- /package/dist/{localSources-Z5IJIIRV.js.map → localSources-PVFCAPN2.js.map} +0 -0
package/src/sandbox/index.ts
CHANGED
|
@@ -24,6 +24,15 @@ export type {
|
|
|
24
24
|
|
|
25
25
|
import type { Settings } from "@opengeni/config";
|
|
26
26
|
import { collectSandboxEnvironment, parseExposedPorts } from "@opengeni/config";
|
|
27
|
+
export type { WorkspaceArchiveSpool, VerifiedHostWorkspaceArchive } from "./archive-spool";
|
|
28
|
+
import type { WorkspaceArchiveSpool } from "./archive-spool";
|
|
29
|
+
import { restoreHostWorkspaceArchive } from "./host-archive-spool";
|
|
30
|
+
export {
|
|
31
|
+
captureWorkspaceArchiveForStorage,
|
|
32
|
+
disposeWorkspaceArchive,
|
|
33
|
+
type VerifiedWorkspaceArchivePayload,
|
|
34
|
+
} from "./workspace-archive";
|
|
35
|
+
|
|
27
36
|
import {
|
|
28
37
|
BROWSER_CONTROL_PORT,
|
|
29
38
|
DESKTOP_STREAM_PORT,
|
|
@@ -31,6 +40,9 @@ import {
|
|
|
31
40
|
TERMINAL_STREAM_PORT,
|
|
32
41
|
omitInlineWorkspaceArchiveWhenObjectRefPresent,
|
|
33
42
|
parseWorkspaceArchiveObjectRef,
|
|
43
|
+
parseWorkspaceArchiveDescriptor,
|
|
44
|
+
type WorkspaceArchiveObjectRef,
|
|
45
|
+
type TarWorkspaceArchiveDescriptor,
|
|
34
46
|
type SandboxBackend,
|
|
35
47
|
type SandboxProviderContinuityRecovery,
|
|
36
48
|
} from "@opengeni/contracts";
|
|
@@ -1676,9 +1688,16 @@ async function terminateCreatedSandbox(
|
|
|
1676
1688
|
}
|
|
1677
1689
|
|
|
1678
1690
|
type ModalNativeWorkspacePersistence = "snapshot_filesystem" | "snapshot_directory";
|
|
1691
|
+
type RestoreWorkspaceArchive =
|
|
1692
|
+
| VerifiedWorkspaceArchive
|
|
1693
|
+
| {
|
|
1694
|
+
kind: "host_spool";
|
|
1695
|
+
descriptor: TarWorkspaceArchiveDescriptor;
|
|
1696
|
+
ref: WorkspaceArchiveObjectRef;
|
|
1697
|
+
};
|
|
1679
1698
|
|
|
1680
1699
|
function modalWorkspacePersistenceForRestore(
|
|
1681
|
-
archive:
|
|
1700
|
+
archive: RestoreWorkspaceArchive | null,
|
|
1682
1701
|
): ModalNativeWorkspacePersistence | null {
|
|
1683
1702
|
if (archive?.kind !== "provider_snapshot") return null;
|
|
1684
1703
|
const snapshot = archive.nativeSnapshot;
|
|
@@ -1707,7 +1726,7 @@ function modalWorkspacePersistenceForRestore(
|
|
|
1707
1726
|
function settingsForWorkspaceArchiveRestore(
|
|
1708
1727
|
backend: SandboxBackend,
|
|
1709
1728
|
settings: Settings,
|
|
1710
|
-
archive:
|
|
1729
|
+
archive: RestoreWorkspaceArchive | null,
|
|
1711
1730
|
): Settings {
|
|
1712
1731
|
if (backend !== "modal") return settings;
|
|
1713
1732
|
const workspacePersistence = modalWorkspacePersistenceForRestore(archive);
|
|
@@ -1735,6 +1754,7 @@ export async function establishSandboxSessionFromEnvelope(
|
|
|
1735
1754
|
opts: {
|
|
1736
1755
|
sessionId: string;
|
|
1737
1756
|
recovery: "create-or-restore" | "resume-only";
|
|
1757
|
+
loadHostWorkspaceArchive?: (ref: WorkspaceArchiveObjectRef) => Promise<WorkspaceArchiveSpool>;
|
|
1738
1758
|
backendOverride?: SandboxBackend;
|
|
1739
1759
|
environment?: Record<string, string>;
|
|
1740
1760
|
onSandboxCreated?: SandboxCreatedCallback;
|
|
@@ -1817,6 +1837,7 @@ export async function establishSandboxSessionFromEnvelope(
|
|
|
1817
1837
|
envelopeSessionState && typeof envelopeSessionState === "object"
|
|
1818
1838
|
? (envelopeSessionState as {
|
|
1819
1839
|
workspaceArchive?: unknown;
|
|
1840
|
+
workspaceArchiveRef?: unknown;
|
|
1820
1841
|
workspaceArchiveMeta?: unknown;
|
|
1821
1842
|
})
|
|
1822
1843
|
: undefined;
|
|
@@ -1838,220 +1859,217 @@ export async function establishSandboxSessionFromEnvelope(
|
|
|
1838
1859
|
// Parse/verify lazily: a warm resume-by-id does not consume its retained
|
|
1839
1860
|
// archive, so a legacy live box remains resumable. Creation/restoration is
|
|
1840
1861
|
// the boundary that requires complete durable metadata.
|
|
1841
|
-
|
|
1862
|
+
let workspaceArchive: RestoreWorkspaceArchive | null = readVerifiedWorkspaceArchive(
|
|
1842
1863
|
workspaceArchiveBase64,
|
|
1843
1864
|
workspaceArchiveMetadata,
|
|
1844
1865
|
);
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
: createSandboxClientForBackend(
|
|
1855
|
-
backend,
|
|
1856
|
-
restoreSettings,
|
|
1857
|
-
environment,
|
|
1858
|
-
opts.metrics,
|
|
1859
|
-
)) as ResumeCapableClient | undefined);
|
|
1860
|
-
if (!restoreClient?.create) {
|
|
1861
|
-
throw new SandboxConfigError(
|
|
1862
|
-
backend,
|
|
1863
|
-
`Sandbox backend "${backend}" does not support fresh archive restoration`,
|
|
1864
|
-
);
|
|
1865
|
-
}
|
|
1866
|
-
let createdClient = restoreClient;
|
|
1867
|
-
const createStarted = Date.now();
|
|
1868
|
-
let restored: Awaited<ReturnType<NonNullable<typeof restoreClient.create>>>;
|
|
1869
|
-
try {
|
|
1870
|
-
restored = await restoreClient.create({ manifest: createManifest });
|
|
1871
|
-
recordSandboxCreateMetric(
|
|
1872
|
-
opts.metrics,
|
|
1873
|
-
restoreClient.backendId,
|
|
1874
|
-
createImageSource,
|
|
1875
|
-
"completed",
|
|
1876
|
-
createStarted,
|
|
1877
|
-
);
|
|
1878
|
-
} catch (error) {
|
|
1879
|
-
recordSandboxCreateMetric(
|
|
1880
|
-
opts.metrics,
|
|
1881
|
-
restoreClient.backendId,
|
|
1882
|
-
createImageSource,
|
|
1883
|
-
"failed",
|
|
1884
|
-
createStarted,
|
|
1885
|
-
);
|
|
1886
|
-
recordSandboxProviderApiThrottleMetric(
|
|
1887
|
-
opts.metrics,
|
|
1888
|
-
restoreClient.backendId,
|
|
1889
|
-
"create",
|
|
1890
|
-
error,
|
|
1891
|
-
);
|
|
1866
|
+
const objectRef = parseWorkspaceArchiveObjectRef(archiveState?.workspaceArchiveRef);
|
|
1867
|
+
const objectDescriptor = parseWorkspaceArchiveDescriptor(workspaceArchiveMetadata);
|
|
1868
|
+
if (
|
|
1869
|
+
process.platform === "linux" &&
|
|
1870
|
+
objectRef &&
|
|
1871
|
+
objectDescriptor?.version === 1 &&
|
|
1872
|
+
objectDescriptor.workspace.projection === "sdk_local_archive_v1" &&
|
|
1873
|
+
opts.loadHostWorkspaceArchive
|
|
1874
|
+
) {
|
|
1892
1875
|
if (
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1876
|
+
(backend !== "local" && backend !== "docker") ||
|
|
1877
|
+
objectRef.sha256 !== objectDescriptor.archiveSha256 ||
|
|
1878
|
+
objectRef.bytes !== objectDescriptor.archiveBytes
|
|
1896
1879
|
) {
|
|
1897
|
-
throw
|
|
1880
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1881
|
+
"archive_metadata_invalid",
|
|
1882
|
+
"Host archive object reference does not match its provider and descriptor",
|
|
1883
|
+
);
|
|
1898
1884
|
}
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1885
|
+
workspaceArchive = { kind: "host_spool", descriptor: objectDescriptor, ref: objectRef };
|
|
1886
|
+
}
|
|
1887
|
+
if (objectRef && !workspaceArchive) {
|
|
1888
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1889
|
+
"archive_hydration_failed",
|
|
1890
|
+
"Selected workspace archive object has no supported restore loader",
|
|
1891
|
+
);
|
|
1892
|
+
}
|
|
1893
|
+
let hostSpool: WorkspaceArchiveSpool | undefined;
|
|
1894
|
+
try {
|
|
1895
|
+
// Preserve the pre-create integrity boundary used by inline archives.
|
|
1896
|
+
// A missing/corrupt object must not create a provider just to delete it.
|
|
1897
|
+
if (workspaceArchive?.kind === "host_spool") {
|
|
1898
|
+
hostSpool = await opts.loadHostWorkspaceArchive!(workspaceArchive.ref);
|
|
1899
|
+
if (
|
|
1900
|
+
hostSpool.sha256 !== workspaceArchive.descriptor.archiveSha256 ||
|
|
1901
|
+
hostSpool.byteSize !== workspaceArchive.descriptor.archiveBytes
|
|
1902
|
+
) {
|
|
1903
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1904
|
+
"archive_hash_mismatch",
|
|
1905
|
+
"Downloaded workspace archive does not match its descriptor",
|
|
1906
|
+
);
|
|
1907
|
+
}
|
|
1907
1908
|
}
|
|
1908
|
-
|
|
1909
|
+
// The selected native artifact is the durable authority for its restore
|
|
1910
|
+
// protocol. The process setting governs new sandboxes only: a mode rollout
|
|
1911
|
+
// must not make an older, verified Modal snapshot impossible to hydrate.
|
|
1912
|
+
const restoreSettings = settingsForWorkspaceArchiveRestore(
|
|
1909
1913
|
backend,
|
|
1910
|
-
|
|
1914
|
+
settings,
|
|
1911
1915
|
workspaceArchive,
|
|
1912
1916
|
);
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1917
|
+
const restoreClient =
|
|
1918
|
+
restoreSettings === settings
|
|
1919
|
+
? client
|
|
1920
|
+
: ((opts.clientFactory
|
|
1921
|
+
? opts.clientFactory(backend, restoreSettings, environment)
|
|
1922
|
+
: createSandboxClientForBackend(
|
|
1923
|
+
backend,
|
|
1924
|
+
restoreSettings,
|
|
1925
|
+
environment,
|
|
1926
|
+
opts.metrics,
|
|
1927
|
+
)) as ResumeCapableClient | undefined);
|
|
1928
|
+
if (!restoreClient?.create) {
|
|
1920
1929
|
throw new SandboxConfigError(
|
|
1921
1930
|
backend,
|
|
1922
|
-
|
|
1931
|
+
`Sandbox backend "${backend}" does not support fresh archive restoration`,
|
|
1923
1932
|
);
|
|
1924
1933
|
}
|
|
1925
|
-
|
|
1934
|
+
let createdClient = restoreClient;
|
|
1935
|
+
const createStarted = Date.now();
|
|
1936
|
+
let restored: Awaited<ReturnType<NonNullable<typeof restoreClient.create>>>;
|
|
1926
1937
|
try {
|
|
1927
|
-
restored = await
|
|
1938
|
+
restored = await restoreClient.create({ manifest: createManifest });
|
|
1928
1939
|
recordSandboxCreateMetric(
|
|
1929
1940
|
opts.metrics,
|
|
1930
|
-
|
|
1931
|
-
|
|
1941
|
+
restoreClient.backendId,
|
|
1942
|
+
createImageSource,
|
|
1932
1943
|
"completed",
|
|
1933
|
-
|
|
1944
|
+
createStarted,
|
|
1934
1945
|
);
|
|
1935
|
-
|
|
1936
|
-
} catch (fallbackError) {
|
|
1946
|
+
} catch (error) {
|
|
1937
1947
|
recordSandboxCreateMetric(
|
|
1938
1948
|
opts.metrics,
|
|
1939
|
-
|
|
1940
|
-
|
|
1949
|
+
restoreClient.backendId,
|
|
1950
|
+
createImageSource,
|
|
1941
1951
|
"failed",
|
|
1942
|
-
|
|
1952
|
+
createStarted,
|
|
1943
1953
|
);
|
|
1944
1954
|
recordSandboxProviderApiThrottleMetric(
|
|
1945
1955
|
opts.metrics,
|
|
1946
|
-
|
|
1956
|
+
restoreClient.backendId,
|
|
1947
1957
|
"create",
|
|
1948
|
-
|
|
1958
|
+
error,
|
|
1949
1959
|
);
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
try {
|
|
1971
|
-
await opts.onSandboxCreated(established);
|
|
1972
|
-
} catch (createCallbackError) {
|
|
1973
|
-
await terminateCreatedSandbox(createdClient, restored, restoredState);
|
|
1974
|
-
throw createCallbackError;
|
|
1975
|
-
}
|
|
1976
|
-
}
|
|
1977
|
-
let hydrationApplied = false;
|
|
1978
|
-
if (workspaceArchive) {
|
|
1979
|
-
const hydrate = (restored as { hydrateWorkspace?: (data: Uint8Array) => Promise<void> })
|
|
1980
|
-
.hydrateWorkspace;
|
|
1981
|
-
if (typeof hydrate !== "function") {
|
|
1982
|
-
await terminateCreatedSandbox(createdClient, restored, restoredState);
|
|
1983
|
-
throw new WorkspaceArchiveIntegrityError(
|
|
1984
|
-
"archive_hydration_failed",
|
|
1985
|
-
`sandbox backend ${createdClient.backendId} cannot hydrate selected archive revision ${workspaceArchive.descriptor.revision}`,
|
|
1986
|
-
);
|
|
1987
|
-
}
|
|
1988
|
-
try {
|
|
1989
|
-
// hydrateWorkspace may internally replace the underlying box.
|
|
1990
|
-
await hydrate.call(restored, workspaceArchive.bytes);
|
|
1991
|
-
} catch (error) {
|
|
1992
|
-
await terminateCreatedSandbox(
|
|
1993
|
-
createdClient,
|
|
1994
|
-
restored,
|
|
1995
|
-
(restored as { state?: unknown }).state,
|
|
1996
|
-
);
|
|
1997
|
-
if (error instanceof WorkspaceArchiveIntegrityError) throw error;
|
|
1998
|
-
throw new WorkspaceArchiveIntegrityError(
|
|
1999
|
-
"archive_hydration_failed",
|
|
2000
|
-
`failed to hydrate selected workspace archive revision ${workspaceArchive.descriptor.revision}`,
|
|
2001
|
-
{ retryable: true, cause: error },
|
|
1960
|
+
if (
|
|
1961
|
+
createImageSource !== "provider_immutable" ||
|
|
1962
|
+
backend !== "modal" ||
|
|
1963
|
+
!isProviderSandboxNotFoundError(restoreClient.backendId, error)
|
|
1964
|
+
) {
|
|
1965
|
+
throw error;
|
|
1966
|
+
}
|
|
1967
|
+
const fallbackBaseSettings =
|
|
1968
|
+
opts.logicalFallbackSettings ??
|
|
1969
|
+
(settings.modalImageRef ? { ...settings, modalImageId: undefined } : null);
|
|
1970
|
+
if (!fallbackBaseSettings) {
|
|
1971
|
+
// An ID-only logical base is supported. Without the exact pre-selection
|
|
1972
|
+
// settings, clearing the optimized ID would silently boot Modal's
|
|
1973
|
+
// default image, so fail closed instead of changing the rig base.
|
|
1974
|
+
throw error;
|
|
1975
|
+
}
|
|
1976
|
+
const fallbackSettings = settingsForWorkspaceArchiveRestore(
|
|
1977
|
+
backend,
|
|
1978
|
+
fallbackBaseSettings,
|
|
1979
|
+
workspaceArchive,
|
|
2002
1980
|
);
|
|
1981
|
+
await ensureModalRegistryImage(fallbackSettings);
|
|
1982
|
+
const fallbackClient = (
|
|
1983
|
+
opts.clientFactory
|
|
1984
|
+
? opts.clientFactory(backend, fallbackSettings, environment)
|
|
1985
|
+
: createSandboxClientForBackend(backend, fallbackSettings, environment, opts.metrics)
|
|
1986
|
+
) as ResumeCapableClient | undefined;
|
|
1987
|
+
if (!fallbackClient?.create) {
|
|
1988
|
+
throw new SandboxConfigError(
|
|
1989
|
+
backend,
|
|
1990
|
+
"Modal logical-image fallback does not support fresh creation",
|
|
1991
|
+
);
|
|
1992
|
+
}
|
|
1993
|
+
const fallbackStarted = Date.now();
|
|
1994
|
+
try {
|
|
1995
|
+
restored = await fallbackClient.create({ manifest: createManifest });
|
|
1996
|
+
recordSandboxCreateMetric(
|
|
1997
|
+
opts.metrics,
|
|
1998
|
+
fallbackClient.backendId,
|
|
1999
|
+
"logical",
|
|
2000
|
+
"completed",
|
|
2001
|
+
fallbackStarted,
|
|
2002
|
+
);
|
|
2003
|
+
createdClient = fallbackClient;
|
|
2004
|
+
} catch (fallbackError) {
|
|
2005
|
+
recordSandboxCreateMetric(
|
|
2006
|
+
opts.metrics,
|
|
2007
|
+
fallbackClient.backendId,
|
|
2008
|
+
"logical",
|
|
2009
|
+
"failed",
|
|
2010
|
+
fallbackStarted,
|
|
2011
|
+
);
|
|
2012
|
+
recordSandboxProviderApiThrottleMetric(
|
|
2013
|
+
opts.metrics,
|
|
2014
|
+
fallbackClient.backendId,
|
|
2015
|
+
"create",
|
|
2016
|
+
fallbackError,
|
|
2017
|
+
);
|
|
2018
|
+
throw fallbackError;
|
|
2019
|
+
}
|
|
2003
2020
|
}
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
// boundary; if it cannot persist the replacement, the caller fails closed
|
|
2009
|
-
// and this exact replacement is terminated below.
|
|
2010
|
-
const hydratedState = (restored as { state?: unknown }).state;
|
|
2011
|
-
const hydratedInstanceId = readInstanceId(backend, restored);
|
|
2012
|
-
if (!hydratedInstanceId) {
|
|
2013
|
-
await terminateCreatedSandbox(createdClient, restored, hydratedState);
|
|
2021
|
+
let restoredState = (restored as { state?: unknown }).state;
|
|
2022
|
+
const restoredInstanceId = readInstanceId(backend, restored);
|
|
2023
|
+
if (!restoredInstanceId) {
|
|
2024
|
+
await terminateCreatedSandbox(createdClient, restored, restoredState);
|
|
2014
2025
|
throw new SandboxConfigError(
|
|
2015
2026
|
backend,
|
|
2016
|
-
`Sandbox backend "${backend}"
|
|
2027
|
+
`Sandbox backend "${backend}" created a handle without its declared provider identity`,
|
|
2017
2028
|
);
|
|
2018
2029
|
}
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
throw createCallbackError;
|
|
2033
|
-
}
|
|
2030
|
+
let established: EstablishedSandboxSession = {
|
|
2031
|
+
client: createdClient,
|
|
2032
|
+
session: restored,
|
|
2033
|
+
sessionState: restoredState ?? resumeFallbackState,
|
|
2034
|
+
instanceId: restoredInstanceId,
|
|
2035
|
+
backendId: createdClient.backendId,
|
|
2036
|
+
};
|
|
2037
|
+
if (opts.onSandboxCreated) {
|
|
2038
|
+
try {
|
|
2039
|
+
await opts.onSandboxCreated(established);
|
|
2040
|
+
} catch (createCallbackError) {
|
|
2041
|
+
await terminateCreatedSandbox(createdClient, restored, restoredState);
|
|
2042
|
+
throw createCallbackError;
|
|
2034
2043
|
}
|
|
2035
2044
|
}
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2045
|
+
let hydrationApplied = false;
|
|
2046
|
+
if (workspaceArchive) {
|
|
2047
|
+
const hydrate = (restored as { hydrateWorkspace?: (data: Uint8Array) => Promise<void> })
|
|
2048
|
+
.hydrateWorkspace;
|
|
2049
|
+
if (workspaceArchive.kind !== "host_spool" && typeof hydrate !== "function") {
|
|
2050
|
+
await terminateCreatedSandbox(createdClient, restored, restoredState);
|
|
2051
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2052
|
+
"archive_hydration_failed",
|
|
2053
|
+
`sandbox backend ${createdClient.backendId} cannot hydrate selected archive revision ${workspaceArchive.descriptor.revision}`,
|
|
2044
2054
|
);
|
|
2045
|
-
throw error;
|
|
2046
2055
|
}
|
|
2047
|
-
}
|
|
2048
|
-
// Native provider snapshots are restored by their exact opaque receipt.
|
|
2049
|
-
// OpenGeni verifies receipt identity/hash before hydration, but it must not
|
|
2050
|
-
// impose a tar/inode equivalence contract on the provider's filesystem
|
|
2051
|
-
// image. Tar archives remain content-verified after hydration.
|
|
2052
|
-
if (workspaceArchive.kind === "tar") {
|
|
2053
2056
|
try {
|
|
2054
|
-
|
|
2057
|
+
// hydrateWorkspace may internally replace the underlying box.
|
|
2058
|
+
if (workspaceArchive.kind === "host_spool") {
|
|
2059
|
+
// This is the exact newly-created, unpublished destination. The lease
|
|
2060
|
+
// restore fence must exclude admitted writers until verification ends;
|
|
2061
|
+
// descriptor-relative filesystem access alone is not writer isolation.
|
|
2062
|
+
const root = (restored as { state?: { workspaceRootPath?: unknown } }).state
|
|
2063
|
+
?.workspaceRootPath;
|
|
2064
|
+
if (typeof root !== "string")
|
|
2065
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2066
|
+
"archive_hydration_failed",
|
|
2067
|
+
"Host archive restore has no exact workspace root",
|
|
2068
|
+
);
|
|
2069
|
+
await restoreHostWorkspaceArchive(root, hostSpool!, {
|
|
2070
|
+
archiveLimits: Reflect.get(restored as object, "archiveLimits"),
|
|
2071
|
+
});
|
|
2072
|
+
} else await hydrate!.call(restored, workspaceArchive.bytes);
|
|
2055
2073
|
} catch (error) {
|
|
2056
2074
|
await terminateCreatedSandbox(
|
|
2057
2075
|
createdClient,
|
|
@@ -2060,35 +2078,106 @@ export async function establishSandboxSessionFromEnvelope(
|
|
|
2060
2078
|
);
|
|
2061
2079
|
if (error instanceof WorkspaceArchiveIntegrityError) throw error;
|
|
2062
2080
|
throw new WorkspaceArchiveIntegrityError(
|
|
2063
|
-
"
|
|
2064
|
-
`failed to
|
|
2065
|
-
{ retryable: true },
|
|
2081
|
+
"archive_hydration_failed",
|
|
2082
|
+
`failed to hydrate selected workspace archive revision ${workspaceArchive.descriptor.revision}`,
|
|
2083
|
+
{ retryable: true, cause: error },
|
|
2066
2084
|
);
|
|
2067
2085
|
}
|
|
2086
|
+
// hydrateWorkspace may replace the provider box (Modal's native snapshot
|
|
2087
|
+
// restore does this). Attribute the newly-active identity immediately,
|
|
2088
|
+
// before restore-state marking, fingerprint verification, or any caller
|
|
2089
|
+
// can publish the box warm. The callback is the durable lease/tagging
|
|
2090
|
+
// boundary; if it cannot persist the replacement, the caller fails closed
|
|
2091
|
+
// and this exact replacement is terminated below.
|
|
2092
|
+
const hydratedState = (restored as { state?: unknown }).state;
|
|
2093
|
+
const hydratedInstanceId = readInstanceId(backend, restored);
|
|
2094
|
+
if (!hydratedInstanceId) {
|
|
2095
|
+
await terminateCreatedSandbox(createdClient, restored, hydratedState);
|
|
2096
|
+
throw new SandboxConfigError(
|
|
2097
|
+
backend,
|
|
2098
|
+
`Sandbox backend "${backend}" hydrated a handle without its declared provider identity`,
|
|
2099
|
+
);
|
|
2100
|
+
}
|
|
2101
|
+
if (hydratedInstanceId !== established.instanceId) {
|
|
2102
|
+
established = {
|
|
2103
|
+
client: createdClient,
|
|
2104
|
+
session: restored,
|
|
2105
|
+
sessionState: hydratedState ?? resumeFallbackState,
|
|
2106
|
+
instanceId: hydratedInstanceId,
|
|
2107
|
+
backendId: createdClient.backendId,
|
|
2108
|
+
};
|
|
2109
|
+
if (opts.onSandboxCreated) {
|
|
2110
|
+
try {
|
|
2111
|
+
await opts.onSandboxCreated(established);
|
|
2112
|
+
} catch (createCallbackError) {
|
|
2113
|
+
await terminateCreatedSandbox(createdClient, restored, hydratedState);
|
|
2114
|
+
throw createCallbackError;
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
}
|
|
2118
|
+
if (opts.onWorkspaceRestoreVerifying) {
|
|
2119
|
+
try {
|
|
2120
|
+
await opts.onWorkspaceRestoreVerifying(workspaceArchive.descriptor);
|
|
2121
|
+
} catch (error) {
|
|
2122
|
+
await terminateCreatedSandbox(
|
|
2123
|
+
createdClient,
|
|
2124
|
+
restored,
|
|
2125
|
+
(restored as { state?: unknown }).state,
|
|
2126
|
+
);
|
|
2127
|
+
throw error;
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
// Native provider snapshots are restored by their exact opaque receipt.
|
|
2131
|
+
// OpenGeni verifies receipt identity/hash before hydration, but it must not
|
|
2132
|
+
// impose a tar/inode equivalence contract on the provider's filesystem
|
|
2133
|
+
// image. Tar archives remain content-verified after hydration.
|
|
2134
|
+
if (workspaceArchive.kind === "tar" || workspaceArchive.kind === "host_spool") {
|
|
2135
|
+
try {
|
|
2136
|
+
await verifyRestoredWorkspace(restored, workspaceArchive.descriptor);
|
|
2137
|
+
} catch (error) {
|
|
2138
|
+
await terminateCreatedSandbox(
|
|
2139
|
+
createdClient,
|
|
2140
|
+
restored,
|
|
2141
|
+
(restored as { state?: unknown }).state,
|
|
2142
|
+
);
|
|
2143
|
+
if (error instanceof WorkspaceArchiveIntegrityError) throw error;
|
|
2144
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2145
|
+
"workspace_fingerprint_unavailable",
|
|
2146
|
+
`failed to verify selected workspace archive revision ${workspaceArchive.descriptor.revision}`,
|
|
2147
|
+
{ retryable: true },
|
|
2148
|
+
);
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
hydrationApplied = true;
|
|
2152
|
+
console.info(
|
|
2153
|
+
`[sandbox] cold-restore applied ${workspaceArchive.kind === "provider_snapshot" ? "native snapshot receipt" : "verified tar archive"} revision ${workspaceArchive.descriptor.revision}`,
|
|
2154
|
+
);
|
|
2068
2155
|
}
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2156
|
+
restoredState = (restored as { state?: unknown }).state;
|
|
2157
|
+
const finalInstanceId = readInstanceId(backend, restored);
|
|
2158
|
+
if (!finalInstanceId) {
|
|
2159
|
+
await terminateCreatedSandbox(createdClient, restored, restoredState);
|
|
2160
|
+
throw new SandboxConfigError(
|
|
2161
|
+
backend,
|
|
2162
|
+
`Sandbox backend "${backend}" returned a handle without its declared provider identity`,
|
|
2163
|
+
);
|
|
2164
|
+
}
|
|
2165
|
+
return {
|
|
2166
|
+
client: createdClient,
|
|
2167
|
+
session: restored,
|
|
2168
|
+
sessionState: restoredState ?? resumeFallbackState,
|
|
2169
|
+
instanceId: finalInstanceId,
|
|
2170
|
+
backendId: createdClient.backendId,
|
|
2171
|
+
origin: hydrationApplied ? ("restored" as const) : ("created" as const),
|
|
2172
|
+
...(workspaceArchive ? { restoredArchive: workspaceArchive.descriptor } : {}),
|
|
2173
|
+
};
|
|
2174
|
+
} finally {
|
|
2175
|
+
await hostSpool?.dispose().catch(() => {
|
|
2176
|
+
// Cleanup must not discard a successfully established provider handle
|
|
2177
|
+
// or replace the authoritative restore failure with a filesystem error.
|
|
2178
|
+
console.warn("workspace archive download spool cleanup failed");
|
|
2179
|
+
});
|
|
2082
2180
|
}
|
|
2083
|
-
return {
|
|
2084
|
-
client: createdClient,
|
|
2085
|
-
session: restored,
|
|
2086
|
-
sessionState: restoredState ?? resumeFallbackState,
|
|
2087
|
-
instanceId: finalInstanceId,
|
|
2088
|
-
backendId: createdClient.backendId,
|
|
2089
|
-
origin: hydrationApplied ? ("restored" as const) : ("created" as const),
|
|
2090
|
-
...(workspaceArchive ? { restoredArchive: workspaceArchive.descriptor } : {}),
|
|
2091
|
-
};
|
|
2092
2181
|
};
|
|
2093
2182
|
|
|
2094
2183
|
// Does the envelope carry a RESUMABLE box id (warm reattach), or only a
|
|
@@ -66,6 +66,7 @@ export interface SelfhostedConnectionBinding {
|
|
|
66
66
|
operationResourcePolicy: SelfhostedOperationAdmission["operationResourcePolicy"];
|
|
67
67
|
operationResourcePolicySupported: boolean;
|
|
68
68
|
operationCpuQuotaSupported: boolean;
|
|
69
|
+
transactionalFsWriteSupported?: boolean;
|
|
69
70
|
}
|
|
70
71
|
|
|
71
72
|
export interface ActiveBackendResolverDeps {
|
|
@@ -404,6 +405,7 @@ export function makeActiveBackendResolver(
|
|
|
404
405
|
operationResourcePolicy: connection.operationResourcePolicy,
|
|
405
406
|
operationResourcePolicySupported: connection.operationResourcePolicySupported,
|
|
406
407
|
operationCpuQuotaSupported: connection.operationCpuQuotaSupported,
|
|
408
|
+
transactionalFsWriteSupported: connection.transactionalFsWriteSupported === true,
|
|
407
409
|
// The routing proxy caches this session by sandbox+epoch, but command
|
|
408
410
|
// policy and runner authority are mutable. Re-read one coherent DB
|
|
409
411
|
// snapshot only when a new exec/Git is admitted.
|
|
@@ -281,6 +281,9 @@ export function agentErrorToControlError(
|
|
|
281
281
|
* redirect the output to a file and read it back in ranges/chunks.
|
|
282
282
|
*/
|
|
283
283
|
export function payloadTooLargeMessage(detail: Record<string, string>): string {
|
|
284
|
+
if (detail.direction === "request") {
|
|
285
|
+
return "The request exceeds the machine link's per-message size limit and was not sent. The machine is not offline. Use a supported bounded transfer for large file content.";
|
|
286
|
+
}
|
|
284
287
|
const encoded = detail.encoded_bytes;
|
|
285
288
|
const max = detail.max_payload;
|
|
286
289
|
const sizes =
|
|
@@ -502,6 +505,25 @@ export class NatsControlRpc implements ControlRpc {
|
|
|
502
505
|
} catch (err) {
|
|
503
506
|
const denied = natsAuthorizationFailure(err);
|
|
504
507
|
if (denied) throw denied;
|
|
508
|
+
// The client rejects this before publishing. Keep the healthy connection;
|
|
509
|
+
// a request size fault is not evidence that the agent is offline.
|
|
510
|
+
if (
|
|
511
|
+
typeof err === "object" &&
|
|
512
|
+
err !== null &&
|
|
513
|
+
"code" in err &&
|
|
514
|
+
err.code === "MAX_PAYLOAD_EXCEEDED"
|
|
515
|
+
) {
|
|
516
|
+
return {
|
|
517
|
+
requestId: req.requestId,
|
|
518
|
+
error: {
|
|
519
|
+
code: ErrorCode.ERROR_CODE_PAYLOAD_TOO_LARGE,
|
|
520
|
+
message: "outbound control request exceeds the transport payload limit",
|
|
521
|
+
retryable: false,
|
|
522
|
+
detail: { direction: "request", encoded_bytes: String(payload.byteLength) },
|
|
523
|
+
},
|
|
524
|
+
result: undefined,
|
|
525
|
+
};
|
|
526
|
+
}
|
|
505
527
|
// Re-allow a future request to re-dial if the cached conn was torn down.
|
|
506
528
|
if (isNoRespondersError(err)) {
|
|
507
529
|
// No subscriber on the subject at all → the request reached no responder and
|
|
@@ -110,6 +110,15 @@ export function renderSelfhostedFault(error: SelfhostedControlError): string {
|
|
|
110
110
|
const detail = error.detail;
|
|
111
111
|
|
|
112
112
|
if (error.payloadTooLarge || error.code === ErrorCode.ERROR_CODE_PAYLOAD_TOO_LARGE) {
|
|
113
|
+
if (detail.direction === "request") {
|
|
114
|
+
return assemble(error, {
|
|
115
|
+
headline: "the request was too large to send",
|
|
116
|
+
happened: "the request was not sent because it exceeded the per-message size limit",
|
|
117
|
+
layer: "the machine link's outbound request size limit, not machine availability",
|
|
118
|
+
preserved: "the rejected request did not execute on the machine",
|
|
119
|
+
tryNext: "use a supported bounded transfer; retrying the unchanged request will not help",
|
|
120
|
+
});
|
|
121
|
+
}
|
|
113
122
|
// The STREAMING retention-ceiling overflow (op-stream OP_OVERFLOW) is a
|
|
114
123
|
// different truth than the legacy reply-size wall: the machine STOPPED the
|
|
115
124
|
// command at the ceiling (it did not run to completion), and none of its
|