@granular-software/sdk 0.4.30 → 0.4.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -20
- package/dist/agent-evals.d.mts +7 -6
- package/dist/agent-evals.d.ts +7 -6
- package/dist/agent-evals.js +721 -286
- package/dist/agent-evals.js.map +1 -1
- package/dist/agent-evals.mjs +721 -286
- package/dist/agent-evals.mjs.map +1 -1
- package/dist/cli/index.js +537 -251
- package/dist/{client-BVvOMfln.d.mts → client-C2Gk641P.d.mts} +291 -163
- package/dist/{client-BVvOMfln.d.ts → client-C2Gk641P.d.ts} +291 -163
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +479 -196
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +478 -197
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/agent-evals.js
CHANGED
|
@@ -4010,7 +4010,11 @@ var WSClient = class {
|
|
|
4010
4010
|
return null;
|
|
4011
4011
|
}
|
|
4012
4012
|
try {
|
|
4013
|
-
const
|
|
4013
|
+
const payloadSegment = parts[1];
|
|
4014
|
+
if (!payloadSegment) {
|
|
4015
|
+
return null;
|
|
4016
|
+
}
|
|
4017
|
+
const payloadRaw = this.decodeBase64Url(payloadSegment);
|
|
4014
4018
|
const payload = JSON.parse(payloadRaw);
|
|
4015
4019
|
if (typeof payload.exp !== "number" || !Number.isFinite(payload.exp)) {
|
|
4016
4020
|
return null;
|
|
@@ -4674,27 +4678,27 @@ var Session = class {
|
|
|
4674
4678
|
}
|
|
4675
4679
|
async publishTools(tools, revision = "1.0.0") {
|
|
4676
4680
|
throw new Error(
|
|
4677
|
-
"Environment-scoped effect publication was removed. Declare effects in the manifest and register live handlers with granular.
|
|
4681
|
+
"Environment-scoped effect publication was removed. Declare effects in the manifest and register live handlers with granular.ontology(environment.sandboxId).effects.registerMany(effects)."
|
|
4678
4682
|
);
|
|
4679
4683
|
}
|
|
4680
4684
|
async publishEffect(effect) {
|
|
4681
4685
|
throw new Error(
|
|
4682
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4686
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.register(effect)."
|
|
4683
4687
|
);
|
|
4684
4688
|
}
|
|
4685
4689
|
async publishEffects(effects) {
|
|
4686
4690
|
throw new Error(
|
|
4687
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4691
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.registerMany(effects)."
|
|
4688
4692
|
);
|
|
4689
4693
|
}
|
|
4690
4694
|
async unpublishEffect(name) {
|
|
4691
4695
|
throw new Error(
|
|
4692
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4696
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.unregister(effectName)."
|
|
4693
4697
|
);
|
|
4694
4698
|
}
|
|
4695
4699
|
async unpublishAllEffects() {
|
|
4696
4700
|
throw new Error(
|
|
4697
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4701
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.clear()."
|
|
4698
4702
|
);
|
|
4699
4703
|
}
|
|
4700
4704
|
/**
|
|
@@ -5087,6 +5091,7 @@ import { ${allImports} } from "./sandbox-tools";
|
|
|
5087
5091
|
this.eventListeners.set(event, []);
|
|
5088
5092
|
}
|
|
5089
5093
|
this.eventListeners.get(event).push(handler);
|
|
5094
|
+
return () => this.off(event, handler);
|
|
5090
5095
|
}
|
|
5091
5096
|
/**
|
|
5092
5097
|
* Unsubscribe from session events
|
|
@@ -5543,6 +5548,16 @@ var JobImplementation = class {
|
|
|
5543
5548
|
handler(message);
|
|
5544
5549
|
}
|
|
5545
5550
|
}
|
|
5551
|
+
return () => {
|
|
5552
|
+
const handlers = this.eventListeners.get(event);
|
|
5553
|
+
if (!handlers) {
|
|
5554
|
+
return;
|
|
5555
|
+
}
|
|
5556
|
+
this.eventListeners.set(
|
|
5557
|
+
event,
|
|
5558
|
+
handlers.filter((current) => current !== handler)
|
|
5559
|
+
);
|
|
5560
|
+
};
|
|
5546
5561
|
}
|
|
5547
5562
|
replayAgentMessage(message) {
|
|
5548
5563
|
this.captureAgentMessage(message);
|
|
@@ -11718,6 +11733,22 @@ function normalizeHeapSnapshot(raw) {
|
|
|
11718
11733
|
updatedAt: typeof heap.updatedAt === "number" ? heap.updatedAt : Date.now()
|
|
11719
11734
|
};
|
|
11720
11735
|
}
|
|
11736
|
+
function deriveRuntimeBaseUrl(apiEndpoint) {
|
|
11737
|
+
try {
|
|
11738
|
+
const endpoint = new URL(apiEndpoint);
|
|
11739
|
+
const graphqlSuffix = "/orchestrator/graphql";
|
|
11740
|
+
if (endpoint.pathname.endsWith(graphqlSuffix)) {
|
|
11741
|
+
endpoint.pathname = endpoint.pathname.slice(0, -graphqlSuffix.length);
|
|
11742
|
+
} else if (endpoint.pathname.endsWith("/graphql")) {
|
|
11743
|
+
endpoint.pathname = endpoint.pathname.slice(0, -"/graphql".length);
|
|
11744
|
+
}
|
|
11745
|
+
endpoint.search = "";
|
|
11746
|
+
endpoint.hash = "";
|
|
11747
|
+
return endpoint.toString().replace(/\/$/, "");
|
|
11748
|
+
} catch {
|
|
11749
|
+
return apiEndpoint.replace(/\/orchestrator\/graphql$/, "").replace(/\/$/, "");
|
|
11750
|
+
}
|
|
11751
|
+
}
|
|
11721
11752
|
function normalizeSubject(subject) {
|
|
11722
11753
|
const granularId = subject.granularId || subject.subjectId;
|
|
11723
11754
|
const userId = subject.userId || subject.identityId || granularId;
|
|
@@ -11757,12 +11788,13 @@ function normalizeEnvironmentData(environment) {
|
|
|
11757
11788
|
tracking: environment.tracking || buildPolicy
|
|
11758
11789
|
};
|
|
11759
11790
|
}
|
|
11760
|
-
var Environment = class
|
|
11791
|
+
var Environment = class {
|
|
11792
|
+
granular;
|
|
11761
11793
|
envData;
|
|
11762
11794
|
_apiKey;
|
|
11763
11795
|
_apiEndpoint;
|
|
11764
|
-
constructor(
|
|
11765
|
-
|
|
11796
|
+
constructor(granular, envData, apiKey, apiEndpoint) {
|
|
11797
|
+
this.granular = granular;
|
|
11766
11798
|
this.envData = envData;
|
|
11767
11799
|
this._apiKey = apiKey;
|
|
11768
11800
|
this._apiEndpoint = apiEndpoint;
|
|
@@ -11803,35 +11835,126 @@ var Environment = class extends Session {
|
|
|
11803
11835
|
get permissionProfileId() {
|
|
11804
11836
|
return this.envData.permissionProfileId;
|
|
11805
11837
|
}
|
|
11838
|
+
/** The current build policy backing this environment */
|
|
11839
|
+
get buildPolicy() {
|
|
11840
|
+
return this.envData.buildPolicy;
|
|
11841
|
+
}
|
|
11842
|
+
/** The current update state relative to the followed tag */
|
|
11843
|
+
get updateState() {
|
|
11844
|
+
return this.envData.updateState;
|
|
11845
|
+
}
|
|
11846
|
+
/** Convenience flag for whether this environment trails the current tag target */
|
|
11847
|
+
get isOutdated() {
|
|
11848
|
+
return this.envData.updateState === "update_available";
|
|
11849
|
+
}
|
|
11850
|
+
/** The followed tag name when this environment is tag-tracked */
|
|
11851
|
+
get tag() {
|
|
11852
|
+
return this.envData.tag?.name || this.envData.buildPolicy.tagName || null;
|
|
11853
|
+
}
|
|
11806
11854
|
/** The GraphQL API endpoint URL */
|
|
11807
11855
|
get apiEndpoint() {
|
|
11808
11856
|
return this._apiEndpoint;
|
|
11809
11857
|
}
|
|
11858
|
+
/** Internal auth token used for control-plane and runtime fallback requests */
|
|
11859
|
+
get authToken() {
|
|
11860
|
+
return this._apiKey;
|
|
11861
|
+
}
|
|
11862
|
+
/** Base runtime URL derived from the GraphQL endpoint */
|
|
11863
|
+
get runtimeBaseUrl() {
|
|
11864
|
+
return this.getRuntimeBaseUrl();
|
|
11865
|
+
}
|
|
11866
|
+
get sessions() {
|
|
11867
|
+
return {
|
|
11868
|
+
list: async (options) => this.listSessions(options?.status || "active"),
|
|
11869
|
+
create: async (options) => this.createSession(options),
|
|
11870
|
+
connect: async (sessionId, options) => this.connectSession(sessionId, options),
|
|
11871
|
+
reopen: async (sessionId, options) => this.reopenSession(sessionId, options),
|
|
11872
|
+
close: async (sessionId, session) => this.closeSession(sessionId, session)
|
|
11873
|
+
};
|
|
11874
|
+
}
|
|
11875
|
+
get data() {
|
|
11876
|
+
return {
|
|
11877
|
+
record: async (record) => this.recordObject(record),
|
|
11878
|
+
recordMany: async (records, options) => this.recordObjects(records, options),
|
|
11879
|
+
import: async (records, options) => this.enqueueRecordImport(records, options),
|
|
11880
|
+
listImports: async (status) => this.listRecordImports(status),
|
|
11881
|
+
getImport: async (importId) => this.getRecordImport(importId),
|
|
11882
|
+
getImportSummary: async () => this.getRecordImportSummary(),
|
|
11883
|
+
cancelImport: async (importId) => this.cancelRecordImport(importId),
|
|
11884
|
+
getAwaitingCount: async () => this.getAwaitingRecordCount()
|
|
11885
|
+
};
|
|
11886
|
+
}
|
|
11887
|
+
get feedback() {
|
|
11888
|
+
return {
|
|
11889
|
+
list: async () => this.listFeedback()
|
|
11890
|
+
};
|
|
11891
|
+
}
|
|
11810
11892
|
/**
|
|
11811
|
-
*
|
|
11812
|
-
*
|
|
11813
|
-
*
|
|
11814
|
-
*
|
|
11893
|
+
* Sessionless environments do not own a live transport, so disconnecting the
|
|
11894
|
+
* environment handle itself is a no-op. This keeps the public surface
|
|
11895
|
+
* symmetric with `EnvironmentSession.disconnect()` and lets callers always
|
|
11896
|
+
* clean up safely without tracking whether they currently hold an environment
|
|
11897
|
+
* or a session.
|
|
11815
11898
|
*/
|
|
11816
|
-
|
|
11817
|
-
const doc = this.document;
|
|
11818
|
-
return normalizeHeapSnapshot(doc?.heap);
|
|
11899
|
+
async disconnect() {
|
|
11819
11900
|
}
|
|
11820
|
-
|
|
11821
|
-
|
|
11822
|
-
const
|
|
11823
|
-
|
|
11824
|
-
|
|
11825
|
-
|
|
11826
|
-
|
|
11827
|
-
|
|
11828
|
-
|
|
11829
|
-
|
|
11830
|
-
|
|
11831
|
-
|
|
11832
|
-
|
|
11833
|
-
|
|
11901
|
+
async listSessions(status = "active") {
|
|
11902
|
+
if (status === "all") {
|
|
11903
|
+
const [active, closed] = await Promise.all([
|
|
11904
|
+
this.granular.listOpenSessions({ environmentId: this.environmentId }),
|
|
11905
|
+
this.granular.listClosedSessions({ environmentId: this.environmentId })
|
|
11906
|
+
]);
|
|
11907
|
+
return [...active, ...closed].sort(
|
|
11908
|
+
(left, right) => Date.parse(right.lastSeenAt) - Date.parse(left.lastSeenAt)
|
|
11909
|
+
);
|
|
11910
|
+
}
|
|
11911
|
+
return status === "closed" ? this.granular.listClosedSessions({ environmentId: this.environmentId }) : this.granular.listOpenSessions({ environmentId: this.environmentId });
|
|
11912
|
+
}
|
|
11913
|
+
async createSession(options) {
|
|
11914
|
+
return this.granular.createSession({
|
|
11915
|
+
environmentId: this.environmentId,
|
|
11916
|
+
clientId: options?.clientId,
|
|
11917
|
+
initialHeap: options?.initialHeap
|
|
11918
|
+
});
|
|
11919
|
+
}
|
|
11920
|
+
async connectSession(sessionId, options) {
|
|
11921
|
+
const session = await this.granular["connectSession"]({
|
|
11922
|
+
sessionId,
|
|
11923
|
+
clientId: options?.clientId
|
|
11924
|
+
});
|
|
11925
|
+
if (session.environmentId !== this.environmentId) {
|
|
11926
|
+
await session.disconnect().catch(() => {
|
|
11927
|
+
session.disconnectTransport();
|
|
11928
|
+
});
|
|
11929
|
+
throw new Error(
|
|
11930
|
+
`Session ${sessionId} belongs to environment ${session.environmentId}, not ${this.environmentId}.`
|
|
11931
|
+
);
|
|
11932
|
+
}
|
|
11933
|
+
return session;
|
|
11934
|
+
}
|
|
11935
|
+
async reopenSession(sessionId, options) {
|
|
11936
|
+
const session = await this.granular.reopenSession(sessionId, {
|
|
11937
|
+
clientId: options?.clientId
|
|
11938
|
+
});
|
|
11939
|
+
if (session.environmentId !== this.environmentId) {
|
|
11940
|
+
await session.disconnect().catch(() => {
|
|
11941
|
+
session.disconnectTransport();
|
|
11942
|
+
});
|
|
11943
|
+
throw new Error(
|
|
11944
|
+
`Session ${sessionId} belongs to environment ${session.environmentId}, not ${this.environmentId}.`
|
|
11945
|
+
);
|
|
11834
11946
|
}
|
|
11947
|
+
return session;
|
|
11948
|
+
}
|
|
11949
|
+
async closeSession(sessionId, session) {
|
|
11950
|
+
await this.granular.closeSession(sessionId, session);
|
|
11951
|
+
}
|
|
11952
|
+
async listFeedback() {
|
|
11953
|
+
const response = await this.controlPlaneRequest(`/control/environments/${this.environmentId}/feedback`);
|
|
11954
|
+
return Array.isArray(response.items) ? response.items : [];
|
|
11955
|
+
}
|
|
11956
|
+
getRuntimeBaseUrl() {
|
|
11957
|
+
return deriveRuntimeBaseUrl(this._apiEndpoint);
|
|
11835
11958
|
}
|
|
11836
11959
|
async controlPlaneRequest(path2, options = {}) {
|
|
11837
11960
|
const runtimeBase = this.getRuntimeBaseUrl();
|
|
@@ -11852,95 +11975,6 @@ var Environment = class extends Session {
|
|
|
11852
11975
|
}
|
|
11853
11976
|
return response.json();
|
|
11854
11977
|
}
|
|
11855
|
-
/**
|
|
11856
|
-
* Close the session and disconnect from the sandbox.
|
|
11857
|
-
*
|
|
11858
|
-
* Sends `client.goodbye` over WebSocket first, then issues an HTTP fallback
|
|
11859
|
-
* to the runtime goodbye endpoint if no definitive WS-side runtime notify
|
|
11860
|
-
* acknowledgement was observed.
|
|
11861
|
-
*/
|
|
11862
|
-
async disconnect() {
|
|
11863
|
-
let wsNotifiedRuntime = false;
|
|
11864
|
-
try {
|
|
11865
|
-
const goodbye = await this.rpc(
|
|
11866
|
-
"client.goodbye",
|
|
11867
|
-
{
|
|
11868
|
-
timestamp: Date.now()
|
|
11869
|
-
}
|
|
11870
|
-
);
|
|
11871
|
-
wsNotifiedRuntime = Boolean(goodbye?.ok && goodbye?.via);
|
|
11872
|
-
} catch {
|
|
11873
|
-
wsNotifiedRuntime = false;
|
|
11874
|
-
}
|
|
11875
|
-
if (!wsNotifiedRuntime) {
|
|
11876
|
-
try {
|
|
11877
|
-
const runtimeBase = this.getRuntimeBaseUrl();
|
|
11878
|
-
await fetch(
|
|
11879
|
-
`${runtimeBase}/orchestrator/runtime/environments/${this.environmentId}/session-goodbye`,
|
|
11880
|
-
{
|
|
11881
|
-
method: "POST",
|
|
11882
|
-
headers: {
|
|
11883
|
-
"Content-Type": "application/json",
|
|
11884
|
-
Authorization: `Bearer ${this._apiKey}`,
|
|
11885
|
-
Connection: "close"
|
|
11886
|
-
},
|
|
11887
|
-
body: JSON.stringify({
|
|
11888
|
-
reason: "sdk_disconnect_http_fallback",
|
|
11889
|
-
sessionId: this.client.currentSessionId
|
|
11890
|
-
})
|
|
11891
|
-
}
|
|
11892
|
-
);
|
|
11893
|
-
} catch {
|
|
11894
|
-
}
|
|
11895
|
-
}
|
|
11896
|
-
this.client.disconnect();
|
|
11897
|
-
}
|
|
11898
|
-
/**
|
|
11899
|
-
* Close only the socket transport without sending `client.goodbye`.
|
|
11900
|
-
*
|
|
11901
|
-
* Use this when the caller intends to immediately reattach to the same
|
|
11902
|
-
* session after an unexpected disconnect.
|
|
11903
|
-
*/
|
|
11904
|
-
disconnectTransport() {
|
|
11905
|
-
this.client.disconnect();
|
|
11906
|
-
}
|
|
11907
|
-
// ==================== GRAPH CONTAINER READINESS ====================
|
|
11908
|
-
/** The last known graph container status, updated by checkReadiness() or on heartbeat */
|
|
11909
|
-
graphContainerStatus = null;
|
|
11910
|
-
/**
|
|
11911
|
-
* Check if the graph container is ready and warm.
|
|
11912
|
-
*
|
|
11913
|
-
* Sends a lightweight heartbeat RPC to the Session DO which internally
|
|
11914
|
-
* pings the FalkorDB container. The response includes `graphContainerStatus`,
|
|
11915
|
-
* which is stored locally and emitted as a `readiness` event.
|
|
11916
|
-
*
|
|
11917
|
-
* Use this method to proactively warm the graph container before any
|
|
11918
|
-
* GraphQL query that requires it, or to poll the container's state in
|
|
11919
|
-
* the background.
|
|
11920
|
-
*
|
|
11921
|
-
* @returns The current graph container status object
|
|
11922
|
-
*
|
|
11923
|
-
* @example
|
|
11924
|
-
* ```typescript
|
|
11925
|
-
* const status = await env.checkReadiness();
|
|
11926
|
-
* console.log(status.status); // 'hot' | 'warming' | 'unknown'
|
|
11927
|
-
*
|
|
11928
|
-
* // Or listen for live updates
|
|
11929
|
-
* env.on('readiness', (status) => {
|
|
11930
|
-
* console.log('Graph is now:', status.status);
|
|
11931
|
-
* });
|
|
11932
|
-
* ```
|
|
11933
|
-
*/
|
|
11934
|
-
async checkReadiness() {
|
|
11935
|
-
const result = await this.client.call("client.heartbeat", {});
|
|
11936
|
-
const containerStatus = result?.graphContainerStatus ?? {
|
|
11937
|
-
lastKeepAliveAt: Date.now(),
|
|
11938
|
-
status: "unknown"
|
|
11939
|
-
};
|
|
11940
|
-
this.graphContainerStatus = containerStatus;
|
|
11941
|
-
this.emit("readiness", containerStatus);
|
|
11942
|
-
return containerStatus;
|
|
11943
|
-
}
|
|
11944
11978
|
// ==================== ID ↔ GRAPH PATH MAPPING ====================
|
|
11945
11979
|
/**
|
|
11946
11980
|
* Convert a class name + real-world ID into a unique graph path.
|
|
@@ -12070,7 +12104,9 @@ var Environment = class extends Session {
|
|
|
12070
12104
|
}
|
|
12071
12105
|
);
|
|
12072
12106
|
if (result.errors?.length) {
|
|
12073
|
-
throw new Error(
|
|
12107
|
+
throw new Error(
|
|
12108
|
+
`defineRelationship failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`
|
|
12109
|
+
);
|
|
12074
12110
|
}
|
|
12075
12111
|
return result.data.at.define_relationship;
|
|
12076
12112
|
}
|
|
@@ -12106,7 +12142,9 @@ var Environment = class extends Session {
|
|
|
12106
12142
|
{ path: modelPath }
|
|
12107
12143
|
);
|
|
12108
12144
|
if (result.errors?.length) {
|
|
12109
|
-
throw new Error(
|
|
12145
|
+
throw new Error(
|
|
12146
|
+
`getRelationships failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`
|
|
12147
|
+
);
|
|
12110
12148
|
}
|
|
12111
12149
|
return result.data?.model?.relationships || [];
|
|
12112
12150
|
}
|
|
@@ -12145,7 +12183,7 @@ var Environment = class extends Session {
|
|
|
12145
12183
|
{ target: targetPath }
|
|
12146
12184
|
);
|
|
12147
12185
|
if (result.errors?.length) {
|
|
12148
|
-
throw new Error(`attach failed: ${result.errors[0]
|
|
12186
|
+
throw new Error(`attach failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`);
|
|
12149
12187
|
}
|
|
12150
12188
|
}
|
|
12151
12189
|
/**
|
|
@@ -12180,7 +12218,7 @@ var Environment = class extends Session {
|
|
|
12180
12218
|
}`
|
|
12181
12219
|
);
|
|
12182
12220
|
if (result.errors?.length) {
|
|
12183
|
-
throw new Error(`detach failed: ${result.errors[0]
|
|
12221
|
+
throw new Error(`detach failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`);
|
|
12184
12222
|
}
|
|
12185
12223
|
}
|
|
12186
12224
|
/**
|
|
@@ -12207,7 +12245,9 @@ var Environment = class extends Session {
|
|
|
12207
12245
|
}`
|
|
12208
12246
|
);
|
|
12209
12247
|
if (result.errors?.length) {
|
|
12210
|
-
throw new Error(
|
|
12248
|
+
throw new Error(
|
|
12249
|
+
`listRelated failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`
|
|
12250
|
+
);
|
|
12211
12251
|
}
|
|
12212
12252
|
return result.data?.at?.at?.list_related || [];
|
|
12213
12253
|
}
|
|
@@ -12300,7 +12340,7 @@ var Environment = class extends Session {
|
|
|
12300
12340
|
async _runGraphql(query, label) {
|
|
12301
12341
|
const result = await this.graphql(query);
|
|
12302
12342
|
if (result.errors?.length) {
|
|
12303
|
-
throw new Error(`${label}: ${result.errors[0]
|
|
12343
|
+
throw new Error(`${label}: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`);
|
|
12304
12344
|
}
|
|
12305
12345
|
return result.data;
|
|
12306
12346
|
}
|
|
@@ -12645,7 +12685,11 @@ var Environment = class extends Session {
|
|
|
12645
12685
|
*/
|
|
12646
12686
|
async recordObject(options) {
|
|
12647
12687
|
const results = await this.recordObjects([options]);
|
|
12648
|
-
|
|
12688
|
+
const result = results[0];
|
|
12689
|
+
if (!result) {
|
|
12690
|
+
throw new Error("recordObject: no result returned for record");
|
|
12691
|
+
}
|
|
12692
|
+
return result;
|
|
12649
12693
|
}
|
|
12650
12694
|
/**
|
|
12651
12695
|
* Batch version of `recordObject()`.
|
|
@@ -12697,7 +12741,13 @@ var Environment = class extends Session {
|
|
|
12697
12741
|
);
|
|
12698
12742
|
}
|
|
12699
12743
|
for (let index = 0; index < items.length; index += 1) {
|
|
12700
|
-
|
|
12744
|
+
const item = items[index];
|
|
12745
|
+
if (!item) {
|
|
12746
|
+
throw new Error(
|
|
12747
|
+
`recordObjects: chunk ${plan.chunkIndex + 1} returned an empty result at index ${index}`
|
|
12748
|
+
);
|
|
12749
|
+
}
|
|
12750
|
+
results[plan.offset + index] = item;
|
|
12701
12751
|
}
|
|
12702
12752
|
if (onChunk) {
|
|
12703
12753
|
const info = {
|
|
@@ -12801,36 +12851,186 @@ var Environment = class extends Session {
|
|
|
12801
12851
|
}
|
|
12802
12852
|
);
|
|
12803
12853
|
}
|
|
12804
|
-
|
|
12854
|
+
};
|
|
12855
|
+
var EnvironmentSession = class extends Session {
|
|
12856
|
+
environment;
|
|
12857
|
+
/** The last known graph container status, updated by checkReadiness() or on heartbeat */
|
|
12858
|
+
graphContainerStatus = null;
|
|
12859
|
+
constructor(client, environment, clientId) {
|
|
12860
|
+
super(client, clientId);
|
|
12861
|
+
this.environment = environment;
|
|
12862
|
+
}
|
|
12863
|
+
get environmentId() {
|
|
12864
|
+
return this.environment.environmentId;
|
|
12865
|
+
}
|
|
12866
|
+
get sandboxId() {
|
|
12867
|
+
return this.environment.sandboxId;
|
|
12868
|
+
}
|
|
12869
|
+
get ontologyId() {
|
|
12870
|
+
return this.environment.ontologyId;
|
|
12871
|
+
}
|
|
12872
|
+
get subjectId() {
|
|
12873
|
+
return this.environment.subjectId;
|
|
12874
|
+
}
|
|
12875
|
+
get envName() {
|
|
12876
|
+
return this.environment.envName;
|
|
12877
|
+
}
|
|
12878
|
+
get versionId() {
|
|
12879
|
+
return this.environment.versionId;
|
|
12880
|
+
}
|
|
12881
|
+
get granularId() {
|
|
12882
|
+
return this.environment.granularId;
|
|
12883
|
+
}
|
|
12884
|
+
get permissionProfileId() {
|
|
12885
|
+
return this.environment.permissionProfileId;
|
|
12886
|
+
}
|
|
12887
|
+
get apiEndpoint() {
|
|
12888
|
+
return this.environment.apiEndpoint;
|
|
12889
|
+
}
|
|
12890
|
+
get data() {
|
|
12891
|
+
return this.environment.data;
|
|
12892
|
+
}
|
|
12893
|
+
get feedback() {
|
|
12894
|
+
return this.environment.feedback;
|
|
12895
|
+
}
|
|
12805
12896
|
/**
|
|
12806
|
-
*
|
|
12897
|
+
* Return a plain JS snapshot of the synced session heap.
|
|
12807
12898
|
*/
|
|
12808
|
-
|
|
12809
|
-
|
|
12899
|
+
getHeap() {
|
|
12900
|
+
const doc = this.document;
|
|
12901
|
+
return normalizeHeapSnapshot(doc?.heap);
|
|
12902
|
+
}
|
|
12903
|
+
async graphql(query, variables) {
|
|
12904
|
+
return this.environment.graphql(query, variables);
|
|
12905
|
+
}
|
|
12906
|
+
async defineRelationship(options) {
|
|
12907
|
+
return this.environment.defineRelationship(options);
|
|
12908
|
+
}
|
|
12909
|
+
async getRelationships(modelPath) {
|
|
12910
|
+
return this.environment.getRelationships(modelPath);
|
|
12911
|
+
}
|
|
12912
|
+
async attach(modelPath, submodelPath, targetPath) {
|
|
12913
|
+
return this.environment.attach(modelPath, submodelPath, targetPath);
|
|
12914
|
+
}
|
|
12915
|
+
async detach(modelPath, submodelPath, targetPath) {
|
|
12916
|
+
return this.environment.detach(modelPath, submodelPath, targetPath);
|
|
12917
|
+
}
|
|
12918
|
+
async listRelated(modelPath, submodelPath) {
|
|
12919
|
+
return this.environment.listRelated(modelPath, submodelPath);
|
|
12920
|
+
}
|
|
12921
|
+
async applyManifest(manifest) {
|
|
12922
|
+
return this.environment.applyManifest(manifest);
|
|
12923
|
+
}
|
|
12924
|
+
async recordObject(options) {
|
|
12925
|
+
return this.environment.recordObject(options);
|
|
12926
|
+
}
|
|
12927
|
+
async recordObjects(records, options) {
|
|
12928
|
+
return this.environment.recordObjects(records, options);
|
|
12929
|
+
}
|
|
12930
|
+
async enqueueRecordImport(records, options = {}) {
|
|
12931
|
+
return this.environment.enqueueRecordImport(records, options);
|
|
12932
|
+
}
|
|
12933
|
+
async listRecordImports(status) {
|
|
12934
|
+
return this.environment.listRecordImports(status);
|
|
12935
|
+
}
|
|
12936
|
+
async getRecordImportSummary() {
|
|
12937
|
+
return this.environment.getRecordImportSummary();
|
|
12938
|
+
}
|
|
12939
|
+
async getAwaitingRecordCount() {
|
|
12940
|
+
return this.environment.getAwaitingRecordCount();
|
|
12941
|
+
}
|
|
12942
|
+
async getRecordImport(importId) {
|
|
12943
|
+
return this.environment.getRecordImport(importId);
|
|
12944
|
+
}
|
|
12945
|
+
async cancelRecordImport(importId) {
|
|
12946
|
+
return this.environment.cancelRecordImport(importId);
|
|
12947
|
+
}
|
|
12948
|
+
async listFeedback() {
|
|
12949
|
+
return this.environment.listFeedback();
|
|
12810
12950
|
}
|
|
12811
12951
|
/**
|
|
12812
|
-
*
|
|
12952
|
+
* Close the session and disconnect from the sandbox.
|
|
12953
|
+
*
|
|
12954
|
+
* Sends `client.goodbye` over WebSocket first, then issues an HTTP fallback
|
|
12955
|
+
* to the runtime goodbye endpoint if no definitive WS-side runtime notify
|
|
12956
|
+
* acknowledgement was observed.
|
|
12813
12957
|
*/
|
|
12814
|
-
async
|
|
12815
|
-
|
|
12958
|
+
async disconnect() {
|
|
12959
|
+
let wsNotifiedRuntime = false;
|
|
12960
|
+
try {
|
|
12961
|
+
const goodbye = await this.rpc(
|
|
12962
|
+
"client.goodbye",
|
|
12963
|
+
{
|
|
12964
|
+
timestamp: Date.now()
|
|
12965
|
+
}
|
|
12966
|
+
);
|
|
12967
|
+
wsNotifiedRuntime = Boolean(goodbye?.ok && goodbye?.via);
|
|
12968
|
+
} catch {
|
|
12969
|
+
wsNotifiedRuntime = false;
|
|
12970
|
+
}
|
|
12971
|
+
if (!wsNotifiedRuntime) {
|
|
12972
|
+
try {
|
|
12973
|
+
await fetch(
|
|
12974
|
+
`${this.environment.runtimeBaseUrl}/orchestrator/runtime/environments/${this.environmentId}/session-goodbye`,
|
|
12975
|
+
{
|
|
12976
|
+
method: "POST",
|
|
12977
|
+
headers: {
|
|
12978
|
+
"Content-Type": "application/json",
|
|
12979
|
+
Authorization: `Bearer ${this.environment.authToken}`,
|
|
12980
|
+
Connection: "close"
|
|
12981
|
+
},
|
|
12982
|
+
body: JSON.stringify({
|
|
12983
|
+
reason: "sdk_disconnect_http_fallback",
|
|
12984
|
+
sessionId: this.client.currentSessionId
|
|
12985
|
+
})
|
|
12986
|
+
}
|
|
12987
|
+
);
|
|
12988
|
+
} catch {
|
|
12989
|
+
}
|
|
12990
|
+
}
|
|
12991
|
+
this.client.disconnect();
|
|
12816
12992
|
}
|
|
12817
12993
|
/**
|
|
12818
|
-
*
|
|
12994
|
+
* Close only the socket transport without sending `client.goodbye`.
|
|
12819
12995
|
*/
|
|
12820
|
-
|
|
12821
|
-
|
|
12996
|
+
disconnectTransport() {
|
|
12997
|
+
this.client.disconnect();
|
|
12822
12998
|
}
|
|
12823
12999
|
/**
|
|
12824
|
-
*
|
|
13000
|
+
* Backwards-compatible alias for `disconnect()`.
|
|
12825
13001
|
*/
|
|
12826
|
-
async
|
|
12827
|
-
|
|
13002
|
+
async close() {
|
|
13003
|
+
await this.disconnect();
|
|
12828
13004
|
}
|
|
12829
13005
|
/**
|
|
12830
|
-
*
|
|
13006
|
+
* Check if the graph container is ready and warm.
|
|
12831
13007
|
*/
|
|
12832
|
-
async
|
|
12833
|
-
|
|
13008
|
+
async checkReadiness() {
|
|
13009
|
+
const result = await this.client.call("client.heartbeat", {});
|
|
13010
|
+
const containerStatus = result?.graphContainerStatus ?? {
|
|
13011
|
+
lastKeepAliveAt: Date.now(),
|
|
13012
|
+
status: "unknown"
|
|
13013
|
+
};
|
|
13014
|
+
this.graphContainerStatus = containerStatus;
|
|
13015
|
+
this.emit("readiness", containerStatus);
|
|
13016
|
+
return containerStatus;
|
|
13017
|
+
}
|
|
13018
|
+
};
|
|
13019
|
+
var OntologyHandle = class {
|
|
13020
|
+
granular;
|
|
13021
|
+
ontologyNameOrId;
|
|
13022
|
+
constructor(granular, ontologyNameOrId) {
|
|
13023
|
+
this.granular = granular;
|
|
13024
|
+
this.ontologyNameOrId = ontologyNameOrId;
|
|
13025
|
+
}
|
|
13026
|
+
get effects() {
|
|
13027
|
+
return {
|
|
13028
|
+
register: async (effect) => this.granular.registerEffect(this.ontologyNameOrId, effect),
|
|
13029
|
+
registerMany: async (effects) => this.granular.registerEffects(this.ontologyNameOrId, effects),
|
|
13030
|
+
unregister: async (name) => this.granular.unregisterEffect(this.ontologyNameOrId, name),
|
|
13031
|
+
clear: async () => this.granular.unregisterAllEffects(this.ontologyNameOrId),
|
|
13032
|
+
disconnect: async () => this.granular.disconnectEffects(this.ontologyNameOrId)
|
|
13033
|
+
};
|
|
12834
13034
|
}
|
|
12835
13035
|
};
|
|
12836
13036
|
var Granular = class _Granular {
|
|
@@ -12867,6 +13067,12 @@ var Granular = class _Granular {
|
|
|
12867
13067
|
this.onReconnectError = options.onReconnectError;
|
|
12868
13068
|
this.httpUrl = this.apiUrl.replace(/^wss:\/\//, "https://").replace(/^ws:\/\//, "http://").replace(/\/ws$/, "");
|
|
12869
13069
|
}
|
|
13070
|
+
/**
|
|
13071
|
+
* Return an ontology-scoped handle for effects and other ontology-level APIs.
|
|
13072
|
+
*/
|
|
13073
|
+
ontology(ontologyNameOrId) {
|
|
13074
|
+
return new OntologyHandle(this, ontologyNameOrId);
|
|
13075
|
+
}
|
|
12870
13076
|
/**
|
|
12871
13077
|
* Records/upserts a user and prepares them for sandbox connections
|
|
12872
13078
|
*
|
|
@@ -12903,7 +13109,23 @@ var Granular = class _Granular {
|
|
|
12903
13109
|
permissions: options.permissions || []
|
|
12904
13110
|
});
|
|
12905
13111
|
}
|
|
13112
|
+
/**
|
|
13113
|
+
* Alias for `recordUser()` with user-facing naming that matches upsert semantics.
|
|
13114
|
+
*/
|
|
13115
|
+
async upsertUser(options) {
|
|
13116
|
+
return this.recordUser(options);
|
|
13117
|
+
}
|
|
12906
13118
|
async resolveConnectUser(options) {
|
|
13119
|
+
const providedIdentityCount = [
|
|
13120
|
+
Boolean(options.user),
|
|
13121
|
+
Boolean(options.userId),
|
|
13122
|
+
Boolean(options.granularId)
|
|
13123
|
+
].filter(Boolean).length;
|
|
13124
|
+
if (providedIdentityCount !== 1) {
|
|
13125
|
+
throw new Error(
|
|
13126
|
+
"openEnvironment() requires exactly one of userId, granularId, or a user object returned by recordUser()."
|
|
13127
|
+
);
|
|
13128
|
+
}
|
|
12907
13129
|
if (options.user) {
|
|
12908
13130
|
const user = normalizeUser(options.user);
|
|
12909
13131
|
return {
|
|
@@ -12940,56 +13162,85 @@ var Granular = class _Granular {
|
|
|
12940
13162
|
};
|
|
12941
13163
|
}
|
|
12942
13164
|
throw new Error(
|
|
12943
|
-
"
|
|
13165
|
+
"openEnvironment() requires either userId, granularId, or a user object returned by recordUser()."
|
|
12944
13166
|
);
|
|
12945
13167
|
}
|
|
12946
13168
|
/**
|
|
12947
|
-
*
|
|
12948
|
-
*
|
|
12949
|
-
* Effects are registered at the sandbox level via `granular.registerEffect()`
|
|
12950
|
-
* or `granular.registerEffects()`. Sessions pick up live availability from
|
|
12951
|
-
* the sandbox registry automatically.
|
|
12952
|
-
*
|
|
12953
|
-
* @param options - Connection options
|
|
12954
|
-
* @returns An active environment session
|
|
13169
|
+
* Open or resolve an ontology environment for one user without opening a session.
|
|
12955
13170
|
*
|
|
12956
13171
|
* @example
|
|
12957
13172
|
* ```typescript
|
|
12958
|
-
* const environment = await granular.
|
|
13173
|
+
* const environment = await granular.openEnvironment({
|
|
12959
13174
|
* ontology: 'my-ontology',
|
|
12960
|
-
*
|
|
13175
|
+
* tag: 'dev',
|
|
12961
13176
|
* userId: 'user_123',
|
|
12962
13177
|
* permissions: ['agent'],
|
|
12963
13178
|
* });
|
|
12964
13179
|
*
|
|
12965
|
-
* await
|
|
12966
|
-
*
|
|
12967
|
-
*
|
|
12968
|
-
*
|
|
12969
|
-
* handler: async () => 'Hello!',
|
|
13180
|
+
* await environment.data.record({
|
|
13181
|
+
* className: 'customer',
|
|
13182
|
+
* id: 'acme',
|
|
13183
|
+
* fields: { name: 'Acme' },
|
|
12970
13184
|
* });
|
|
12971
13185
|
*
|
|
12972
|
-
*
|
|
12973
|
-
* const job = await
|
|
12974
|
-
*
|
|
12975
|
-
* return await tools.greet({});
|
|
12976
|
-
* `);
|
|
12977
|
-
*
|
|
12978
|
-
* console.log(await job.result); // 'Hello!'
|
|
13186
|
+
* const session = await environment.sessions.create();
|
|
13187
|
+
* const job = await session.submitJob(`return "hello";`);
|
|
13188
|
+
* console.log(await job.result);
|
|
12979
13189
|
* ```
|
|
12980
13190
|
*/
|
|
13191
|
+
async openEnvironment(options) {
|
|
13192
|
+
const envData = await this.resolveOpenEnvironmentData(
|
|
13193
|
+
options,
|
|
13194
|
+
"openEnvironment"
|
|
13195
|
+
);
|
|
13196
|
+
return this.bindEnvironmentHandle(envData);
|
|
13197
|
+
}
|
|
13198
|
+
/**
|
|
13199
|
+
* Deprecated compatibility alias for `openEnvironment()`.
|
|
13200
|
+
*
|
|
13201
|
+
* `connect()` no longer opens a runtime session automatically.
|
|
13202
|
+
*/
|
|
12981
13203
|
async connect(options) {
|
|
12982
|
-
|
|
13204
|
+
return this.openEnvironment({
|
|
13205
|
+
...options,
|
|
13206
|
+
tag: this.resolveRequestedTag(options, "connect"),
|
|
13207
|
+
permissions: options.permissions || options.user?.permissions || []
|
|
13208
|
+
});
|
|
13209
|
+
}
|
|
13210
|
+
resolveRequestedTag(options, methodName) {
|
|
13211
|
+
const tag = options.tag?.trim() || options.tagName?.trim() || options.environment?.trim();
|
|
13212
|
+
if (!tag) {
|
|
13213
|
+
throw new Error(`${methodName}() requires \`tag\`.`);
|
|
13214
|
+
}
|
|
13215
|
+
return tag;
|
|
13216
|
+
}
|
|
13217
|
+
buildManagedEnvironmentName(tag, versionId) {
|
|
13218
|
+
return `__sdk__${tag}__${versionId}`;
|
|
13219
|
+
}
|
|
13220
|
+
matchesTagTrackedEnvironment(environment, tagName, tagId) {
|
|
13221
|
+
const environmentTagName = environment.tag?.name || environment.buildPolicy.tagName || null;
|
|
13222
|
+
return environment.tagId === tagId || environmentTagName === tagName || environment.environment === tagName || environment.envName === tagName || environment.environment === this.buildManagedEnvironmentName(tagName, environment.versionId) || environment.envName === this.buildManagedEnvironmentName(tagName, environment.versionId);
|
|
13223
|
+
}
|
|
13224
|
+
sortEnvironmentsByRecency(environments) {
|
|
13225
|
+
return [...environments].sort(
|
|
13226
|
+
(left, right) => right.updatedAt - left.updatedAt
|
|
13227
|
+
);
|
|
13228
|
+
}
|
|
13229
|
+
async resolveOpenEnvironmentData(options, methodName) {
|
|
12983
13230
|
const ontology = options.ontology;
|
|
12984
13231
|
if (!ontology) {
|
|
12985
|
-
throw new Error(
|
|
13232
|
+
throw new Error(`${methodName}() requires \`ontology\`.`);
|
|
12986
13233
|
}
|
|
12987
|
-
const
|
|
12988
|
-
if (!
|
|
12989
|
-
throw new Error(
|
|
13234
|
+
const tagName = options.tag?.trim();
|
|
13235
|
+
if (!tagName) {
|
|
13236
|
+
throw new Error(`${methodName}() requires \`tag\`.`);
|
|
12990
13237
|
}
|
|
12991
|
-
const tagName = options.tagName?.trim() || void 0;
|
|
12992
13238
|
const user = await this.resolveConnectUser(options);
|
|
13239
|
+
if (!Array.isArray(user.permissions) || user.permissions.length === 0) {
|
|
13240
|
+
throw new Error(
|
|
13241
|
+
`${methodName}() requires at least one permission so the SDK can ensure assignments for new users.`
|
|
13242
|
+
);
|
|
13243
|
+
}
|
|
12993
13244
|
const sandbox = await this.findOrCreateSandbox(ontology);
|
|
12994
13245
|
for (const profileName of user.permissions) {
|
|
12995
13246
|
const profileId = await this.ensurePermissionProfile(
|
|
@@ -13002,22 +13253,49 @@ var Granular = class _Granular {
|
|
|
13002
13253
|
profileId
|
|
13003
13254
|
);
|
|
13004
13255
|
}
|
|
13005
|
-
const
|
|
13256
|
+
const tags = await this.request(
|
|
13257
|
+
`/control/sandboxes/${sandbox.sandboxId}/tags`
|
|
13258
|
+
);
|
|
13259
|
+
const tag = (tags.items || []).find(
|
|
13260
|
+
(candidate) => Boolean(candidate?.name === tagName)
|
|
13261
|
+
);
|
|
13262
|
+
if (!tag) {
|
|
13263
|
+
throw new Error(
|
|
13264
|
+
`Tag "${tagName}" was not found for ontology ${sandbox.sandboxId}.`
|
|
13265
|
+
);
|
|
13266
|
+
}
|
|
13267
|
+
const targetVersionId = tag.targetVersionId || tag.targetBuildId;
|
|
13268
|
+
if (!targetVersionId) {
|
|
13269
|
+
throw new Error(
|
|
13270
|
+
`Tag "${tagName}" does not currently point to a build/version.`
|
|
13271
|
+
);
|
|
13272
|
+
}
|
|
13273
|
+
const allEnvironments = await this.environments.list(sandbox.sandboxId);
|
|
13274
|
+
const userEnvironments = allEnvironments.filter(
|
|
13275
|
+
(environment) => environment.subjectId === user.granularId
|
|
13276
|
+
);
|
|
13277
|
+
const currentMatches = this.sortEnvironmentsByRecency(
|
|
13278
|
+
userEnvironments.filter(
|
|
13279
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId) && environment.versionId === targetVersionId
|
|
13280
|
+
)
|
|
13281
|
+
);
|
|
13282
|
+
if (currentMatches.length > 0) {
|
|
13283
|
+
return currentMatches[0];
|
|
13284
|
+
}
|
|
13285
|
+
const outdatedMatches = this.sortEnvironmentsByRecency(
|
|
13286
|
+
userEnvironments.filter(
|
|
13287
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId)
|
|
13288
|
+
)
|
|
13289
|
+
);
|
|
13290
|
+
if (outdatedMatches.length > 0 && options.createFreshIfOutdated !== true) {
|
|
13291
|
+
return outdatedMatches[0];
|
|
13292
|
+
}
|
|
13293
|
+
return this.environments.create(sandbox.sandboxId, {
|
|
13006
13294
|
subjectId: user.granularId,
|
|
13007
|
-
environment:
|
|
13008
|
-
|
|
13295
|
+
environment: this.buildManagedEnvironmentName(tagName, targetVersionId),
|
|
13296
|
+
tagId: tag.tagId,
|
|
13009
13297
|
permissionProfileId: null
|
|
13010
13298
|
});
|
|
13011
|
-
await this.activateEnvironment(envData.environmentId);
|
|
13012
|
-
const session = await this.request("/ws/sessions", {
|
|
13013
|
-
method: "POST",
|
|
13014
|
-
body: JSON.stringify({
|
|
13015
|
-
environmentId: envData.environmentId,
|
|
13016
|
-
clientId,
|
|
13017
|
-
initialHeap: options.initialHeap
|
|
13018
|
-
})
|
|
13019
|
-
});
|
|
13020
|
-
return this.bindWebSocketEnvironment(envData, clientId, session);
|
|
13021
13299
|
}
|
|
13022
13300
|
/**
|
|
13023
13301
|
* List active (open) sessions for an environment — each session is one agent conversation thread.
|
|
@@ -13080,6 +13358,7 @@ var Granular = class _Granular {
|
|
|
13080
13358
|
const clientId = options.clientId || `client_${Date.now()}`;
|
|
13081
13359
|
await this.activateEnvironment(options.environmentId);
|
|
13082
13360
|
const envData = await this.environments.get(options.environmentId);
|
|
13361
|
+
const environment = this.bindEnvironmentHandle(envData);
|
|
13083
13362
|
const session = await this.request("/ws/sessions", {
|
|
13084
13363
|
method: "POST",
|
|
13085
13364
|
body: JSON.stringify({
|
|
@@ -13088,7 +13367,7 @@ var Granular = class _Granular {
|
|
|
13088
13367
|
initialHeap: options.initialHeap
|
|
13089
13368
|
})
|
|
13090
13369
|
});
|
|
13091
|
-
return this.
|
|
13370
|
+
return this.bindWebSocketEnvironmentSession(environment, clientId, session);
|
|
13092
13371
|
}
|
|
13093
13372
|
/**
|
|
13094
13373
|
* Connect to an existing open session (same conversation thread) using a freshly minted WebSocket token.
|
|
@@ -13100,7 +13379,8 @@ var Granular = class _Granular {
|
|
|
13100
13379
|
body: JSON.stringify({})
|
|
13101
13380
|
});
|
|
13102
13381
|
const envData = await this.environments.get(minted.environmentId);
|
|
13103
|
-
|
|
13382
|
+
const environment = this.bindEnvironmentHandle(envData);
|
|
13383
|
+
return this.bindWebSocketEnvironmentSession(environment, clientId, minted);
|
|
13104
13384
|
}
|
|
13105
13385
|
/**
|
|
13106
13386
|
* Mark a session closed in the control plane. If `environment` is the connected handle for that
|
|
@@ -13132,7 +13412,11 @@ var Granular = class _Granular {
|
|
|
13132
13412
|
});
|
|
13133
13413
|
return this.connectSession({ sessionId, clientId: options?.clientId });
|
|
13134
13414
|
}
|
|
13135
|
-
|
|
13415
|
+
bindEnvironmentHandle(envData) {
|
|
13416
|
+
const graphqlEndpoint = `${this.httpUrl}/orchestrator/graphql`;
|
|
13417
|
+
return new Environment(this, envData, this.apiKey, graphqlEndpoint);
|
|
13418
|
+
}
|
|
13419
|
+
async bindWebSocketEnvironmentSession(environment, clientId, session) {
|
|
13136
13420
|
const client = new WSClient({
|
|
13137
13421
|
url: session.wsUrl,
|
|
13138
13422
|
sessionId: session.sessionId,
|
|
@@ -13143,16 +13427,13 @@ var Granular = class _Granular {
|
|
|
13143
13427
|
onReconnectError: this.onReconnectError
|
|
13144
13428
|
});
|
|
13145
13429
|
await client.connect();
|
|
13146
|
-
const
|
|
13147
|
-
const environment = new Environment(
|
|
13430
|
+
const environmentSession = new EnvironmentSession(
|
|
13148
13431
|
client,
|
|
13149
|
-
|
|
13150
|
-
clientId
|
|
13151
|
-
this.apiKey,
|
|
13152
|
-
graphqlEndpoint
|
|
13432
|
+
environment,
|
|
13433
|
+
clientId
|
|
13153
13434
|
);
|
|
13154
|
-
await
|
|
13155
|
-
return
|
|
13435
|
+
await environmentSession.hello();
|
|
13436
|
+
return environmentSession;
|
|
13156
13437
|
}
|
|
13157
13438
|
async activateEnvironment(environmentId) {
|
|
13158
13439
|
await this.request(`/orchestrator/runtime/environments/${environmentId}/activate`, {
|
|
@@ -15091,19 +15372,27 @@ function matchesPattern(text, matcher) {
|
|
|
15091
15372
|
function assertMatches(label, text, includes = [], excludes = []) {
|
|
15092
15373
|
for (const matcher of includes) {
|
|
15093
15374
|
if (!matchesPattern(text, matcher)) {
|
|
15094
|
-
throw new Error(
|
|
15095
|
-
${
|
|
15375
|
+
throw new Error(
|
|
15376
|
+
`${label} did not match ${matcherToString(matcher)}:
|
|
15377
|
+
${text}`
|
|
15378
|
+
);
|
|
15096
15379
|
}
|
|
15097
15380
|
}
|
|
15098
15381
|
for (const matcher of excludes) {
|
|
15099
15382
|
if (matchesPattern(text, matcher)) {
|
|
15100
|
-
throw new Error(
|
|
15101
|
-
${
|
|
15383
|
+
throw new Error(
|
|
15384
|
+
`${label} matched forbidden ${matcherToString(matcher)}:
|
|
15385
|
+
${text}`
|
|
15386
|
+
);
|
|
15102
15387
|
}
|
|
15103
15388
|
}
|
|
15104
15389
|
}
|
|
15105
15390
|
function buildArtifactDir(baseDir, suiteName = "granular-agent-evals") {
|
|
15106
|
-
return path__default.default.join(
|
|
15391
|
+
return path__default.default.join(
|
|
15392
|
+
baseDir || path__default.default.join(process.cwd(), "test-artifacts"),
|
|
15393
|
+
suiteName,
|
|
15394
|
+
timestampId()
|
|
15395
|
+
);
|
|
15107
15396
|
}
|
|
15108
15397
|
function createTimestampedArtifactDirectory(options) {
|
|
15109
15398
|
return buildArtifactDir(options?.baseDir, options?.suiteName);
|
|
@@ -15117,17 +15406,16 @@ function buildScenarioSteps(scenario) {
|
|
|
15117
15406
|
return scenario.steps;
|
|
15118
15407
|
}
|
|
15119
15408
|
if (!scenario.request) {
|
|
15120
|
-
throw new Error(
|
|
15409
|
+
throw new Error(
|
|
15410
|
+
`Scenario ${scenario.id} must provide either request or steps`
|
|
15411
|
+
);
|
|
15121
15412
|
}
|
|
15122
15413
|
const compatibilityStep = {
|
|
15123
15414
|
id: scenario.id,
|
|
15124
15415
|
request: scenario.request,
|
|
15125
15416
|
human: scenario.human,
|
|
15126
15417
|
expect: scenario.expect,
|
|
15127
|
-
inspect: [
|
|
15128
|
-
...asArray2(scenario.inspect),
|
|
15129
|
-
...asArray2(scenario.verify)
|
|
15130
|
-
],
|
|
15418
|
+
inspect: [...asArray2(scenario.inspect), ...asArray2(scenario.verify)],
|
|
15131
15419
|
check: scenario.check,
|
|
15132
15420
|
maxIterations: scenario.maxIterations,
|
|
15133
15421
|
setup: {
|
|
@@ -15145,22 +15433,27 @@ function buildAssistantHistoryContent(entry) {
|
|
|
15145
15433
|
${entry.content}`);
|
|
15146
15434
|
if (entry.jobStatus) parts.push(`[Job status]
|
|
15147
15435
|
${entry.jobStatus}`);
|
|
15148
|
-
if (entry.jobResultPreview)
|
|
15436
|
+
if (entry.jobResultPreview)
|
|
15437
|
+
parts.push(`[Job result]
|
|
15149
15438
|
${entry.jobResultPreview}`);
|
|
15150
15439
|
if (entry.error) parts.push(`[Job error]
|
|
15151
15440
|
${entry.error}`);
|
|
15152
15441
|
return parts.join("\n\n") || entry.content;
|
|
15153
15442
|
}
|
|
15154
15443
|
function buildHistory(entries) {
|
|
15155
|
-
return entries.reduce(
|
|
15156
|
-
|
|
15157
|
-
if (entry.
|
|
15444
|
+
return entries.reduce(
|
|
15445
|
+
(history, entry) => {
|
|
15446
|
+
if (entry.role === "user") {
|
|
15447
|
+
if (entry.content.trim())
|
|
15448
|
+
history.push({ role: "user", content: entry.content });
|
|
15449
|
+
return history;
|
|
15450
|
+
}
|
|
15451
|
+
const content = buildAssistantHistoryContent(entry).trim();
|
|
15452
|
+
if (content) history.push({ role: "assistant", content });
|
|
15158
15453
|
return history;
|
|
15159
|
-
}
|
|
15160
|
-
|
|
15161
|
-
|
|
15162
|
-
return history;
|
|
15163
|
-
}, []);
|
|
15454
|
+
},
|
|
15455
|
+
[]
|
|
15456
|
+
);
|
|
15164
15457
|
}
|
|
15165
15458
|
function getOpenPromptsFromDoc(liveDoc) {
|
|
15166
15459
|
const jobsById = asRecord4(asRecord4(liveDoc?.jobs)?.byId) || {};
|
|
@@ -15169,7 +15462,8 @@ function getOpenPromptsFromDoc(liveDoc) {
|
|
|
15169
15462
|
const promptRecords = asRecord4(asRecord4(job)?.prompts) || {};
|
|
15170
15463
|
for (const raw of Object.values(promptRecords)) {
|
|
15171
15464
|
const record = asRecord4(raw);
|
|
15172
|
-
if (!record || record.status !== "open" || typeof record.promptId !== "string")
|
|
15465
|
+
if (!record || record.status !== "open" || typeof record.promptId !== "string")
|
|
15466
|
+
continue;
|
|
15173
15467
|
const prompt = normalizePrompt({
|
|
15174
15468
|
promptId: record.promptId,
|
|
15175
15469
|
kind: record.kind,
|
|
@@ -15213,7 +15507,9 @@ ${prompt.message || ""}`;
|
|
|
15213
15507
|
return resolvePromptAnswer(prompt, rawAnswer);
|
|
15214
15508
|
}
|
|
15215
15509
|
if (fallback) return fallback({ prompt, history });
|
|
15216
|
-
throw new Error(
|
|
15510
|
+
throw new Error(
|
|
15511
|
+
`No scripted prompt responder matched prompt ${prompt.id}: ${promptText}`
|
|
15512
|
+
);
|
|
15217
15513
|
};
|
|
15218
15514
|
}
|
|
15219
15515
|
function extractJsonObject(text) {
|
|
@@ -15237,13 +15533,19 @@ function modelOutputInstruction() {
|
|
|
15237
15533
|
].join("\n");
|
|
15238
15534
|
}
|
|
15239
15535
|
function createOpenAIChatTurnGenerator(options) {
|
|
15240
|
-
const baseUrl = (options.baseUrl || "https://api.openai.com/v1").replace(
|
|
15536
|
+
const baseUrl = (options.baseUrl || "https://api.openai.com/v1").replace(
|
|
15537
|
+
/\/$/,
|
|
15538
|
+
""
|
|
15539
|
+
);
|
|
15241
15540
|
const model = options.model || "gpt-5-mini";
|
|
15242
15541
|
return async (input) => {
|
|
15243
15542
|
const messages = [
|
|
15244
|
-
{
|
|
15543
|
+
{
|
|
15544
|
+
role: "system",
|
|
15545
|
+
content: `${input.systemPrompt}
|
|
15245
15546
|
|
|
15246
|
-
${modelOutputInstruction()}`
|
|
15547
|
+
${modelOutputInstruction()}`
|
|
15548
|
+
},
|
|
15247
15549
|
...input.history,
|
|
15248
15550
|
{ role: "user", content: input.request }
|
|
15249
15551
|
];
|
|
@@ -15272,10 +15574,14 @@ ${modelOutputInstruction()}` },
|
|
|
15272
15574
|
await sleep2(500 * attempt);
|
|
15273
15575
|
continue;
|
|
15274
15576
|
}
|
|
15275
|
-
throw new Error(
|
|
15577
|
+
throw new Error(
|
|
15578
|
+
`OpenAI chat generation failed: ${response.status} ${errorText}`
|
|
15579
|
+
);
|
|
15276
15580
|
}
|
|
15277
15581
|
const raw = await response.json();
|
|
15278
|
-
const content = asRecord4(
|
|
15582
|
+
const content = asRecord4(
|
|
15583
|
+
asRecord4(raw.choices?.[0])?.message
|
|
15584
|
+
)?.content;
|
|
15279
15585
|
const text = typeof content === "string" ? content : Array.isArray(content) ? content.map((part) => asRecord4(part)?.text || "").join("") : "";
|
|
15280
15586
|
const parsed = extractJsonObject(text);
|
|
15281
15587
|
if (!parsed) {
|
|
@@ -15293,7 +15599,9 @@ ${text}`);
|
|
|
15293
15599
|
};
|
|
15294
15600
|
} catch (error) {
|
|
15295
15601
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
15296
|
-
if (attempt < 3 && /socket connection was closed unexpectedly|ECONNRESET|network|timed out/i.test(
|
|
15602
|
+
if (attempt < 3 && /socket connection was closed unexpectedly|ECONNRESET|network|timed out/i.test(
|
|
15603
|
+
lastError.message
|
|
15604
|
+
)) {
|
|
15297
15605
|
await sleep2(500 * attempt);
|
|
15298
15606
|
continue;
|
|
15299
15607
|
}
|
|
@@ -15315,7 +15623,10 @@ async function withTimeout(promise, ms, label) {
|
|
|
15315
15623
|
return await Promise.race([
|
|
15316
15624
|
promise,
|
|
15317
15625
|
new Promise((_, reject) => {
|
|
15318
|
-
timeoutId = setTimeout(
|
|
15626
|
+
timeoutId = setTimeout(
|
|
15627
|
+
() => reject(new Error(`${label} timed out after ${ms}ms`)),
|
|
15628
|
+
ms
|
|
15629
|
+
);
|
|
15319
15630
|
})
|
|
15320
15631
|
]);
|
|
15321
15632
|
} finally {
|
|
@@ -15325,7 +15636,9 @@ async function withTimeout(promise, ms, label) {
|
|
|
15325
15636
|
function getActionSummary(liveDoc, jobId) {
|
|
15326
15637
|
const jobsById = asRecord4(asRecord4(liveDoc?.jobs)?.byId) || {};
|
|
15327
15638
|
const job = asRecord4(jobsById[jobId]);
|
|
15328
|
-
return Array.isArray(job?.actionSummary) ? job.actionSummary.filter(
|
|
15639
|
+
return Array.isArray(job?.actionSummary) ? job.actionSummary.filter(
|
|
15640
|
+
(line) => typeof line === "string"
|
|
15641
|
+
) : [];
|
|
15329
15642
|
}
|
|
15330
15643
|
function normalizeHeapSnapshot2(heap) {
|
|
15331
15644
|
return {
|
|
@@ -15355,12 +15668,20 @@ async function waitForJobOutcome(input) {
|
|
|
15355
15668
|
const startedAt = Date.now();
|
|
15356
15669
|
while (Date.now() - startedAt < input.timeoutMs) {
|
|
15357
15670
|
const liveDoc = cloneJson(input.environment.document);
|
|
15358
|
-
const prompts = filterPromptsByBoundary(
|
|
15671
|
+
const prompts = filterPromptsByBoundary(
|
|
15672
|
+
liveDoc,
|
|
15673
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
15674
|
+
input.boundaryTimestamp
|
|
15675
|
+
);
|
|
15359
15676
|
if (prompts.length > 0) {
|
|
15360
15677
|
return { kind: "prompt", prompts, liveDoc, stdout, stderr };
|
|
15361
15678
|
}
|
|
15362
15679
|
try {
|
|
15363
|
-
const result = await withTimeout(
|
|
15680
|
+
const result = await withTimeout(
|
|
15681
|
+
input.job.result,
|
|
15682
|
+
input.pollIntervalMs,
|
|
15683
|
+
`job ${input.job.id} tick`
|
|
15684
|
+
);
|
|
15364
15685
|
return { kind: "completed", result, liveDoc, stdout, stderr };
|
|
15365
15686
|
} catch (error) {
|
|
15366
15687
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -15426,7 +15747,9 @@ function buildResultReport(result) {
|
|
|
15426
15747
|
...result.actionSummary.length ? result.actionSummary.map((line) => `- ${line}`) : ["- None"],
|
|
15427
15748
|
"",
|
|
15428
15749
|
"## Prompt Interactions",
|
|
15429
|
-
...result.promptInteractions.length ? result.promptInteractions.map(
|
|
15750
|
+
...result.promptInteractions.length ? result.promptInteractions.map(
|
|
15751
|
+
(interaction) => `- [${interaction.type}] ${interaction.message || interaction.title} -> ${JSON.stringify(interaction.answer)}`
|
|
15752
|
+
) : ["- None"],
|
|
15430
15753
|
"",
|
|
15431
15754
|
...stepSection,
|
|
15432
15755
|
"## Raw Files",
|
|
@@ -15441,7 +15764,9 @@ function buildSuiteIndex(results) {
|
|
|
15441
15764
|
const lines = [
|
|
15442
15765
|
"# Agent Eval Report Index",
|
|
15443
15766
|
"",
|
|
15444
|
-
...results.map(
|
|
15767
|
+
...results.map(
|
|
15768
|
+
(result) => `- [${result.scenario.id}](./${result.scenario.id}/REPORT.md) - ${result.status}`
|
|
15769
|
+
)
|
|
15445
15770
|
];
|
|
15446
15771
|
return `${lines.join("\n")}
|
|
15447
15772
|
`;
|
|
@@ -15455,7 +15780,7 @@ async function applySetup(setup, context) {
|
|
|
15455
15780
|
await context.environment.recordObjects(setup.records);
|
|
15456
15781
|
}
|
|
15457
15782
|
if (setup.effects?.length) {
|
|
15458
|
-
await context.environment.
|
|
15783
|
+
await context.granular.ontology(context.environment.sandboxId).effects.registerMany(setup.effects);
|
|
15459
15784
|
}
|
|
15460
15785
|
if (setup.run) {
|
|
15461
15786
|
await setup.run(context);
|
|
@@ -15477,6 +15802,7 @@ async function runAgentEvalSuite(options) {
|
|
|
15477
15802
|
);
|
|
15478
15803
|
try {
|
|
15479
15804
|
await applySetup(scenario.setup, {
|
|
15805
|
+
granular: options.harness.granular,
|
|
15480
15806
|
conversation,
|
|
15481
15807
|
environment: conversation.environment,
|
|
15482
15808
|
turnDir: conversation.artifactDir
|
|
@@ -15489,14 +15815,19 @@ async function runAgentEvalSuite(options) {
|
|
|
15489
15815
|
conversation,
|
|
15490
15816
|
request: step.request,
|
|
15491
15817
|
prepare: async (ctx) => {
|
|
15492
|
-
await applySetup(step.setup,
|
|
15818
|
+
await applySetup(step.setup, {
|
|
15819
|
+
granular: options.harness.granular,
|
|
15820
|
+
...ctx
|
|
15821
|
+
});
|
|
15493
15822
|
},
|
|
15494
15823
|
human: step.human,
|
|
15495
15824
|
maxIterations: step.maxIterations,
|
|
15496
15825
|
autoAnswerPrompts: step.autoAnswerPrompts
|
|
15497
15826
|
});
|
|
15498
15827
|
if ("prompts" in completed) {
|
|
15499
|
-
throw new Error(
|
|
15828
|
+
throw new Error(
|
|
15829
|
+
`Scenario ${scenario.id} step ${index + 1} paused for human input instead of completing automatically`
|
|
15830
|
+
);
|
|
15500
15831
|
}
|
|
15501
15832
|
const inspectionResults = [];
|
|
15502
15833
|
const stepChecks = asArray2(step.check);
|
|
@@ -15512,12 +15843,23 @@ async function runAgentEvalSuite(options) {
|
|
|
15512
15843
|
actionSummary: completed.actionSummary,
|
|
15513
15844
|
promptInteractions: completed.promptInteractions,
|
|
15514
15845
|
result: completed.result,
|
|
15515
|
-
heap: normalizeHeapSnapshot2(
|
|
15516
|
-
|
|
15846
|
+
heap: normalizeHeapSnapshot2(
|
|
15847
|
+
asRecord4(
|
|
15848
|
+
cloneJson(conversation.environment.document)?.heap
|
|
15849
|
+
)
|
|
15850
|
+
),
|
|
15851
|
+
openPrompts: getOpenPromptsFromDoc(
|
|
15852
|
+
cloneJson(conversation.environment.document)
|
|
15853
|
+
),
|
|
15517
15854
|
liveDoc: cloneJson(conversation.environment.document),
|
|
15518
15855
|
inspect: async (code) => {
|
|
15519
|
-
const
|
|
15520
|
-
|
|
15856
|
+
const session = conversation.environment;
|
|
15857
|
+
const job = await session.submitJob(code);
|
|
15858
|
+
return withTimeout(
|
|
15859
|
+
job.result,
|
|
15860
|
+
9e4,
|
|
15861
|
+
`inspection job ${job.id}`
|
|
15862
|
+
);
|
|
15521
15863
|
},
|
|
15522
15864
|
assertMatches
|
|
15523
15865
|
};
|
|
@@ -15575,7 +15917,9 @@ async function runAgentEvalSuite(options) {
|
|
|
15575
15917
|
}
|
|
15576
15918
|
const lastStep = stepResults[stepResults.length - 1];
|
|
15577
15919
|
if (!lastStep) {
|
|
15578
|
-
throw new Error(
|
|
15920
|
+
throw new Error(
|
|
15921
|
+
`Scenario ${scenario.id} produced no completed steps`
|
|
15922
|
+
);
|
|
15579
15923
|
}
|
|
15580
15924
|
const result = {
|
|
15581
15925
|
scenario,
|
|
@@ -15589,9 +15933,18 @@ async function runAgentEvalSuite(options) {
|
|
|
15589
15933
|
steps: stepResults,
|
|
15590
15934
|
turnDir: conversation.artifactDir
|
|
15591
15935
|
};
|
|
15592
|
-
await writeJson(
|
|
15593
|
-
|
|
15594
|
-
|
|
15936
|
+
await writeJson(
|
|
15937
|
+
path__default.default.join(conversation.artifactDir, "result.json"),
|
|
15938
|
+
result
|
|
15939
|
+
);
|
|
15940
|
+
await writeJson(
|
|
15941
|
+
path__default.default.join(conversation.artifactDir, "report.json"),
|
|
15942
|
+
result
|
|
15943
|
+
);
|
|
15944
|
+
await promises.writeFile(
|
|
15945
|
+
path__default.default.join(conversation.artifactDir, "REPORT.md"),
|
|
15946
|
+
buildResultReport(result)
|
|
15947
|
+
);
|
|
15595
15948
|
finalResult = result;
|
|
15596
15949
|
} catch (error) {
|
|
15597
15950
|
const failureMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -15613,7 +15966,10 @@ async function runAgentEvalSuite(options) {
|
|
|
15613
15966
|
await ensureDir(failed.turnDir);
|
|
15614
15967
|
await writeJson(path__default.default.join(failed.turnDir, "result.json"), failed);
|
|
15615
15968
|
await writeJson(path__default.default.join(failed.turnDir, "report.json"), failed);
|
|
15616
|
-
await promises.writeFile(
|
|
15969
|
+
await promises.writeFile(
|
|
15970
|
+
path__default.default.join(failed.turnDir, "REPORT.md"),
|
|
15971
|
+
buildResultReport(failed)
|
|
15972
|
+
);
|
|
15617
15973
|
finalResult = failed;
|
|
15618
15974
|
} finally {
|
|
15619
15975
|
await options.harness.closeConversation(conversation);
|
|
@@ -15634,12 +15990,21 @@ async function runAgentEvalSuite(options) {
|
|
|
15634
15990
|
}
|
|
15635
15991
|
results.push(finalResult);
|
|
15636
15992
|
}
|
|
15637
|
-
await writeJson(
|
|
15638
|
-
|
|
15993
|
+
await writeJson(
|
|
15994
|
+
path__default.default.join(options.harness.artifactDir, "summary.json"),
|
|
15995
|
+
results
|
|
15996
|
+
);
|
|
15997
|
+
await promises.writeFile(
|
|
15998
|
+
path__default.default.join(options.harness.artifactDir, "REPORT_INDEX.md"),
|
|
15999
|
+
buildSuiteIndex(results)
|
|
16000
|
+
);
|
|
15639
16001
|
return { artifactDir: options.harness.artifactDir, results };
|
|
15640
16002
|
}
|
|
15641
16003
|
function createAgentEvalHarness(options) {
|
|
15642
|
-
const artifactDir = buildArtifactDir(
|
|
16004
|
+
const artifactDir = buildArtifactDir(
|
|
16005
|
+
options.artifactBaseDir,
|
|
16006
|
+
options.suiteName
|
|
16007
|
+
);
|
|
15643
16008
|
const controllerBudgets = {
|
|
15644
16009
|
...DEFAULT_CONTROLLER_BUDGETS,
|
|
15645
16010
|
...options.controllerBudgets || {}
|
|
@@ -15651,7 +16016,9 @@ function createAgentEvalHarness(options) {
|
|
|
15651
16016
|
await ensureDir(artifactDir);
|
|
15652
16017
|
const clientId = `${slugify(label)}-${Date.now()}`;
|
|
15653
16018
|
if (!options.openEnvironment && !options.environmentId) {
|
|
15654
|
-
throw new Error(
|
|
16019
|
+
throw new Error(
|
|
16020
|
+
"createAgentEvalHarness requires either environmentId or openEnvironment"
|
|
16021
|
+
);
|
|
15655
16022
|
}
|
|
15656
16023
|
const environment = options.openEnvironment ? await options.openEnvironment({ label, clientId }) : await options.granular.createSession({
|
|
15657
16024
|
environmentId: options.environmentId,
|
|
@@ -15675,12 +16042,15 @@ function createAgentEvalHarness(options) {
|
|
|
15675
16042
|
}
|
|
15676
16043
|
async function closeConversation(conversation) {
|
|
15677
16044
|
try {
|
|
15678
|
-
await options.granular.closeSession(
|
|
16045
|
+
await options.granular.closeSession(
|
|
16046
|
+
conversation.environment.sessionId,
|
|
16047
|
+
conversation.environment
|
|
16048
|
+
);
|
|
15679
16049
|
} catch {
|
|
15680
16050
|
}
|
|
15681
16051
|
}
|
|
15682
|
-
async function runCheckJob(code,
|
|
15683
|
-
const job = await
|
|
16052
|
+
async function runCheckJob(code, session) {
|
|
16053
|
+
const job = await session.submitJob(code);
|
|
15684
16054
|
return withTimeout(job.result, jobTimeoutMs, `check job ${job.id}`);
|
|
15685
16055
|
}
|
|
15686
16056
|
function buildCheckContext(conversation, completed, turnDir) {
|
|
@@ -15725,8 +16095,12 @@ function createAgentEvalHarness(options) {
|
|
|
15725
16095
|
async function resumePendingTurn(pending, responder) {
|
|
15726
16096
|
const prompt = pending.prompts[0];
|
|
15727
16097
|
if (!prompt) throw new Error("Pending turn has no prompts to answer");
|
|
15728
|
-
const answer = await responder({
|
|
15729
|
-
|
|
16098
|
+
const answer = await responder({
|
|
16099
|
+
prompt,
|
|
16100
|
+
history: pending.promptInteractions
|
|
16101
|
+
});
|
|
16102
|
+
const session = pending.conversation.environment;
|
|
16103
|
+
await session.answerPrompt(prompt.id, answer);
|
|
15730
16104
|
pending.promptInteractions.push({
|
|
15731
16105
|
promptId: prompt.id,
|
|
15732
16106
|
type: prompt.type,
|
|
@@ -15750,7 +16124,9 @@ function createAgentEvalHarness(options) {
|
|
|
15750
16124
|
};
|
|
15751
16125
|
}
|
|
15752
16126
|
await sleep2(350);
|
|
15753
|
-
const liveDoc = cloneJson(
|
|
16127
|
+
const liveDoc = cloneJson(
|
|
16128
|
+
pending.conversation.environment.document
|
|
16129
|
+
);
|
|
15754
16130
|
const presentation = resolveJobPresentation({
|
|
15755
16131
|
jobId: pending.job.id,
|
|
15756
16132
|
result: resumed.result,
|
|
@@ -15795,25 +16171,40 @@ function createAgentEvalHarness(options) {
|
|
|
15795
16171
|
await conversation.environment.recordObjects(input.prepareRecords);
|
|
15796
16172
|
}
|
|
15797
16173
|
if (input.prepareTools?.length) {
|
|
15798
|
-
await conversation.environment.
|
|
16174
|
+
await options.granular.ontology(conversation.environment.sandboxId).effects.registerMany(input.prepareTools);
|
|
15799
16175
|
}
|
|
15800
16176
|
if (input.prepare) {
|
|
15801
|
-
await input.prepare({
|
|
16177
|
+
await input.prepare({
|
|
16178
|
+
conversation,
|
|
16179
|
+
environment: conversation.environment,
|
|
16180
|
+
turnDir
|
|
16181
|
+
});
|
|
15802
16182
|
}
|
|
15803
16183
|
const boundaryTimestamp = Date.now();
|
|
15804
16184
|
conversation.history.push({ role: "user", content: input.request });
|
|
15805
|
-
await writeJson(path__default.default.join(turnDir, "request.json"), {
|
|
16185
|
+
await writeJson(path__default.default.join(turnDir, "request.json"), {
|
|
16186
|
+
request: input.request,
|
|
16187
|
+
boundaryTimestamp
|
|
16188
|
+
});
|
|
15806
16189
|
let iteration = 0;
|
|
15807
16190
|
let noProgressCount = 0;
|
|
15808
16191
|
let previousSnapshot = null;
|
|
15809
16192
|
let latestCheckpoint = null;
|
|
15810
16193
|
const maxIterations = input.maxIterations || controllerBudgets.maxIterations;
|
|
15811
16194
|
const autoAnswerPrompts = input.autoAnswerPrompts ?? true;
|
|
15812
|
-
const baselineClosureId = getCurrentClosureId(
|
|
16195
|
+
const baselineClosureId = getCurrentClosureId(
|
|
16196
|
+
cloneJson(conversation.environment.document)
|
|
16197
|
+
);
|
|
15813
16198
|
while (iteration < maxIterations) {
|
|
15814
16199
|
const liveDoc = cloneJson(conversation.environment.document);
|
|
15815
|
-
const pendingPrompts = filterPromptsByBoundary(
|
|
15816
|
-
|
|
16200
|
+
const pendingPrompts = filterPromptsByBoundary(
|
|
16201
|
+
liveDoc,
|
|
16202
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
16203
|
+
boundaryTimestamp
|
|
16204
|
+
);
|
|
16205
|
+
const workflowFocus = projectWorkflowFocus(liveDoc, pendingPrompts, {
|
|
16206
|
+
boundaryTimestamp
|
|
16207
|
+
});
|
|
15817
16208
|
const systemPrompt = buildGranularAgentSystemPrompt({
|
|
15818
16209
|
domainDocumentation: await conversation.environment.getDomainDocumentation(),
|
|
15819
16210
|
sessionContext: {
|
|
@@ -15824,8 +16215,12 @@ function createAgentEvalHarness(options) {
|
|
|
15824
16215
|
heapSummary: projectHeapSummary(liveDoc, {
|
|
15825
16216
|
focus: workflowFocus
|
|
15826
16217
|
}),
|
|
15827
|
-
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
15828
|
-
|
|
16218
|
+
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
16219
|
+
boundaryTimestamp
|
|
16220
|
+
}),
|
|
16221
|
+
workflowSummary: projectWorkflowSummary(liveDoc, pendingPrompts, {
|
|
16222
|
+
boundaryTimestamp
|
|
16223
|
+
}),
|
|
15829
16224
|
tools: conversation.environment.getEffects().map((tool) => ({
|
|
15830
16225
|
name: tool.name,
|
|
15831
16226
|
description: tool.description,
|
|
@@ -15835,14 +16230,23 @@ function createAgentEvalHarness(options) {
|
|
|
15835
16230
|
})),
|
|
15836
16231
|
checkpoint: latestCheckpoint
|
|
15837
16232
|
});
|
|
15838
|
-
const request = iteration === 0 ? input.request : buildContinuationInstruction(
|
|
15839
|
-
|
|
15840
|
-
|
|
15841
|
-
|
|
15842
|
-
|
|
15843
|
-
|
|
15844
|
-
|
|
15845
|
-
|
|
16233
|
+
const request = iteration === 0 ? input.request : buildContinuationInstruction(
|
|
16234
|
+
buildContinuationPreview(latestCheckpoint, noProgressCount)
|
|
16235
|
+
);
|
|
16236
|
+
const generation = await withTimeout(
|
|
16237
|
+
generateTurnWithRepair(options.generator, {
|
|
16238
|
+
systemPrompt,
|
|
16239
|
+
history: buildHistory(conversation.history),
|
|
16240
|
+
request,
|
|
16241
|
+
attempt: 1
|
|
16242
|
+
}),
|
|
16243
|
+
chatTimeoutMs,
|
|
16244
|
+
`chat generation for ${conversation.label} iteration ${iteration + 1}`
|
|
16245
|
+
);
|
|
16246
|
+
await writeJson(
|
|
16247
|
+
path__default.default.join(turnDir, `iteration-${iteration + 1}-generation.json`),
|
|
16248
|
+
generation
|
|
16249
|
+
);
|
|
15846
16250
|
if (!generation.code) {
|
|
15847
16251
|
const responseText2 = generation.reply?.trim() || "Done.";
|
|
15848
16252
|
conversation.history.push({ role: "assistant", content: responseText2 });
|
|
@@ -15858,12 +16262,18 @@ function createAgentEvalHarness(options) {
|
|
|
15858
16262
|
result: generation.reply?.trim() || responseText2
|
|
15859
16263
|
};
|
|
15860
16264
|
if (input.verification) {
|
|
15861
|
-
completed.verification = await runInspection(
|
|
16265
|
+
completed.verification = await runInspection(
|
|
16266
|
+
conversation,
|
|
16267
|
+
input.verification,
|
|
16268
|
+
completed,
|
|
16269
|
+
turnDir
|
|
16270
|
+
);
|
|
15862
16271
|
}
|
|
15863
16272
|
await writeJson(path__default.default.join(turnDir, "result.json"), completed);
|
|
15864
16273
|
return completed;
|
|
15865
16274
|
}
|
|
15866
|
-
const
|
|
16275
|
+
const session = conversation.environment;
|
|
16276
|
+
const job = await session.submitJob(generation.code);
|
|
15867
16277
|
const outcome = await waitForJobOutcome({
|
|
15868
16278
|
environment: conversation.environment,
|
|
15869
16279
|
job,
|
|
@@ -15888,7 +16298,9 @@ function createAgentEvalHarness(options) {
|
|
|
15888
16298
|
};
|
|
15889
16299
|
}
|
|
15890
16300
|
if (!input.human) {
|
|
15891
|
-
throw new Error(
|
|
16301
|
+
throw new Error(
|
|
16302
|
+
"This turn reached a human prompt but no responder was provided"
|
|
16303
|
+
);
|
|
15892
16304
|
}
|
|
15893
16305
|
let pending = {
|
|
15894
16306
|
conversation,
|
|
@@ -15910,16 +16322,25 @@ function createAgentEvalHarness(options) {
|
|
|
15910
16322
|
continue;
|
|
15911
16323
|
}
|
|
15912
16324
|
if (input.verification) {
|
|
15913
|
-
resumed.verification = await runInspection(
|
|
16325
|
+
resumed.verification = await runInspection(
|
|
16326
|
+
conversation,
|
|
16327
|
+
input.verification,
|
|
16328
|
+
resumed,
|
|
16329
|
+
turnDir
|
|
16330
|
+
);
|
|
15914
16331
|
}
|
|
15915
16332
|
return resumed;
|
|
15916
16333
|
}
|
|
15917
16334
|
}
|
|
15918
16335
|
if (outcome.kind !== "completed") {
|
|
15919
|
-
throw new Error(
|
|
16336
|
+
throw new Error(
|
|
16337
|
+
"Unexpected non-completed outcome after prompt handling"
|
|
16338
|
+
);
|
|
15920
16339
|
}
|
|
15921
16340
|
await sleep2(350);
|
|
15922
|
-
const settledLiveDoc = cloneJson(
|
|
16341
|
+
const settledLiveDoc = cloneJson(
|
|
16342
|
+
conversation.environment.document
|
|
16343
|
+
);
|
|
15923
16344
|
const sessionHeap = normalizeHeapSnapshot2(asRecord4(settledLiveDoc?.heap));
|
|
15924
16345
|
const presentation = resolveJobPresentation({
|
|
15925
16346
|
jobId: job.id,
|
|
@@ -15940,7 +16361,11 @@ function createAgentEvalHarness(options) {
|
|
|
15940
16361
|
baselineClosureId,
|
|
15941
16362
|
currentClosureId: getCurrentClosureId(settledLiveDoc),
|
|
15942
16363
|
liveDoc: settledLiveDoc,
|
|
15943
|
-
pendingPrompts: filterPromptsByBoundary(
|
|
16364
|
+
pendingPrompts: filterPromptsByBoundary(
|
|
16365
|
+
settledLiveDoc,
|
|
16366
|
+
getOpenPromptsFromDoc(settledLiveDoc),
|
|
16367
|
+
boundaryTimestamp
|
|
16368
|
+
),
|
|
15944
16369
|
projectionOptions: { boundaryTimestamp },
|
|
15945
16370
|
latestResponseText: responseText,
|
|
15946
16371
|
previousSnapshot,
|
|
@@ -15965,12 +16390,15 @@ function createAgentEvalHarness(options) {
|
|
|
15965
16390
|
jobStatus: "succeeded",
|
|
15966
16391
|
jobResultPreview: JSON.stringify(outcome.result, null, 2)
|
|
15967
16392
|
});
|
|
15968
|
-
await writeJson(
|
|
15969
|
-
|
|
15970
|
-
|
|
15971
|
-
|
|
15972
|
-
|
|
15973
|
-
|
|
16393
|
+
await writeJson(
|
|
16394
|
+
path__default.default.join(turnDir, `iteration-${iteration + 1}-result.json`),
|
|
16395
|
+
{
|
|
16396
|
+
responseText,
|
|
16397
|
+
continuation,
|
|
16398
|
+
actionSummary: latestCheckpoint.latestActionSummary,
|
|
16399
|
+
result: outcome.result
|
|
16400
|
+
}
|
|
16401
|
+
);
|
|
15974
16402
|
if (!continuation.shouldContinue) {
|
|
15975
16403
|
const completed = {
|
|
15976
16404
|
conversation,
|
|
@@ -15985,17 +16413,25 @@ function createAgentEvalHarness(options) {
|
|
|
15985
16413
|
result: outcome.result
|
|
15986
16414
|
};
|
|
15987
16415
|
if (input.verification) {
|
|
15988
|
-
completed.verification = await runInspection(
|
|
16416
|
+
completed.verification = await runInspection(
|
|
16417
|
+
conversation,
|
|
16418
|
+
input.verification,
|
|
16419
|
+
completed,
|
|
16420
|
+
turnDir
|
|
16421
|
+
);
|
|
15989
16422
|
}
|
|
15990
16423
|
await writeJson(path__default.default.join(turnDir, "result.json"), completed);
|
|
15991
16424
|
return completed;
|
|
15992
16425
|
}
|
|
15993
16426
|
iteration += 1;
|
|
15994
16427
|
}
|
|
15995
|
-
throw new Error(
|
|
16428
|
+
throw new Error(
|
|
16429
|
+
`Turn exceeded iteration budget (${maxIterations}) for ${conversation.label}`
|
|
16430
|
+
);
|
|
15996
16431
|
}
|
|
15997
16432
|
return {
|
|
15998
16433
|
artifactDir,
|
|
16434
|
+
granular: options.granular,
|
|
15999
16435
|
openConversation,
|
|
16000
16436
|
closeConversation,
|
|
16001
16437
|
runTurn,
|
|
@@ -16031,12 +16467,11 @@ function createAgentTester(options) {
|
|
|
16031
16467
|
}
|
|
16032
16468
|
if ("connect" in options.target && !connectSeeded) {
|
|
16033
16469
|
connectSeeded = true;
|
|
16034
|
-
const environment = await granular.
|
|
16035
|
-
...options.target.connect
|
|
16036
|
-
clientId
|
|
16470
|
+
const environment = await granular.openEnvironment({
|
|
16471
|
+
...options.target.connect
|
|
16037
16472
|
});
|
|
16038
16473
|
resolvedEnvironmentId = environment.environmentId;
|
|
16039
|
-
return environment;
|
|
16474
|
+
return environment.sessions.create({ clientId });
|
|
16040
16475
|
}
|
|
16041
16476
|
if ("createEnvironment" in options.target && !resolvedEnvironmentId) {
|
|
16042
16477
|
const envData = await granular.environments.create(
|