@granular-software/sdk 0.4.30 → 0.4.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -20
- package/dist/agent-evals.d.mts +7 -6
- package/dist/agent-evals.d.ts +7 -6
- package/dist/agent-evals.js +721 -286
- package/dist/agent-evals.js.map +1 -1
- package/dist/agent-evals.mjs +721 -286
- package/dist/agent-evals.mjs.map +1 -1
- package/dist/cli/index.js +537 -251
- package/dist/{client-BVvOMfln.d.mts → client-C2Gk641P.d.mts} +291 -163
- package/dist/{client-BVvOMfln.d.ts → client-C2Gk641P.d.ts} +291 -163
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +479 -196
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +478 -197
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/agent-evals.mjs
CHANGED
|
@@ -3985,7 +3985,11 @@ var WSClient = class {
|
|
|
3985
3985
|
return null;
|
|
3986
3986
|
}
|
|
3987
3987
|
try {
|
|
3988
|
-
const
|
|
3988
|
+
const payloadSegment = parts[1];
|
|
3989
|
+
if (!payloadSegment) {
|
|
3990
|
+
return null;
|
|
3991
|
+
}
|
|
3992
|
+
const payloadRaw = this.decodeBase64Url(payloadSegment);
|
|
3989
3993
|
const payload = JSON.parse(payloadRaw);
|
|
3990
3994
|
if (typeof payload.exp !== "number" || !Number.isFinite(payload.exp)) {
|
|
3991
3995
|
return null;
|
|
@@ -4649,27 +4653,27 @@ var Session = class {
|
|
|
4649
4653
|
}
|
|
4650
4654
|
async publishTools(tools, revision = "1.0.0") {
|
|
4651
4655
|
throw new Error(
|
|
4652
|
-
"Environment-scoped effect publication was removed. Declare effects in the manifest and register live handlers with granular.
|
|
4656
|
+
"Environment-scoped effect publication was removed. Declare effects in the manifest and register live handlers with granular.ontology(environment.sandboxId).effects.registerMany(effects)."
|
|
4653
4657
|
);
|
|
4654
4658
|
}
|
|
4655
4659
|
async publishEffect(effect) {
|
|
4656
4660
|
throw new Error(
|
|
4657
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4661
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.register(effect)."
|
|
4658
4662
|
);
|
|
4659
4663
|
}
|
|
4660
4664
|
async publishEffects(effects) {
|
|
4661
4665
|
throw new Error(
|
|
4662
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4666
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.registerMany(effects)."
|
|
4663
4667
|
);
|
|
4664
4668
|
}
|
|
4665
4669
|
async unpublishEffect(name) {
|
|
4666
4670
|
throw new Error(
|
|
4667
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4671
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.unregister(effectName)."
|
|
4668
4672
|
);
|
|
4669
4673
|
}
|
|
4670
4674
|
async unpublishAllEffects() {
|
|
4671
4675
|
throw new Error(
|
|
4672
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4676
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.clear()."
|
|
4673
4677
|
);
|
|
4674
4678
|
}
|
|
4675
4679
|
/**
|
|
@@ -5062,6 +5066,7 @@ import { ${allImports} } from "./sandbox-tools";
|
|
|
5062
5066
|
this.eventListeners.set(event, []);
|
|
5063
5067
|
}
|
|
5064
5068
|
this.eventListeners.get(event).push(handler);
|
|
5069
|
+
return () => this.off(event, handler);
|
|
5065
5070
|
}
|
|
5066
5071
|
/**
|
|
5067
5072
|
* Unsubscribe from session events
|
|
@@ -5518,6 +5523,16 @@ var JobImplementation = class {
|
|
|
5518
5523
|
handler(message);
|
|
5519
5524
|
}
|
|
5520
5525
|
}
|
|
5526
|
+
return () => {
|
|
5527
|
+
const handlers = this.eventListeners.get(event);
|
|
5528
|
+
if (!handlers) {
|
|
5529
|
+
return;
|
|
5530
|
+
}
|
|
5531
|
+
this.eventListeners.set(
|
|
5532
|
+
event,
|
|
5533
|
+
handlers.filter((current) => current !== handler)
|
|
5534
|
+
);
|
|
5535
|
+
};
|
|
5521
5536
|
}
|
|
5522
5537
|
replayAgentMessage(message) {
|
|
5523
5538
|
this.captureAgentMessage(message);
|
|
@@ -11693,6 +11708,22 @@ function normalizeHeapSnapshot(raw) {
|
|
|
11693
11708
|
updatedAt: typeof heap.updatedAt === "number" ? heap.updatedAt : Date.now()
|
|
11694
11709
|
};
|
|
11695
11710
|
}
|
|
11711
|
+
function deriveRuntimeBaseUrl(apiEndpoint) {
|
|
11712
|
+
try {
|
|
11713
|
+
const endpoint = new URL(apiEndpoint);
|
|
11714
|
+
const graphqlSuffix = "/orchestrator/graphql";
|
|
11715
|
+
if (endpoint.pathname.endsWith(graphqlSuffix)) {
|
|
11716
|
+
endpoint.pathname = endpoint.pathname.slice(0, -graphqlSuffix.length);
|
|
11717
|
+
} else if (endpoint.pathname.endsWith("/graphql")) {
|
|
11718
|
+
endpoint.pathname = endpoint.pathname.slice(0, -"/graphql".length);
|
|
11719
|
+
}
|
|
11720
|
+
endpoint.search = "";
|
|
11721
|
+
endpoint.hash = "";
|
|
11722
|
+
return endpoint.toString().replace(/\/$/, "");
|
|
11723
|
+
} catch {
|
|
11724
|
+
return apiEndpoint.replace(/\/orchestrator\/graphql$/, "").replace(/\/$/, "");
|
|
11725
|
+
}
|
|
11726
|
+
}
|
|
11696
11727
|
function normalizeSubject(subject) {
|
|
11697
11728
|
const granularId = subject.granularId || subject.subjectId;
|
|
11698
11729
|
const userId = subject.userId || subject.identityId || granularId;
|
|
@@ -11732,12 +11763,13 @@ function normalizeEnvironmentData(environment) {
|
|
|
11732
11763
|
tracking: environment.tracking || buildPolicy
|
|
11733
11764
|
};
|
|
11734
11765
|
}
|
|
11735
|
-
var Environment = class
|
|
11766
|
+
var Environment = class {
|
|
11767
|
+
granular;
|
|
11736
11768
|
envData;
|
|
11737
11769
|
_apiKey;
|
|
11738
11770
|
_apiEndpoint;
|
|
11739
|
-
constructor(
|
|
11740
|
-
|
|
11771
|
+
constructor(granular, envData, apiKey, apiEndpoint) {
|
|
11772
|
+
this.granular = granular;
|
|
11741
11773
|
this.envData = envData;
|
|
11742
11774
|
this._apiKey = apiKey;
|
|
11743
11775
|
this._apiEndpoint = apiEndpoint;
|
|
@@ -11778,35 +11810,126 @@ var Environment = class extends Session {
|
|
|
11778
11810
|
get permissionProfileId() {
|
|
11779
11811
|
return this.envData.permissionProfileId;
|
|
11780
11812
|
}
|
|
11813
|
+
/** The current build policy backing this environment */
|
|
11814
|
+
get buildPolicy() {
|
|
11815
|
+
return this.envData.buildPolicy;
|
|
11816
|
+
}
|
|
11817
|
+
/** The current update state relative to the followed tag */
|
|
11818
|
+
get updateState() {
|
|
11819
|
+
return this.envData.updateState;
|
|
11820
|
+
}
|
|
11821
|
+
/** Convenience flag for whether this environment trails the current tag target */
|
|
11822
|
+
get isOutdated() {
|
|
11823
|
+
return this.envData.updateState === "update_available";
|
|
11824
|
+
}
|
|
11825
|
+
/** The followed tag name when this environment is tag-tracked */
|
|
11826
|
+
get tag() {
|
|
11827
|
+
return this.envData.tag?.name || this.envData.buildPolicy.tagName || null;
|
|
11828
|
+
}
|
|
11781
11829
|
/** The GraphQL API endpoint URL */
|
|
11782
11830
|
get apiEndpoint() {
|
|
11783
11831
|
return this._apiEndpoint;
|
|
11784
11832
|
}
|
|
11833
|
+
/** Internal auth token used for control-plane and runtime fallback requests */
|
|
11834
|
+
get authToken() {
|
|
11835
|
+
return this._apiKey;
|
|
11836
|
+
}
|
|
11837
|
+
/** Base runtime URL derived from the GraphQL endpoint */
|
|
11838
|
+
get runtimeBaseUrl() {
|
|
11839
|
+
return this.getRuntimeBaseUrl();
|
|
11840
|
+
}
|
|
11841
|
+
get sessions() {
|
|
11842
|
+
return {
|
|
11843
|
+
list: async (options) => this.listSessions(options?.status || "active"),
|
|
11844
|
+
create: async (options) => this.createSession(options),
|
|
11845
|
+
connect: async (sessionId, options) => this.connectSession(sessionId, options),
|
|
11846
|
+
reopen: async (sessionId, options) => this.reopenSession(sessionId, options),
|
|
11847
|
+
close: async (sessionId, session) => this.closeSession(sessionId, session)
|
|
11848
|
+
};
|
|
11849
|
+
}
|
|
11850
|
+
get data() {
|
|
11851
|
+
return {
|
|
11852
|
+
record: async (record) => this.recordObject(record),
|
|
11853
|
+
recordMany: async (records, options) => this.recordObjects(records, options),
|
|
11854
|
+
import: async (records, options) => this.enqueueRecordImport(records, options),
|
|
11855
|
+
listImports: async (status) => this.listRecordImports(status),
|
|
11856
|
+
getImport: async (importId) => this.getRecordImport(importId),
|
|
11857
|
+
getImportSummary: async () => this.getRecordImportSummary(),
|
|
11858
|
+
cancelImport: async (importId) => this.cancelRecordImport(importId),
|
|
11859
|
+
getAwaitingCount: async () => this.getAwaitingRecordCount()
|
|
11860
|
+
};
|
|
11861
|
+
}
|
|
11862
|
+
get feedback() {
|
|
11863
|
+
return {
|
|
11864
|
+
list: async () => this.listFeedback()
|
|
11865
|
+
};
|
|
11866
|
+
}
|
|
11785
11867
|
/**
|
|
11786
|
-
*
|
|
11787
|
-
*
|
|
11788
|
-
*
|
|
11789
|
-
*
|
|
11868
|
+
* Sessionless environments do not own a live transport, so disconnecting the
|
|
11869
|
+
* environment handle itself is a no-op. This keeps the public surface
|
|
11870
|
+
* symmetric with `EnvironmentSession.disconnect()` and lets callers always
|
|
11871
|
+
* clean up safely without tracking whether they currently hold an environment
|
|
11872
|
+
* or a session.
|
|
11790
11873
|
*/
|
|
11791
|
-
|
|
11792
|
-
const doc = this.document;
|
|
11793
|
-
return normalizeHeapSnapshot(doc?.heap);
|
|
11874
|
+
async disconnect() {
|
|
11794
11875
|
}
|
|
11795
|
-
|
|
11796
|
-
|
|
11797
|
-
const
|
|
11798
|
-
|
|
11799
|
-
|
|
11800
|
-
|
|
11801
|
-
|
|
11802
|
-
|
|
11803
|
-
|
|
11804
|
-
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
|
|
11876
|
+
async listSessions(status = "active") {
|
|
11877
|
+
if (status === "all") {
|
|
11878
|
+
const [active, closed] = await Promise.all([
|
|
11879
|
+
this.granular.listOpenSessions({ environmentId: this.environmentId }),
|
|
11880
|
+
this.granular.listClosedSessions({ environmentId: this.environmentId })
|
|
11881
|
+
]);
|
|
11882
|
+
return [...active, ...closed].sort(
|
|
11883
|
+
(left, right) => Date.parse(right.lastSeenAt) - Date.parse(left.lastSeenAt)
|
|
11884
|
+
);
|
|
11885
|
+
}
|
|
11886
|
+
return status === "closed" ? this.granular.listClosedSessions({ environmentId: this.environmentId }) : this.granular.listOpenSessions({ environmentId: this.environmentId });
|
|
11887
|
+
}
|
|
11888
|
+
async createSession(options) {
|
|
11889
|
+
return this.granular.createSession({
|
|
11890
|
+
environmentId: this.environmentId,
|
|
11891
|
+
clientId: options?.clientId,
|
|
11892
|
+
initialHeap: options?.initialHeap
|
|
11893
|
+
});
|
|
11894
|
+
}
|
|
11895
|
+
async connectSession(sessionId, options) {
|
|
11896
|
+
const session = await this.granular["connectSession"]({
|
|
11897
|
+
sessionId,
|
|
11898
|
+
clientId: options?.clientId
|
|
11899
|
+
});
|
|
11900
|
+
if (session.environmentId !== this.environmentId) {
|
|
11901
|
+
await session.disconnect().catch(() => {
|
|
11902
|
+
session.disconnectTransport();
|
|
11903
|
+
});
|
|
11904
|
+
throw new Error(
|
|
11905
|
+
`Session ${sessionId} belongs to environment ${session.environmentId}, not ${this.environmentId}.`
|
|
11906
|
+
);
|
|
11907
|
+
}
|
|
11908
|
+
return session;
|
|
11909
|
+
}
|
|
11910
|
+
async reopenSession(sessionId, options) {
|
|
11911
|
+
const session = await this.granular.reopenSession(sessionId, {
|
|
11912
|
+
clientId: options?.clientId
|
|
11913
|
+
});
|
|
11914
|
+
if (session.environmentId !== this.environmentId) {
|
|
11915
|
+
await session.disconnect().catch(() => {
|
|
11916
|
+
session.disconnectTransport();
|
|
11917
|
+
});
|
|
11918
|
+
throw new Error(
|
|
11919
|
+
`Session ${sessionId} belongs to environment ${session.environmentId}, not ${this.environmentId}.`
|
|
11920
|
+
);
|
|
11809
11921
|
}
|
|
11922
|
+
return session;
|
|
11923
|
+
}
|
|
11924
|
+
async closeSession(sessionId, session) {
|
|
11925
|
+
await this.granular.closeSession(sessionId, session);
|
|
11926
|
+
}
|
|
11927
|
+
async listFeedback() {
|
|
11928
|
+
const response = await this.controlPlaneRequest(`/control/environments/${this.environmentId}/feedback`);
|
|
11929
|
+
return Array.isArray(response.items) ? response.items : [];
|
|
11930
|
+
}
|
|
11931
|
+
getRuntimeBaseUrl() {
|
|
11932
|
+
return deriveRuntimeBaseUrl(this._apiEndpoint);
|
|
11810
11933
|
}
|
|
11811
11934
|
async controlPlaneRequest(path2, options = {}) {
|
|
11812
11935
|
const runtimeBase = this.getRuntimeBaseUrl();
|
|
@@ -11827,95 +11950,6 @@ var Environment = class extends Session {
|
|
|
11827
11950
|
}
|
|
11828
11951
|
return response.json();
|
|
11829
11952
|
}
|
|
11830
|
-
/**
|
|
11831
|
-
* Close the session and disconnect from the sandbox.
|
|
11832
|
-
*
|
|
11833
|
-
* Sends `client.goodbye` over WebSocket first, then issues an HTTP fallback
|
|
11834
|
-
* to the runtime goodbye endpoint if no definitive WS-side runtime notify
|
|
11835
|
-
* acknowledgement was observed.
|
|
11836
|
-
*/
|
|
11837
|
-
async disconnect() {
|
|
11838
|
-
let wsNotifiedRuntime = false;
|
|
11839
|
-
try {
|
|
11840
|
-
const goodbye = await this.rpc(
|
|
11841
|
-
"client.goodbye",
|
|
11842
|
-
{
|
|
11843
|
-
timestamp: Date.now()
|
|
11844
|
-
}
|
|
11845
|
-
);
|
|
11846
|
-
wsNotifiedRuntime = Boolean(goodbye?.ok && goodbye?.via);
|
|
11847
|
-
} catch {
|
|
11848
|
-
wsNotifiedRuntime = false;
|
|
11849
|
-
}
|
|
11850
|
-
if (!wsNotifiedRuntime) {
|
|
11851
|
-
try {
|
|
11852
|
-
const runtimeBase = this.getRuntimeBaseUrl();
|
|
11853
|
-
await fetch(
|
|
11854
|
-
`${runtimeBase}/orchestrator/runtime/environments/${this.environmentId}/session-goodbye`,
|
|
11855
|
-
{
|
|
11856
|
-
method: "POST",
|
|
11857
|
-
headers: {
|
|
11858
|
-
"Content-Type": "application/json",
|
|
11859
|
-
Authorization: `Bearer ${this._apiKey}`,
|
|
11860
|
-
Connection: "close"
|
|
11861
|
-
},
|
|
11862
|
-
body: JSON.stringify({
|
|
11863
|
-
reason: "sdk_disconnect_http_fallback",
|
|
11864
|
-
sessionId: this.client.currentSessionId
|
|
11865
|
-
})
|
|
11866
|
-
}
|
|
11867
|
-
);
|
|
11868
|
-
} catch {
|
|
11869
|
-
}
|
|
11870
|
-
}
|
|
11871
|
-
this.client.disconnect();
|
|
11872
|
-
}
|
|
11873
|
-
/**
|
|
11874
|
-
* Close only the socket transport without sending `client.goodbye`.
|
|
11875
|
-
*
|
|
11876
|
-
* Use this when the caller intends to immediately reattach to the same
|
|
11877
|
-
* session after an unexpected disconnect.
|
|
11878
|
-
*/
|
|
11879
|
-
disconnectTransport() {
|
|
11880
|
-
this.client.disconnect();
|
|
11881
|
-
}
|
|
11882
|
-
// ==================== GRAPH CONTAINER READINESS ====================
|
|
11883
|
-
/** The last known graph container status, updated by checkReadiness() or on heartbeat */
|
|
11884
|
-
graphContainerStatus = null;
|
|
11885
|
-
/**
|
|
11886
|
-
* Check if the graph container is ready and warm.
|
|
11887
|
-
*
|
|
11888
|
-
* Sends a lightweight heartbeat RPC to the Session DO which internally
|
|
11889
|
-
* pings the FalkorDB container. The response includes `graphContainerStatus`,
|
|
11890
|
-
* which is stored locally and emitted as a `readiness` event.
|
|
11891
|
-
*
|
|
11892
|
-
* Use this method to proactively warm the graph container before any
|
|
11893
|
-
* GraphQL query that requires it, or to poll the container's state in
|
|
11894
|
-
* the background.
|
|
11895
|
-
*
|
|
11896
|
-
* @returns The current graph container status object
|
|
11897
|
-
*
|
|
11898
|
-
* @example
|
|
11899
|
-
* ```typescript
|
|
11900
|
-
* const status = await env.checkReadiness();
|
|
11901
|
-
* console.log(status.status); // 'hot' | 'warming' | 'unknown'
|
|
11902
|
-
*
|
|
11903
|
-
* // Or listen for live updates
|
|
11904
|
-
* env.on('readiness', (status) => {
|
|
11905
|
-
* console.log('Graph is now:', status.status);
|
|
11906
|
-
* });
|
|
11907
|
-
* ```
|
|
11908
|
-
*/
|
|
11909
|
-
async checkReadiness() {
|
|
11910
|
-
const result = await this.client.call("client.heartbeat", {});
|
|
11911
|
-
const containerStatus = result?.graphContainerStatus ?? {
|
|
11912
|
-
lastKeepAliveAt: Date.now(),
|
|
11913
|
-
status: "unknown"
|
|
11914
|
-
};
|
|
11915
|
-
this.graphContainerStatus = containerStatus;
|
|
11916
|
-
this.emit("readiness", containerStatus);
|
|
11917
|
-
return containerStatus;
|
|
11918
|
-
}
|
|
11919
11953
|
// ==================== ID ↔ GRAPH PATH MAPPING ====================
|
|
11920
11954
|
/**
|
|
11921
11955
|
* Convert a class name + real-world ID into a unique graph path.
|
|
@@ -12045,7 +12079,9 @@ var Environment = class extends Session {
|
|
|
12045
12079
|
}
|
|
12046
12080
|
);
|
|
12047
12081
|
if (result.errors?.length) {
|
|
12048
|
-
throw new Error(
|
|
12082
|
+
throw new Error(
|
|
12083
|
+
`defineRelationship failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`
|
|
12084
|
+
);
|
|
12049
12085
|
}
|
|
12050
12086
|
return result.data.at.define_relationship;
|
|
12051
12087
|
}
|
|
@@ -12081,7 +12117,9 @@ var Environment = class extends Session {
|
|
|
12081
12117
|
{ path: modelPath }
|
|
12082
12118
|
);
|
|
12083
12119
|
if (result.errors?.length) {
|
|
12084
|
-
throw new Error(
|
|
12120
|
+
throw new Error(
|
|
12121
|
+
`getRelationships failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`
|
|
12122
|
+
);
|
|
12085
12123
|
}
|
|
12086
12124
|
return result.data?.model?.relationships || [];
|
|
12087
12125
|
}
|
|
@@ -12120,7 +12158,7 @@ var Environment = class extends Session {
|
|
|
12120
12158
|
{ target: targetPath }
|
|
12121
12159
|
);
|
|
12122
12160
|
if (result.errors?.length) {
|
|
12123
|
-
throw new Error(`attach failed: ${result.errors[0]
|
|
12161
|
+
throw new Error(`attach failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`);
|
|
12124
12162
|
}
|
|
12125
12163
|
}
|
|
12126
12164
|
/**
|
|
@@ -12155,7 +12193,7 @@ var Environment = class extends Session {
|
|
|
12155
12193
|
}`
|
|
12156
12194
|
);
|
|
12157
12195
|
if (result.errors?.length) {
|
|
12158
|
-
throw new Error(`detach failed: ${result.errors[0]
|
|
12196
|
+
throw new Error(`detach failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`);
|
|
12159
12197
|
}
|
|
12160
12198
|
}
|
|
12161
12199
|
/**
|
|
@@ -12182,7 +12220,9 @@ var Environment = class extends Session {
|
|
|
12182
12220
|
}`
|
|
12183
12221
|
);
|
|
12184
12222
|
if (result.errors?.length) {
|
|
12185
|
-
throw new Error(
|
|
12223
|
+
throw new Error(
|
|
12224
|
+
`listRelated failed: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`
|
|
12225
|
+
);
|
|
12186
12226
|
}
|
|
12187
12227
|
return result.data?.at?.at?.list_related || [];
|
|
12188
12228
|
}
|
|
@@ -12275,7 +12315,7 @@ var Environment = class extends Session {
|
|
|
12275
12315
|
async _runGraphql(query, label) {
|
|
12276
12316
|
const result = await this.graphql(query);
|
|
12277
12317
|
if (result.errors?.length) {
|
|
12278
|
-
throw new Error(`${label}: ${result.errors[0]
|
|
12318
|
+
throw new Error(`${label}: ${result.errors[0]?.message ?? "Unknown GraphQL error"}`);
|
|
12279
12319
|
}
|
|
12280
12320
|
return result.data;
|
|
12281
12321
|
}
|
|
@@ -12620,7 +12660,11 @@ var Environment = class extends Session {
|
|
|
12620
12660
|
*/
|
|
12621
12661
|
async recordObject(options) {
|
|
12622
12662
|
const results = await this.recordObjects([options]);
|
|
12623
|
-
|
|
12663
|
+
const result = results[0];
|
|
12664
|
+
if (!result) {
|
|
12665
|
+
throw new Error("recordObject: no result returned for record");
|
|
12666
|
+
}
|
|
12667
|
+
return result;
|
|
12624
12668
|
}
|
|
12625
12669
|
/**
|
|
12626
12670
|
* Batch version of `recordObject()`.
|
|
@@ -12672,7 +12716,13 @@ var Environment = class extends Session {
|
|
|
12672
12716
|
);
|
|
12673
12717
|
}
|
|
12674
12718
|
for (let index = 0; index < items.length; index += 1) {
|
|
12675
|
-
|
|
12719
|
+
const item = items[index];
|
|
12720
|
+
if (!item) {
|
|
12721
|
+
throw new Error(
|
|
12722
|
+
`recordObjects: chunk ${plan.chunkIndex + 1} returned an empty result at index ${index}`
|
|
12723
|
+
);
|
|
12724
|
+
}
|
|
12725
|
+
results[plan.offset + index] = item;
|
|
12676
12726
|
}
|
|
12677
12727
|
if (onChunk) {
|
|
12678
12728
|
const info = {
|
|
@@ -12776,36 +12826,186 @@ var Environment = class extends Session {
|
|
|
12776
12826
|
}
|
|
12777
12827
|
);
|
|
12778
12828
|
}
|
|
12779
|
-
|
|
12829
|
+
};
|
|
12830
|
+
var EnvironmentSession = class extends Session {
|
|
12831
|
+
environment;
|
|
12832
|
+
/** The last known graph container status, updated by checkReadiness() or on heartbeat */
|
|
12833
|
+
graphContainerStatus = null;
|
|
12834
|
+
constructor(client, environment, clientId) {
|
|
12835
|
+
super(client, clientId);
|
|
12836
|
+
this.environment = environment;
|
|
12837
|
+
}
|
|
12838
|
+
get environmentId() {
|
|
12839
|
+
return this.environment.environmentId;
|
|
12840
|
+
}
|
|
12841
|
+
get sandboxId() {
|
|
12842
|
+
return this.environment.sandboxId;
|
|
12843
|
+
}
|
|
12844
|
+
get ontologyId() {
|
|
12845
|
+
return this.environment.ontologyId;
|
|
12846
|
+
}
|
|
12847
|
+
get subjectId() {
|
|
12848
|
+
return this.environment.subjectId;
|
|
12849
|
+
}
|
|
12850
|
+
get envName() {
|
|
12851
|
+
return this.environment.envName;
|
|
12852
|
+
}
|
|
12853
|
+
get versionId() {
|
|
12854
|
+
return this.environment.versionId;
|
|
12855
|
+
}
|
|
12856
|
+
get granularId() {
|
|
12857
|
+
return this.environment.granularId;
|
|
12858
|
+
}
|
|
12859
|
+
get permissionProfileId() {
|
|
12860
|
+
return this.environment.permissionProfileId;
|
|
12861
|
+
}
|
|
12862
|
+
get apiEndpoint() {
|
|
12863
|
+
return this.environment.apiEndpoint;
|
|
12864
|
+
}
|
|
12865
|
+
get data() {
|
|
12866
|
+
return this.environment.data;
|
|
12867
|
+
}
|
|
12868
|
+
get feedback() {
|
|
12869
|
+
return this.environment.feedback;
|
|
12870
|
+
}
|
|
12780
12871
|
/**
|
|
12781
|
-
*
|
|
12872
|
+
* Return a plain JS snapshot of the synced session heap.
|
|
12782
12873
|
*/
|
|
12783
|
-
|
|
12784
|
-
|
|
12874
|
+
getHeap() {
|
|
12875
|
+
const doc = this.document;
|
|
12876
|
+
return normalizeHeapSnapshot(doc?.heap);
|
|
12877
|
+
}
|
|
12878
|
+
async graphql(query, variables) {
|
|
12879
|
+
return this.environment.graphql(query, variables);
|
|
12880
|
+
}
|
|
12881
|
+
async defineRelationship(options) {
|
|
12882
|
+
return this.environment.defineRelationship(options);
|
|
12883
|
+
}
|
|
12884
|
+
async getRelationships(modelPath) {
|
|
12885
|
+
return this.environment.getRelationships(modelPath);
|
|
12886
|
+
}
|
|
12887
|
+
async attach(modelPath, submodelPath, targetPath) {
|
|
12888
|
+
return this.environment.attach(modelPath, submodelPath, targetPath);
|
|
12889
|
+
}
|
|
12890
|
+
async detach(modelPath, submodelPath, targetPath) {
|
|
12891
|
+
return this.environment.detach(modelPath, submodelPath, targetPath);
|
|
12892
|
+
}
|
|
12893
|
+
async listRelated(modelPath, submodelPath) {
|
|
12894
|
+
return this.environment.listRelated(modelPath, submodelPath);
|
|
12895
|
+
}
|
|
12896
|
+
async applyManifest(manifest) {
|
|
12897
|
+
return this.environment.applyManifest(manifest);
|
|
12898
|
+
}
|
|
12899
|
+
async recordObject(options) {
|
|
12900
|
+
return this.environment.recordObject(options);
|
|
12901
|
+
}
|
|
12902
|
+
async recordObjects(records, options) {
|
|
12903
|
+
return this.environment.recordObjects(records, options);
|
|
12904
|
+
}
|
|
12905
|
+
async enqueueRecordImport(records, options = {}) {
|
|
12906
|
+
return this.environment.enqueueRecordImport(records, options);
|
|
12907
|
+
}
|
|
12908
|
+
async listRecordImports(status) {
|
|
12909
|
+
return this.environment.listRecordImports(status);
|
|
12910
|
+
}
|
|
12911
|
+
async getRecordImportSummary() {
|
|
12912
|
+
return this.environment.getRecordImportSummary();
|
|
12913
|
+
}
|
|
12914
|
+
async getAwaitingRecordCount() {
|
|
12915
|
+
return this.environment.getAwaitingRecordCount();
|
|
12916
|
+
}
|
|
12917
|
+
async getRecordImport(importId) {
|
|
12918
|
+
return this.environment.getRecordImport(importId);
|
|
12919
|
+
}
|
|
12920
|
+
async cancelRecordImport(importId) {
|
|
12921
|
+
return this.environment.cancelRecordImport(importId);
|
|
12922
|
+
}
|
|
12923
|
+
async listFeedback() {
|
|
12924
|
+
return this.environment.listFeedback();
|
|
12785
12925
|
}
|
|
12786
12926
|
/**
|
|
12787
|
-
*
|
|
12927
|
+
* Close the session and disconnect from the sandbox.
|
|
12928
|
+
*
|
|
12929
|
+
* Sends `client.goodbye` over WebSocket first, then issues an HTTP fallback
|
|
12930
|
+
* to the runtime goodbye endpoint if no definitive WS-side runtime notify
|
|
12931
|
+
* acknowledgement was observed.
|
|
12788
12932
|
*/
|
|
12789
|
-
async
|
|
12790
|
-
|
|
12933
|
+
async disconnect() {
|
|
12934
|
+
let wsNotifiedRuntime = false;
|
|
12935
|
+
try {
|
|
12936
|
+
const goodbye = await this.rpc(
|
|
12937
|
+
"client.goodbye",
|
|
12938
|
+
{
|
|
12939
|
+
timestamp: Date.now()
|
|
12940
|
+
}
|
|
12941
|
+
);
|
|
12942
|
+
wsNotifiedRuntime = Boolean(goodbye?.ok && goodbye?.via);
|
|
12943
|
+
} catch {
|
|
12944
|
+
wsNotifiedRuntime = false;
|
|
12945
|
+
}
|
|
12946
|
+
if (!wsNotifiedRuntime) {
|
|
12947
|
+
try {
|
|
12948
|
+
await fetch(
|
|
12949
|
+
`${this.environment.runtimeBaseUrl}/orchestrator/runtime/environments/${this.environmentId}/session-goodbye`,
|
|
12950
|
+
{
|
|
12951
|
+
method: "POST",
|
|
12952
|
+
headers: {
|
|
12953
|
+
"Content-Type": "application/json",
|
|
12954
|
+
Authorization: `Bearer ${this.environment.authToken}`,
|
|
12955
|
+
Connection: "close"
|
|
12956
|
+
},
|
|
12957
|
+
body: JSON.stringify({
|
|
12958
|
+
reason: "sdk_disconnect_http_fallback",
|
|
12959
|
+
sessionId: this.client.currentSessionId
|
|
12960
|
+
})
|
|
12961
|
+
}
|
|
12962
|
+
);
|
|
12963
|
+
} catch {
|
|
12964
|
+
}
|
|
12965
|
+
}
|
|
12966
|
+
this.client.disconnect();
|
|
12791
12967
|
}
|
|
12792
12968
|
/**
|
|
12793
|
-
*
|
|
12969
|
+
* Close only the socket transport without sending `client.goodbye`.
|
|
12794
12970
|
*/
|
|
12795
|
-
|
|
12796
|
-
|
|
12971
|
+
disconnectTransport() {
|
|
12972
|
+
this.client.disconnect();
|
|
12797
12973
|
}
|
|
12798
12974
|
/**
|
|
12799
|
-
*
|
|
12975
|
+
* Backwards-compatible alias for `disconnect()`.
|
|
12800
12976
|
*/
|
|
12801
|
-
async
|
|
12802
|
-
|
|
12977
|
+
async close() {
|
|
12978
|
+
await this.disconnect();
|
|
12803
12979
|
}
|
|
12804
12980
|
/**
|
|
12805
|
-
*
|
|
12981
|
+
* Check if the graph container is ready and warm.
|
|
12806
12982
|
*/
|
|
12807
|
-
async
|
|
12808
|
-
|
|
12983
|
+
async checkReadiness() {
|
|
12984
|
+
const result = await this.client.call("client.heartbeat", {});
|
|
12985
|
+
const containerStatus = result?.graphContainerStatus ?? {
|
|
12986
|
+
lastKeepAliveAt: Date.now(),
|
|
12987
|
+
status: "unknown"
|
|
12988
|
+
};
|
|
12989
|
+
this.graphContainerStatus = containerStatus;
|
|
12990
|
+
this.emit("readiness", containerStatus);
|
|
12991
|
+
return containerStatus;
|
|
12992
|
+
}
|
|
12993
|
+
};
|
|
12994
|
+
var OntologyHandle = class {
|
|
12995
|
+
granular;
|
|
12996
|
+
ontologyNameOrId;
|
|
12997
|
+
constructor(granular, ontologyNameOrId) {
|
|
12998
|
+
this.granular = granular;
|
|
12999
|
+
this.ontologyNameOrId = ontologyNameOrId;
|
|
13000
|
+
}
|
|
13001
|
+
get effects() {
|
|
13002
|
+
return {
|
|
13003
|
+
register: async (effect) => this.granular.registerEffect(this.ontologyNameOrId, effect),
|
|
13004
|
+
registerMany: async (effects) => this.granular.registerEffects(this.ontologyNameOrId, effects),
|
|
13005
|
+
unregister: async (name) => this.granular.unregisterEffect(this.ontologyNameOrId, name),
|
|
13006
|
+
clear: async () => this.granular.unregisterAllEffects(this.ontologyNameOrId),
|
|
13007
|
+
disconnect: async () => this.granular.disconnectEffects(this.ontologyNameOrId)
|
|
13008
|
+
};
|
|
12809
13009
|
}
|
|
12810
13010
|
};
|
|
12811
13011
|
var Granular = class _Granular {
|
|
@@ -12842,6 +13042,12 @@ var Granular = class _Granular {
|
|
|
12842
13042
|
this.onReconnectError = options.onReconnectError;
|
|
12843
13043
|
this.httpUrl = this.apiUrl.replace(/^wss:\/\//, "https://").replace(/^ws:\/\//, "http://").replace(/\/ws$/, "");
|
|
12844
13044
|
}
|
|
13045
|
+
/**
|
|
13046
|
+
* Return an ontology-scoped handle for effects and other ontology-level APIs.
|
|
13047
|
+
*/
|
|
13048
|
+
ontology(ontologyNameOrId) {
|
|
13049
|
+
return new OntologyHandle(this, ontologyNameOrId);
|
|
13050
|
+
}
|
|
12845
13051
|
/**
|
|
12846
13052
|
* Records/upserts a user and prepares them for sandbox connections
|
|
12847
13053
|
*
|
|
@@ -12878,7 +13084,23 @@ var Granular = class _Granular {
|
|
|
12878
13084
|
permissions: options.permissions || []
|
|
12879
13085
|
});
|
|
12880
13086
|
}
|
|
13087
|
+
/**
|
|
13088
|
+
* Alias for `recordUser()` with user-facing naming that matches upsert semantics.
|
|
13089
|
+
*/
|
|
13090
|
+
async upsertUser(options) {
|
|
13091
|
+
return this.recordUser(options);
|
|
13092
|
+
}
|
|
12881
13093
|
async resolveConnectUser(options) {
|
|
13094
|
+
const providedIdentityCount = [
|
|
13095
|
+
Boolean(options.user),
|
|
13096
|
+
Boolean(options.userId),
|
|
13097
|
+
Boolean(options.granularId)
|
|
13098
|
+
].filter(Boolean).length;
|
|
13099
|
+
if (providedIdentityCount !== 1) {
|
|
13100
|
+
throw new Error(
|
|
13101
|
+
"openEnvironment() requires exactly one of userId, granularId, or a user object returned by recordUser()."
|
|
13102
|
+
);
|
|
13103
|
+
}
|
|
12882
13104
|
if (options.user) {
|
|
12883
13105
|
const user = normalizeUser(options.user);
|
|
12884
13106
|
return {
|
|
@@ -12915,56 +13137,85 @@ var Granular = class _Granular {
|
|
|
12915
13137
|
};
|
|
12916
13138
|
}
|
|
12917
13139
|
throw new Error(
|
|
12918
|
-
"
|
|
13140
|
+
"openEnvironment() requires either userId, granularId, or a user object returned by recordUser()."
|
|
12919
13141
|
);
|
|
12920
13142
|
}
|
|
12921
13143
|
/**
|
|
12922
|
-
*
|
|
12923
|
-
*
|
|
12924
|
-
* Effects are registered at the sandbox level via `granular.registerEffect()`
|
|
12925
|
-
* or `granular.registerEffects()`. Sessions pick up live availability from
|
|
12926
|
-
* the sandbox registry automatically.
|
|
12927
|
-
*
|
|
12928
|
-
* @param options - Connection options
|
|
12929
|
-
* @returns An active environment session
|
|
13144
|
+
* Open or resolve an ontology environment for one user without opening a session.
|
|
12930
13145
|
*
|
|
12931
13146
|
* @example
|
|
12932
13147
|
* ```typescript
|
|
12933
|
-
* const environment = await granular.
|
|
13148
|
+
* const environment = await granular.openEnvironment({
|
|
12934
13149
|
* ontology: 'my-ontology',
|
|
12935
|
-
*
|
|
13150
|
+
* tag: 'dev',
|
|
12936
13151
|
* userId: 'user_123',
|
|
12937
13152
|
* permissions: ['agent'],
|
|
12938
13153
|
* });
|
|
12939
13154
|
*
|
|
12940
|
-
* await
|
|
12941
|
-
*
|
|
12942
|
-
*
|
|
12943
|
-
*
|
|
12944
|
-
* handler: async () => 'Hello!',
|
|
13155
|
+
* await environment.data.record({
|
|
13156
|
+
* className: 'customer',
|
|
13157
|
+
* id: 'acme',
|
|
13158
|
+
* fields: { name: 'Acme' },
|
|
12945
13159
|
* });
|
|
12946
13160
|
*
|
|
12947
|
-
*
|
|
12948
|
-
* const job = await
|
|
12949
|
-
*
|
|
12950
|
-
* return await tools.greet({});
|
|
12951
|
-
* `);
|
|
12952
|
-
*
|
|
12953
|
-
* console.log(await job.result); // 'Hello!'
|
|
13161
|
+
* const session = await environment.sessions.create();
|
|
13162
|
+
* const job = await session.submitJob(`return "hello";`);
|
|
13163
|
+
* console.log(await job.result);
|
|
12954
13164
|
* ```
|
|
12955
13165
|
*/
|
|
13166
|
+
async openEnvironment(options) {
|
|
13167
|
+
const envData = await this.resolveOpenEnvironmentData(
|
|
13168
|
+
options,
|
|
13169
|
+
"openEnvironment"
|
|
13170
|
+
);
|
|
13171
|
+
return this.bindEnvironmentHandle(envData);
|
|
13172
|
+
}
|
|
13173
|
+
/**
|
|
13174
|
+
* Deprecated compatibility alias for `openEnvironment()`.
|
|
13175
|
+
*
|
|
13176
|
+
* `connect()` no longer opens a runtime session automatically.
|
|
13177
|
+
*/
|
|
12956
13178
|
async connect(options) {
|
|
12957
|
-
|
|
13179
|
+
return this.openEnvironment({
|
|
13180
|
+
...options,
|
|
13181
|
+
tag: this.resolveRequestedTag(options, "connect"),
|
|
13182
|
+
permissions: options.permissions || options.user?.permissions || []
|
|
13183
|
+
});
|
|
13184
|
+
}
|
|
13185
|
+
resolveRequestedTag(options, methodName) {
|
|
13186
|
+
const tag = options.tag?.trim() || options.tagName?.trim() || options.environment?.trim();
|
|
13187
|
+
if (!tag) {
|
|
13188
|
+
throw new Error(`${methodName}() requires \`tag\`.`);
|
|
13189
|
+
}
|
|
13190
|
+
return tag;
|
|
13191
|
+
}
|
|
13192
|
+
buildManagedEnvironmentName(tag, versionId) {
|
|
13193
|
+
return `__sdk__${tag}__${versionId}`;
|
|
13194
|
+
}
|
|
13195
|
+
matchesTagTrackedEnvironment(environment, tagName, tagId) {
|
|
13196
|
+
const environmentTagName = environment.tag?.name || environment.buildPolicy.tagName || null;
|
|
13197
|
+
return environment.tagId === tagId || environmentTagName === tagName || environment.environment === tagName || environment.envName === tagName || environment.environment === this.buildManagedEnvironmentName(tagName, environment.versionId) || environment.envName === this.buildManagedEnvironmentName(tagName, environment.versionId);
|
|
13198
|
+
}
|
|
13199
|
+
sortEnvironmentsByRecency(environments) {
|
|
13200
|
+
return [...environments].sort(
|
|
13201
|
+
(left, right) => right.updatedAt - left.updatedAt
|
|
13202
|
+
);
|
|
13203
|
+
}
|
|
13204
|
+
async resolveOpenEnvironmentData(options, methodName) {
|
|
12958
13205
|
const ontology = options.ontology;
|
|
12959
13206
|
if (!ontology) {
|
|
12960
|
-
throw new Error(
|
|
13207
|
+
throw new Error(`${methodName}() requires \`ontology\`.`);
|
|
12961
13208
|
}
|
|
12962
|
-
const
|
|
12963
|
-
if (!
|
|
12964
|
-
throw new Error(
|
|
13209
|
+
const tagName = options.tag?.trim();
|
|
13210
|
+
if (!tagName) {
|
|
13211
|
+
throw new Error(`${methodName}() requires \`tag\`.`);
|
|
12965
13212
|
}
|
|
12966
|
-
const tagName = options.tagName?.trim() || void 0;
|
|
12967
13213
|
const user = await this.resolveConnectUser(options);
|
|
13214
|
+
if (!Array.isArray(user.permissions) || user.permissions.length === 0) {
|
|
13215
|
+
throw new Error(
|
|
13216
|
+
`${methodName}() requires at least one permission so the SDK can ensure assignments for new users.`
|
|
13217
|
+
);
|
|
13218
|
+
}
|
|
12968
13219
|
const sandbox = await this.findOrCreateSandbox(ontology);
|
|
12969
13220
|
for (const profileName of user.permissions) {
|
|
12970
13221
|
const profileId = await this.ensurePermissionProfile(
|
|
@@ -12977,22 +13228,49 @@ var Granular = class _Granular {
|
|
|
12977
13228
|
profileId
|
|
12978
13229
|
);
|
|
12979
13230
|
}
|
|
12980
|
-
const
|
|
13231
|
+
const tags = await this.request(
|
|
13232
|
+
`/control/sandboxes/${sandbox.sandboxId}/tags`
|
|
13233
|
+
);
|
|
13234
|
+
const tag = (tags.items || []).find(
|
|
13235
|
+
(candidate) => Boolean(candidate?.name === tagName)
|
|
13236
|
+
);
|
|
13237
|
+
if (!tag) {
|
|
13238
|
+
throw new Error(
|
|
13239
|
+
`Tag "${tagName}" was not found for ontology ${sandbox.sandboxId}.`
|
|
13240
|
+
);
|
|
13241
|
+
}
|
|
13242
|
+
const targetVersionId = tag.targetVersionId || tag.targetBuildId;
|
|
13243
|
+
if (!targetVersionId) {
|
|
13244
|
+
throw new Error(
|
|
13245
|
+
`Tag "${tagName}" does not currently point to a build/version.`
|
|
13246
|
+
);
|
|
13247
|
+
}
|
|
13248
|
+
const allEnvironments = await this.environments.list(sandbox.sandboxId);
|
|
13249
|
+
const userEnvironments = allEnvironments.filter(
|
|
13250
|
+
(environment) => environment.subjectId === user.granularId
|
|
13251
|
+
);
|
|
13252
|
+
const currentMatches = this.sortEnvironmentsByRecency(
|
|
13253
|
+
userEnvironments.filter(
|
|
13254
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId) && environment.versionId === targetVersionId
|
|
13255
|
+
)
|
|
13256
|
+
);
|
|
13257
|
+
if (currentMatches.length > 0) {
|
|
13258
|
+
return currentMatches[0];
|
|
13259
|
+
}
|
|
13260
|
+
const outdatedMatches = this.sortEnvironmentsByRecency(
|
|
13261
|
+
userEnvironments.filter(
|
|
13262
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId)
|
|
13263
|
+
)
|
|
13264
|
+
);
|
|
13265
|
+
if (outdatedMatches.length > 0 && options.createFreshIfOutdated !== true) {
|
|
13266
|
+
return outdatedMatches[0];
|
|
13267
|
+
}
|
|
13268
|
+
return this.environments.create(sandbox.sandboxId, {
|
|
12981
13269
|
subjectId: user.granularId,
|
|
12982
|
-
environment:
|
|
12983
|
-
|
|
13270
|
+
environment: this.buildManagedEnvironmentName(tagName, targetVersionId),
|
|
13271
|
+
tagId: tag.tagId,
|
|
12984
13272
|
permissionProfileId: null
|
|
12985
13273
|
});
|
|
12986
|
-
await this.activateEnvironment(envData.environmentId);
|
|
12987
|
-
const session = await this.request("/ws/sessions", {
|
|
12988
|
-
method: "POST",
|
|
12989
|
-
body: JSON.stringify({
|
|
12990
|
-
environmentId: envData.environmentId,
|
|
12991
|
-
clientId,
|
|
12992
|
-
initialHeap: options.initialHeap
|
|
12993
|
-
})
|
|
12994
|
-
});
|
|
12995
|
-
return this.bindWebSocketEnvironment(envData, clientId, session);
|
|
12996
13274
|
}
|
|
12997
13275
|
/**
|
|
12998
13276
|
* List active (open) sessions for an environment — each session is one agent conversation thread.
|
|
@@ -13055,6 +13333,7 @@ var Granular = class _Granular {
|
|
|
13055
13333
|
const clientId = options.clientId || `client_${Date.now()}`;
|
|
13056
13334
|
await this.activateEnvironment(options.environmentId);
|
|
13057
13335
|
const envData = await this.environments.get(options.environmentId);
|
|
13336
|
+
const environment = this.bindEnvironmentHandle(envData);
|
|
13058
13337
|
const session = await this.request("/ws/sessions", {
|
|
13059
13338
|
method: "POST",
|
|
13060
13339
|
body: JSON.stringify({
|
|
@@ -13063,7 +13342,7 @@ var Granular = class _Granular {
|
|
|
13063
13342
|
initialHeap: options.initialHeap
|
|
13064
13343
|
})
|
|
13065
13344
|
});
|
|
13066
|
-
return this.
|
|
13345
|
+
return this.bindWebSocketEnvironmentSession(environment, clientId, session);
|
|
13067
13346
|
}
|
|
13068
13347
|
/**
|
|
13069
13348
|
* Connect to an existing open session (same conversation thread) using a freshly minted WebSocket token.
|
|
@@ -13075,7 +13354,8 @@ var Granular = class _Granular {
|
|
|
13075
13354
|
body: JSON.stringify({})
|
|
13076
13355
|
});
|
|
13077
13356
|
const envData = await this.environments.get(minted.environmentId);
|
|
13078
|
-
|
|
13357
|
+
const environment = this.bindEnvironmentHandle(envData);
|
|
13358
|
+
return this.bindWebSocketEnvironmentSession(environment, clientId, minted);
|
|
13079
13359
|
}
|
|
13080
13360
|
/**
|
|
13081
13361
|
* Mark a session closed in the control plane. If `environment` is the connected handle for that
|
|
@@ -13107,7 +13387,11 @@ var Granular = class _Granular {
|
|
|
13107
13387
|
});
|
|
13108
13388
|
return this.connectSession({ sessionId, clientId: options?.clientId });
|
|
13109
13389
|
}
|
|
13110
|
-
|
|
13390
|
+
bindEnvironmentHandle(envData) {
|
|
13391
|
+
const graphqlEndpoint = `${this.httpUrl}/orchestrator/graphql`;
|
|
13392
|
+
return new Environment(this, envData, this.apiKey, graphqlEndpoint);
|
|
13393
|
+
}
|
|
13394
|
+
async bindWebSocketEnvironmentSession(environment, clientId, session) {
|
|
13111
13395
|
const client = new WSClient({
|
|
13112
13396
|
url: session.wsUrl,
|
|
13113
13397
|
sessionId: session.sessionId,
|
|
@@ -13118,16 +13402,13 @@ var Granular = class _Granular {
|
|
|
13118
13402
|
onReconnectError: this.onReconnectError
|
|
13119
13403
|
});
|
|
13120
13404
|
await client.connect();
|
|
13121
|
-
const
|
|
13122
|
-
const environment = new Environment(
|
|
13405
|
+
const environmentSession = new EnvironmentSession(
|
|
13123
13406
|
client,
|
|
13124
|
-
|
|
13125
|
-
clientId
|
|
13126
|
-
this.apiKey,
|
|
13127
|
-
graphqlEndpoint
|
|
13407
|
+
environment,
|
|
13408
|
+
clientId
|
|
13128
13409
|
);
|
|
13129
|
-
await
|
|
13130
|
-
return
|
|
13410
|
+
await environmentSession.hello();
|
|
13411
|
+
return environmentSession;
|
|
13131
13412
|
}
|
|
13132
13413
|
async activateEnvironment(environmentId) {
|
|
13133
13414
|
await this.request(`/orchestrator/runtime/environments/${environmentId}/activate`, {
|
|
@@ -15066,19 +15347,27 @@ function matchesPattern(text, matcher) {
|
|
|
15066
15347
|
function assertMatches(label, text, includes = [], excludes = []) {
|
|
15067
15348
|
for (const matcher of includes) {
|
|
15068
15349
|
if (!matchesPattern(text, matcher)) {
|
|
15069
|
-
throw new Error(
|
|
15070
|
-
${
|
|
15350
|
+
throw new Error(
|
|
15351
|
+
`${label} did not match ${matcherToString(matcher)}:
|
|
15352
|
+
${text}`
|
|
15353
|
+
);
|
|
15071
15354
|
}
|
|
15072
15355
|
}
|
|
15073
15356
|
for (const matcher of excludes) {
|
|
15074
15357
|
if (matchesPattern(text, matcher)) {
|
|
15075
|
-
throw new Error(
|
|
15076
|
-
${
|
|
15358
|
+
throw new Error(
|
|
15359
|
+
`${label} matched forbidden ${matcherToString(matcher)}:
|
|
15360
|
+
${text}`
|
|
15361
|
+
);
|
|
15077
15362
|
}
|
|
15078
15363
|
}
|
|
15079
15364
|
}
|
|
15080
15365
|
function buildArtifactDir(baseDir, suiteName = "granular-agent-evals") {
|
|
15081
|
-
return path.join(
|
|
15366
|
+
return path.join(
|
|
15367
|
+
baseDir || path.join(process.cwd(), "test-artifacts"),
|
|
15368
|
+
suiteName,
|
|
15369
|
+
timestampId()
|
|
15370
|
+
);
|
|
15082
15371
|
}
|
|
15083
15372
|
function createTimestampedArtifactDirectory(options) {
|
|
15084
15373
|
return buildArtifactDir(options?.baseDir, options?.suiteName);
|
|
@@ -15092,17 +15381,16 @@ function buildScenarioSteps(scenario) {
|
|
|
15092
15381
|
return scenario.steps;
|
|
15093
15382
|
}
|
|
15094
15383
|
if (!scenario.request) {
|
|
15095
|
-
throw new Error(
|
|
15384
|
+
throw new Error(
|
|
15385
|
+
`Scenario ${scenario.id} must provide either request or steps`
|
|
15386
|
+
);
|
|
15096
15387
|
}
|
|
15097
15388
|
const compatibilityStep = {
|
|
15098
15389
|
id: scenario.id,
|
|
15099
15390
|
request: scenario.request,
|
|
15100
15391
|
human: scenario.human,
|
|
15101
15392
|
expect: scenario.expect,
|
|
15102
|
-
inspect: [
|
|
15103
|
-
...asArray2(scenario.inspect),
|
|
15104
|
-
...asArray2(scenario.verify)
|
|
15105
|
-
],
|
|
15393
|
+
inspect: [...asArray2(scenario.inspect), ...asArray2(scenario.verify)],
|
|
15106
15394
|
check: scenario.check,
|
|
15107
15395
|
maxIterations: scenario.maxIterations,
|
|
15108
15396
|
setup: {
|
|
@@ -15120,22 +15408,27 @@ function buildAssistantHistoryContent(entry) {
|
|
|
15120
15408
|
${entry.content}`);
|
|
15121
15409
|
if (entry.jobStatus) parts.push(`[Job status]
|
|
15122
15410
|
${entry.jobStatus}`);
|
|
15123
|
-
if (entry.jobResultPreview)
|
|
15411
|
+
if (entry.jobResultPreview)
|
|
15412
|
+
parts.push(`[Job result]
|
|
15124
15413
|
${entry.jobResultPreview}`);
|
|
15125
15414
|
if (entry.error) parts.push(`[Job error]
|
|
15126
15415
|
${entry.error}`);
|
|
15127
15416
|
return parts.join("\n\n") || entry.content;
|
|
15128
15417
|
}
|
|
15129
15418
|
function buildHistory(entries) {
|
|
15130
|
-
return entries.reduce(
|
|
15131
|
-
|
|
15132
|
-
if (entry.
|
|
15419
|
+
return entries.reduce(
|
|
15420
|
+
(history, entry) => {
|
|
15421
|
+
if (entry.role === "user") {
|
|
15422
|
+
if (entry.content.trim())
|
|
15423
|
+
history.push({ role: "user", content: entry.content });
|
|
15424
|
+
return history;
|
|
15425
|
+
}
|
|
15426
|
+
const content = buildAssistantHistoryContent(entry).trim();
|
|
15427
|
+
if (content) history.push({ role: "assistant", content });
|
|
15133
15428
|
return history;
|
|
15134
|
-
}
|
|
15135
|
-
|
|
15136
|
-
|
|
15137
|
-
return history;
|
|
15138
|
-
}, []);
|
|
15429
|
+
},
|
|
15430
|
+
[]
|
|
15431
|
+
);
|
|
15139
15432
|
}
|
|
15140
15433
|
function getOpenPromptsFromDoc(liveDoc) {
|
|
15141
15434
|
const jobsById = asRecord4(asRecord4(liveDoc?.jobs)?.byId) || {};
|
|
@@ -15144,7 +15437,8 @@ function getOpenPromptsFromDoc(liveDoc) {
|
|
|
15144
15437
|
const promptRecords = asRecord4(asRecord4(job)?.prompts) || {};
|
|
15145
15438
|
for (const raw of Object.values(promptRecords)) {
|
|
15146
15439
|
const record = asRecord4(raw);
|
|
15147
|
-
if (!record || record.status !== "open" || typeof record.promptId !== "string")
|
|
15440
|
+
if (!record || record.status !== "open" || typeof record.promptId !== "string")
|
|
15441
|
+
continue;
|
|
15148
15442
|
const prompt = normalizePrompt({
|
|
15149
15443
|
promptId: record.promptId,
|
|
15150
15444
|
kind: record.kind,
|
|
@@ -15188,7 +15482,9 @@ ${prompt.message || ""}`;
|
|
|
15188
15482
|
return resolvePromptAnswer(prompt, rawAnswer);
|
|
15189
15483
|
}
|
|
15190
15484
|
if (fallback) return fallback({ prompt, history });
|
|
15191
|
-
throw new Error(
|
|
15485
|
+
throw new Error(
|
|
15486
|
+
`No scripted prompt responder matched prompt ${prompt.id}: ${promptText}`
|
|
15487
|
+
);
|
|
15192
15488
|
};
|
|
15193
15489
|
}
|
|
15194
15490
|
function extractJsonObject(text) {
|
|
@@ -15212,13 +15508,19 @@ function modelOutputInstruction() {
|
|
|
15212
15508
|
].join("\n");
|
|
15213
15509
|
}
|
|
15214
15510
|
function createOpenAIChatTurnGenerator(options) {
|
|
15215
|
-
const baseUrl = (options.baseUrl || "https://api.openai.com/v1").replace(
|
|
15511
|
+
const baseUrl = (options.baseUrl || "https://api.openai.com/v1").replace(
|
|
15512
|
+
/\/$/,
|
|
15513
|
+
""
|
|
15514
|
+
);
|
|
15216
15515
|
const model = options.model || "gpt-5-mini";
|
|
15217
15516
|
return async (input) => {
|
|
15218
15517
|
const messages = [
|
|
15219
|
-
{
|
|
15518
|
+
{
|
|
15519
|
+
role: "system",
|
|
15520
|
+
content: `${input.systemPrompt}
|
|
15220
15521
|
|
|
15221
|
-
${modelOutputInstruction()}`
|
|
15522
|
+
${modelOutputInstruction()}`
|
|
15523
|
+
},
|
|
15222
15524
|
...input.history,
|
|
15223
15525
|
{ role: "user", content: input.request }
|
|
15224
15526
|
];
|
|
@@ -15247,10 +15549,14 @@ ${modelOutputInstruction()}` },
|
|
|
15247
15549
|
await sleep2(500 * attempt);
|
|
15248
15550
|
continue;
|
|
15249
15551
|
}
|
|
15250
|
-
throw new Error(
|
|
15552
|
+
throw new Error(
|
|
15553
|
+
`OpenAI chat generation failed: ${response.status} ${errorText}`
|
|
15554
|
+
);
|
|
15251
15555
|
}
|
|
15252
15556
|
const raw = await response.json();
|
|
15253
|
-
const content = asRecord4(
|
|
15557
|
+
const content = asRecord4(
|
|
15558
|
+
asRecord4(raw.choices?.[0])?.message
|
|
15559
|
+
)?.content;
|
|
15254
15560
|
const text = typeof content === "string" ? content : Array.isArray(content) ? content.map((part) => asRecord4(part)?.text || "").join("") : "";
|
|
15255
15561
|
const parsed = extractJsonObject(text);
|
|
15256
15562
|
if (!parsed) {
|
|
@@ -15268,7 +15574,9 @@ ${text}`);
|
|
|
15268
15574
|
};
|
|
15269
15575
|
} catch (error) {
|
|
15270
15576
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
15271
|
-
if (attempt < 3 && /socket connection was closed unexpectedly|ECONNRESET|network|timed out/i.test(
|
|
15577
|
+
if (attempt < 3 && /socket connection was closed unexpectedly|ECONNRESET|network|timed out/i.test(
|
|
15578
|
+
lastError.message
|
|
15579
|
+
)) {
|
|
15272
15580
|
await sleep2(500 * attempt);
|
|
15273
15581
|
continue;
|
|
15274
15582
|
}
|
|
@@ -15290,7 +15598,10 @@ async function withTimeout(promise, ms, label) {
|
|
|
15290
15598
|
return await Promise.race([
|
|
15291
15599
|
promise,
|
|
15292
15600
|
new Promise((_, reject) => {
|
|
15293
|
-
timeoutId = setTimeout(
|
|
15601
|
+
timeoutId = setTimeout(
|
|
15602
|
+
() => reject(new Error(`${label} timed out after ${ms}ms`)),
|
|
15603
|
+
ms
|
|
15604
|
+
);
|
|
15294
15605
|
})
|
|
15295
15606
|
]);
|
|
15296
15607
|
} finally {
|
|
@@ -15300,7 +15611,9 @@ async function withTimeout(promise, ms, label) {
|
|
|
15300
15611
|
function getActionSummary(liveDoc, jobId) {
|
|
15301
15612
|
const jobsById = asRecord4(asRecord4(liveDoc?.jobs)?.byId) || {};
|
|
15302
15613
|
const job = asRecord4(jobsById[jobId]);
|
|
15303
|
-
return Array.isArray(job?.actionSummary) ? job.actionSummary.filter(
|
|
15614
|
+
return Array.isArray(job?.actionSummary) ? job.actionSummary.filter(
|
|
15615
|
+
(line) => typeof line === "string"
|
|
15616
|
+
) : [];
|
|
15304
15617
|
}
|
|
15305
15618
|
function normalizeHeapSnapshot2(heap) {
|
|
15306
15619
|
return {
|
|
@@ -15330,12 +15643,20 @@ async function waitForJobOutcome(input) {
|
|
|
15330
15643
|
const startedAt = Date.now();
|
|
15331
15644
|
while (Date.now() - startedAt < input.timeoutMs) {
|
|
15332
15645
|
const liveDoc = cloneJson(input.environment.document);
|
|
15333
|
-
const prompts = filterPromptsByBoundary(
|
|
15646
|
+
const prompts = filterPromptsByBoundary(
|
|
15647
|
+
liveDoc,
|
|
15648
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
15649
|
+
input.boundaryTimestamp
|
|
15650
|
+
);
|
|
15334
15651
|
if (prompts.length > 0) {
|
|
15335
15652
|
return { kind: "prompt", prompts, liveDoc, stdout, stderr };
|
|
15336
15653
|
}
|
|
15337
15654
|
try {
|
|
15338
|
-
const result = await withTimeout(
|
|
15655
|
+
const result = await withTimeout(
|
|
15656
|
+
input.job.result,
|
|
15657
|
+
input.pollIntervalMs,
|
|
15658
|
+
`job ${input.job.id} tick`
|
|
15659
|
+
);
|
|
15339
15660
|
return { kind: "completed", result, liveDoc, stdout, stderr };
|
|
15340
15661
|
} catch (error) {
|
|
15341
15662
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -15401,7 +15722,9 @@ function buildResultReport(result) {
|
|
|
15401
15722
|
...result.actionSummary.length ? result.actionSummary.map((line) => `- ${line}`) : ["- None"],
|
|
15402
15723
|
"",
|
|
15403
15724
|
"## Prompt Interactions",
|
|
15404
|
-
...result.promptInteractions.length ? result.promptInteractions.map(
|
|
15725
|
+
...result.promptInteractions.length ? result.promptInteractions.map(
|
|
15726
|
+
(interaction) => `- [${interaction.type}] ${interaction.message || interaction.title} -> ${JSON.stringify(interaction.answer)}`
|
|
15727
|
+
) : ["- None"],
|
|
15405
15728
|
"",
|
|
15406
15729
|
...stepSection,
|
|
15407
15730
|
"## Raw Files",
|
|
@@ -15416,7 +15739,9 @@ function buildSuiteIndex(results) {
|
|
|
15416
15739
|
const lines = [
|
|
15417
15740
|
"# Agent Eval Report Index",
|
|
15418
15741
|
"",
|
|
15419
|
-
...results.map(
|
|
15742
|
+
...results.map(
|
|
15743
|
+
(result) => `- [${result.scenario.id}](./${result.scenario.id}/REPORT.md) - ${result.status}`
|
|
15744
|
+
)
|
|
15420
15745
|
];
|
|
15421
15746
|
return `${lines.join("\n")}
|
|
15422
15747
|
`;
|
|
@@ -15430,7 +15755,7 @@ async function applySetup(setup, context) {
|
|
|
15430
15755
|
await context.environment.recordObjects(setup.records);
|
|
15431
15756
|
}
|
|
15432
15757
|
if (setup.effects?.length) {
|
|
15433
|
-
await context.environment.
|
|
15758
|
+
await context.granular.ontology(context.environment.sandboxId).effects.registerMany(setup.effects);
|
|
15434
15759
|
}
|
|
15435
15760
|
if (setup.run) {
|
|
15436
15761
|
await setup.run(context);
|
|
@@ -15452,6 +15777,7 @@ async function runAgentEvalSuite(options) {
|
|
|
15452
15777
|
);
|
|
15453
15778
|
try {
|
|
15454
15779
|
await applySetup(scenario.setup, {
|
|
15780
|
+
granular: options.harness.granular,
|
|
15455
15781
|
conversation,
|
|
15456
15782
|
environment: conversation.environment,
|
|
15457
15783
|
turnDir: conversation.artifactDir
|
|
@@ -15464,14 +15790,19 @@ async function runAgentEvalSuite(options) {
|
|
|
15464
15790
|
conversation,
|
|
15465
15791
|
request: step.request,
|
|
15466
15792
|
prepare: async (ctx) => {
|
|
15467
|
-
await applySetup(step.setup,
|
|
15793
|
+
await applySetup(step.setup, {
|
|
15794
|
+
granular: options.harness.granular,
|
|
15795
|
+
...ctx
|
|
15796
|
+
});
|
|
15468
15797
|
},
|
|
15469
15798
|
human: step.human,
|
|
15470
15799
|
maxIterations: step.maxIterations,
|
|
15471
15800
|
autoAnswerPrompts: step.autoAnswerPrompts
|
|
15472
15801
|
});
|
|
15473
15802
|
if ("prompts" in completed) {
|
|
15474
|
-
throw new Error(
|
|
15803
|
+
throw new Error(
|
|
15804
|
+
`Scenario ${scenario.id} step ${index + 1} paused for human input instead of completing automatically`
|
|
15805
|
+
);
|
|
15475
15806
|
}
|
|
15476
15807
|
const inspectionResults = [];
|
|
15477
15808
|
const stepChecks = asArray2(step.check);
|
|
@@ -15487,12 +15818,23 @@ async function runAgentEvalSuite(options) {
|
|
|
15487
15818
|
actionSummary: completed.actionSummary,
|
|
15488
15819
|
promptInteractions: completed.promptInteractions,
|
|
15489
15820
|
result: completed.result,
|
|
15490
|
-
heap: normalizeHeapSnapshot2(
|
|
15491
|
-
|
|
15821
|
+
heap: normalizeHeapSnapshot2(
|
|
15822
|
+
asRecord4(
|
|
15823
|
+
cloneJson(conversation.environment.document)?.heap
|
|
15824
|
+
)
|
|
15825
|
+
),
|
|
15826
|
+
openPrompts: getOpenPromptsFromDoc(
|
|
15827
|
+
cloneJson(conversation.environment.document)
|
|
15828
|
+
),
|
|
15492
15829
|
liveDoc: cloneJson(conversation.environment.document),
|
|
15493
15830
|
inspect: async (code) => {
|
|
15494
|
-
const
|
|
15495
|
-
|
|
15831
|
+
const session = conversation.environment;
|
|
15832
|
+
const job = await session.submitJob(code);
|
|
15833
|
+
return withTimeout(
|
|
15834
|
+
job.result,
|
|
15835
|
+
9e4,
|
|
15836
|
+
`inspection job ${job.id}`
|
|
15837
|
+
);
|
|
15496
15838
|
},
|
|
15497
15839
|
assertMatches
|
|
15498
15840
|
};
|
|
@@ -15550,7 +15892,9 @@ async function runAgentEvalSuite(options) {
|
|
|
15550
15892
|
}
|
|
15551
15893
|
const lastStep = stepResults[stepResults.length - 1];
|
|
15552
15894
|
if (!lastStep) {
|
|
15553
|
-
throw new Error(
|
|
15895
|
+
throw new Error(
|
|
15896
|
+
`Scenario ${scenario.id} produced no completed steps`
|
|
15897
|
+
);
|
|
15554
15898
|
}
|
|
15555
15899
|
const result = {
|
|
15556
15900
|
scenario,
|
|
@@ -15564,9 +15908,18 @@ async function runAgentEvalSuite(options) {
|
|
|
15564
15908
|
steps: stepResults,
|
|
15565
15909
|
turnDir: conversation.artifactDir
|
|
15566
15910
|
};
|
|
15567
|
-
await writeJson(
|
|
15568
|
-
|
|
15569
|
-
|
|
15911
|
+
await writeJson(
|
|
15912
|
+
path.join(conversation.artifactDir, "result.json"),
|
|
15913
|
+
result
|
|
15914
|
+
);
|
|
15915
|
+
await writeJson(
|
|
15916
|
+
path.join(conversation.artifactDir, "report.json"),
|
|
15917
|
+
result
|
|
15918
|
+
);
|
|
15919
|
+
await writeFile(
|
|
15920
|
+
path.join(conversation.artifactDir, "REPORT.md"),
|
|
15921
|
+
buildResultReport(result)
|
|
15922
|
+
);
|
|
15570
15923
|
finalResult = result;
|
|
15571
15924
|
} catch (error) {
|
|
15572
15925
|
const failureMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -15588,7 +15941,10 @@ async function runAgentEvalSuite(options) {
|
|
|
15588
15941
|
await ensureDir(failed.turnDir);
|
|
15589
15942
|
await writeJson(path.join(failed.turnDir, "result.json"), failed);
|
|
15590
15943
|
await writeJson(path.join(failed.turnDir, "report.json"), failed);
|
|
15591
|
-
await writeFile(
|
|
15944
|
+
await writeFile(
|
|
15945
|
+
path.join(failed.turnDir, "REPORT.md"),
|
|
15946
|
+
buildResultReport(failed)
|
|
15947
|
+
);
|
|
15592
15948
|
finalResult = failed;
|
|
15593
15949
|
} finally {
|
|
15594
15950
|
await options.harness.closeConversation(conversation);
|
|
@@ -15609,12 +15965,21 @@ async function runAgentEvalSuite(options) {
|
|
|
15609
15965
|
}
|
|
15610
15966
|
results.push(finalResult);
|
|
15611
15967
|
}
|
|
15612
|
-
await writeJson(
|
|
15613
|
-
|
|
15968
|
+
await writeJson(
|
|
15969
|
+
path.join(options.harness.artifactDir, "summary.json"),
|
|
15970
|
+
results
|
|
15971
|
+
);
|
|
15972
|
+
await writeFile(
|
|
15973
|
+
path.join(options.harness.artifactDir, "REPORT_INDEX.md"),
|
|
15974
|
+
buildSuiteIndex(results)
|
|
15975
|
+
);
|
|
15614
15976
|
return { artifactDir: options.harness.artifactDir, results };
|
|
15615
15977
|
}
|
|
15616
15978
|
function createAgentEvalHarness(options) {
|
|
15617
|
-
const artifactDir = buildArtifactDir(
|
|
15979
|
+
const artifactDir = buildArtifactDir(
|
|
15980
|
+
options.artifactBaseDir,
|
|
15981
|
+
options.suiteName
|
|
15982
|
+
);
|
|
15618
15983
|
const controllerBudgets = {
|
|
15619
15984
|
...DEFAULT_CONTROLLER_BUDGETS,
|
|
15620
15985
|
...options.controllerBudgets || {}
|
|
@@ -15626,7 +15991,9 @@ function createAgentEvalHarness(options) {
|
|
|
15626
15991
|
await ensureDir(artifactDir);
|
|
15627
15992
|
const clientId = `${slugify(label)}-${Date.now()}`;
|
|
15628
15993
|
if (!options.openEnvironment && !options.environmentId) {
|
|
15629
|
-
throw new Error(
|
|
15994
|
+
throw new Error(
|
|
15995
|
+
"createAgentEvalHarness requires either environmentId or openEnvironment"
|
|
15996
|
+
);
|
|
15630
15997
|
}
|
|
15631
15998
|
const environment = options.openEnvironment ? await options.openEnvironment({ label, clientId }) : await options.granular.createSession({
|
|
15632
15999
|
environmentId: options.environmentId,
|
|
@@ -15650,12 +16017,15 @@ function createAgentEvalHarness(options) {
|
|
|
15650
16017
|
}
|
|
15651
16018
|
async function closeConversation(conversation) {
|
|
15652
16019
|
try {
|
|
15653
|
-
await options.granular.closeSession(
|
|
16020
|
+
await options.granular.closeSession(
|
|
16021
|
+
conversation.environment.sessionId,
|
|
16022
|
+
conversation.environment
|
|
16023
|
+
);
|
|
15654
16024
|
} catch {
|
|
15655
16025
|
}
|
|
15656
16026
|
}
|
|
15657
|
-
async function runCheckJob(code,
|
|
15658
|
-
const job = await
|
|
16027
|
+
async function runCheckJob(code, session) {
|
|
16028
|
+
const job = await session.submitJob(code);
|
|
15659
16029
|
return withTimeout(job.result, jobTimeoutMs, `check job ${job.id}`);
|
|
15660
16030
|
}
|
|
15661
16031
|
function buildCheckContext(conversation, completed, turnDir) {
|
|
@@ -15700,8 +16070,12 @@ function createAgentEvalHarness(options) {
|
|
|
15700
16070
|
async function resumePendingTurn(pending, responder) {
|
|
15701
16071
|
const prompt = pending.prompts[0];
|
|
15702
16072
|
if (!prompt) throw new Error("Pending turn has no prompts to answer");
|
|
15703
|
-
const answer = await responder({
|
|
15704
|
-
|
|
16073
|
+
const answer = await responder({
|
|
16074
|
+
prompt,
|
|
16075
|
+
history: pending.promptInteractions
|
|
16076
|
+
});
|
|
16077
|
+
const session = pending.conversation.environment;
|
|
16078
|
+
await session.answerPrompt(prompt.id, answer);
|
|
15705
16079
|
pending.promptInteractions.push({
|
|
15706
16080
|
promptId: prompt.id,
|
|
15707
16081
|
type: prompt.type,
|
|
@@ -15725,7 +16099,9 @@ function createAgentEvalHarness(options) {
|
|
|
15725
16099
|
};
|
|
15726
16100
|
}
|
|
15727
16101
|
await sleep2(350);
|
|
15728
|
-
const liveDoc = cloneJson(
|
|
16102
|
+
const liveDoc = cloneJson(
|
|
16103
|
+
pending.conversation.environment.document
|
|
16104
|
+
);
|
|
15729
16105
|
const presentation = resolveJobPresentation({
|
|
15730
16106
|
jobId: pending.job.id,
|
|
15731
16107
|
result: resumed.result,
|
|
@@ -15770,25 +16146,40 @@ function createAgentEvalHarness(options) {
|
|
|
15770
16146
|
await conversation.environment.recordObjects(input.prepareRecords);
|
|
15771
16147
|
}
|
|
15772
16148
|
if (input.prepareTools?.length) {
|
|
15773
|
-
await conversation.environment.
|
|
16149
|
+
await options.granular.ontology(conversation.environment.sandboxId).effects.registerMany(input.prepareTools);
|
|
15774
16150
|
}
|
|
15775
16151
|
if (input.prepare) {
|
|
15776
|
-
await input.prepare({
|
|
16152
|
+
await input.prepare({
|
|
16153
|
+
conversation,
|
|
16154
|
+
environment: conversation.environment,
|
|
16155
|
+
turnDir
|
|
16156
|
+
});
|
|
15777
16157
|
}
|
|
15778
16158
|
const boundaryTimestamp = Date.now();
|
|
15779
16159
|
conversation.history.push({ role: "user", content: input.request });
|
|
15780
|
-
await writeJson(path.join(turnDir, "request.json"), {
|
|
16160
|
+
await writeJson(path.join(turnDir, "request.json"), {
|
|
16161
|
+
request: input.request,
|
|
16162
|
+
boundaryTimestamp
|
|
16163
|
+
});
|
|
15781
16164
|
let iteration = 0;
|
|
15782
16165
|
let noProgressCount = 0;
|
|
15783
16166
|
let previousSnapshot = null;
|
|
15784
16167
|
let latestCheckpoint = null;
|
|
15785
16168
|
const maxIterations = input.maxIterations || controllerBudgets.maxIterations;
|
|
15786
16169
|
const autoAnswerPrompts = input.autoAnswerPrompts ?? true;
|
|
15787
|
-
const baselineClosureId = getCurrentClosureId(
|
|
16170
|
+
const baselineClosureId = getCurrentClosureId(
|
|
16171
|
+
cloneJson(conversation.environment.document)
|
|
16172
|
+
);
|
|
15788
16173
|
while (iteration < maxIterations) {
|
|
15789
16174
|
const liveDoc = cloneJson(conversation.environment.document);
|
|
15790
|
-
const pendingPrompts = filterPromptsByBoundary(
|
|
15791
|
-
|
|
16175
|
+
const pendingPrompts = filterPromptsByBoundary(
|
|
16176
|
+
liveDoc,
|
|
16177
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
16178
|
+
boundaryTimestamp
|
|
16179
|
+
);
|
|
16180
|
+
const workflowFocus = projectWorkflowFocus(liveDoc, pendingPrompts, {
|
|
16181
|
+
boundaryTimestamp
|
|
16182
|
+
});
|
|
15792
16183
|
const systemPrompt = buildGranularAgentSystemPrompt({
|
|
15793
16184
|
domainDocumentation: await conversation.environment.getDomainDocumentation(),
|
|
15794
16185
|
sessionContext: {
|
|
@@ -15799,8 +16190,12 @@ function createAgentEvalHarness(options) {
|
|
|
15799
16190
|
heapSummary: projectHeapSummary(liveDoc, {
|
|
15800
16191
|
focus: workflowFocus
|
|
15801
16192
|
}),
|
|
15802
|
-
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
15803
|
-
|
|
16193
|
+
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
16194
|
+
boundaryTimestamp
|
|
16195
|
+
}),
|
|
16196
|
+
workflowSummary: projectWorkflowSummary(liveDoc, pendingPrompts, {
|
|
16197
|
+
boundaryTimestamp
|
|
16198
|
+
}),
|
|
15804
16199
|
tools: conversation.environment.getEffects().map((tool) => ({
|
|
15805
16200
|
name: tool.name,
|
|
15806
16201
|
description: tool.description,
|
|
@@ -15810,14 +16205,23 @@ function createAgentEvalHarness(options) {
|
|
|
15810
16205
|
})),
|
|
15811
16206
|
checkpoint: latestCheckpoint
|
|
15812
16207
|
});
|
|
15813
|
-
const request = iteration === 0 ? input.request : buildContinuationInstruction(
|
|
15814
|
-
|
|
15815
|
-
|
|
15816
|
-
|
|
15817
|
-
|
|
15818
|
-
|
|
15819
|
-
|
|
15820
|
-
|
|
16208
|
+
const request = iteration === 0 ? input.request : buildContinuationInstruction(
|
|
16209
|
+
buildContinuationPreview(latestCheckpoint, noProgressCount)
|
|
16210
|
+
);
|
|
16211
|
+
const generation = await withTimeout(
|
|
16212
|
+
generateTurnWithRepair(options.generator, {
|
|
16213
|
+
systemPrompt,
|
|
16214
|
+
history: buildHistory(conversation.history),
|
|
16215
|
+
request,
|
|
16216
|
+
attempt: 1
|
|
16217
|
+
}),
|
|
16218
|
+
chatTimeoutMs,
|
|
16219
|
+
`chat generation for ${conversation.label} iteration ${iteration + 1}`
|
|
16220
|
+
);
|
|
16221
|
+
await writeJson(
|
|
16222
|
+
path.join(turnDir, `iteration-${iteration + 1}-generation.json`),
|
|
16223
|
+
generation
|
|
16224
|
+
);
|
|
15821
16225
|
if (!generation.code) {
|
|
15822
16226
|
const responseText2 = generation.reply?.trim() || "Done.";
|
|
15823
16227
|
conversation.history.push({ role: "assistant", content: responseText2 });
|
|
@@ -15833,12 +16237,18 @@ function createAgentEvalHarness(options) {
|
|
|
15833
16237
|
result: generation.reply?.trim() || responseText2
|
|
15834
16238
|
};
|
|
15835
16239
|
if (input.verification) {
|
|
15836
|
-
completed.verification = await runInspection(
|
|
16240
|
+
completed.verification = await runInspection(
|
|
16241
|
+
conversation,
|
|
16242
|
+
input.verification,
|
|
16243
|
+
completed,
|
|
16244
|
+
turnDir
|
|
16245
|
+
);
|
|
15837
16246
|
}
|
|
15838
16247
|
await writeJson(path.join(turnDir, "result.json"), completed);
|
|
15839
16248
|
return completed;
|
|
15840
16249
|
}
|
|
15841
|
-
const
|
|
16250
|
+
const session = conversation.environment;
|
|
16251
|
+
const job = await session.submitJob(generation.code);
|
|
15842
16252
|
const outcome = await waitForJobOutcome({
|
|
15843
16253
|
environment: conversation.environment,
|
|
15844
16254
|
job,
|
|
@@ -15863,7 +16273,9 @@ function createAgentEvalHarness(options) {
|
|
|
15863
16273
|
};
|
|
15864
16274
|
}
|
|
15865
16275
|
if (!input.human) {
|
|
15866
|
-
throw new Error(
|
|
16276
|
+
throw new Error(
|
|
16277
|
+
"This turn reached a human prompt but no responder was provided"
|
|
16278
|
+
);
|
|
15867
16279
|
}
|
|
15868
16280
|
let pending = {
|
|
15869
16281
|
conversation,
|
|
@@ -15885,16 +16297,25 @@ function createAgentEvalHarness(options) {
|
|
|
15885
16297
|
continue;
|
|
15886
16298
|
}
|
|
15887
16299
|
if (input.verification) {
|
|
15888
|
-
resumed.verification = await runInspection(
|
|
16300
|
+
resumed.verification = await runInspection(
|
|
16301
|
+
conversation,
|
|
16302
|
+
input.verification,
|
|
16303
|
+
resumed,
|
|
16304
|
+
turnDir
|
|
16305
|
+
);
|
|
15889
16306
|
}
|
|
15890
16307
|
return resumed;
|
|
15891
16308
|
}
|
|
15892
16309
|
}
|
|
15893
16310
|
if (outcome.kind !== "completed") {
|
|
15894
|
-
throw new Error(
|
|
16311
|
+
throw new Error(
|
|
16312
|
+
"Unexpected non-completed outcome after prompt handling"
|
|
16313
|
+
);
|
|
15895
16314
|
}
|
|
15896
16315
|
await sleep2(350);
|
|
15897
|
-
const settledLiveDoc = cloneJson(
|
|
16316
|
+
const settledLiveDoc = cloneJson(
|
|
16317
|
+
conversation.environment.document
|
|
16318
|
+
);
|
|
15898
16319
|
const sessionHeap = normalizeHeapSnapshot2(asRecord4(settledLiveDoc?.heap));
|
|
15899
16320
|
const presentation = resolveJobPresentation({
|
|
15900
16321
|
jobId: job.id,
|
|
@@ -15915,7 +16336,11 @@ function createAgentEvalHarness(options) {
|
|
|
15915
16336
|
baselineClosureId,
|
|
15916
16337
|
currentClosureId: getCurrentClosureId(settledLiveDoc),
|
|
15917
16338
|
liveDoc: settledLiveDoc,
|
|
15918
|
-
pendingPrompts: filterPromptsByBoundary(
|
|
16339
|
+
pendingPrompts: filterPromptsByBoundary(
|
|
16340
|
+
settledLiveDoc,
|
|
16341
|
+
getOpenPromptsFromDoc(settledLiveDoc),
|
|
16342
|
+
boundaryTimestamp
|
|
16343
|
+
),
|
|
15919
16344
|
projectionOptions: { boundaryTimestamp },
|
|
15920
16345
|
latestResponseText: responseText,
|
|
15921
16346
|
previousSnapshot,
|
|
@@ -15940,12 +16365,15 @@ function createAgentEvalHarness(options) {
|
|
|
15940
16365
|
jobStatus: "succeeded",
|
|
15941
16366
|
jobResultPreview: JSON.stringify(outcome.result, null, 2)
|
|
15942
16367
|
});
|
|
15943
|
-
await writeJson(
|
|
15944
|
-
|
|
15945
|
-
|
|
15946
|
-
|
|
15947
|
-
|
|
15948
|
-
|
|
16368
|
+
await writeJson(
|
|
16369
|
+
path.join(turnDir, `iteration-${iteration + 1}-result.json`),
|
|
16370
|
+
{
|
|
16371
|
+
responseText,
|
|
16372
|
+
continuation,
|
|
16373
|
+
actionSummary: latestCheckpoint.latestActionSummary,
|
|
16374
|
+
result: outcome.result
|
|
16375
|
+
}
|
|
16376
|
+
);
|
|
15949
16377
|
if (!continuation.shouldContinue) {
|
|
15950
16378
|
const completed = {
|
|
15951
16379
|
conversation,
|
|
@@ -15960,17 +16388,25 @@ function createAgentEvalHarness(options) {
|
|
|
15960
16388
|
result: outcome.result
|
|
15961
16389
|
};
|
|
15962
16390
|
if (input.verification) {
|
|
15963
|
-
completed.verification = await runInspection(
|
|
16391
|
+
completed.verification = await runInspection(
|
|
16392
|
+
conversation,
|
|
16393
|
+
input.verification,
|
|
16394
|
+
completed,
|
|
16395
|
+
turnDir
|
|
16396
|
+
);
|
|
15964
16397
|
}
|
|
15965
16398
|
await writeJson(path.join(turnDir, "result.json"), completed);
|
|
15966
16399
|
return completed;
|
|
15967
16400
|
}
|
|
15968
16401
|
iteration += 1;
|
|
15969
16402
|
}
|
|
15970
|
-
throw new Error(
|
|
16403
|
+
throw new Error(
|
|
16404
|
+
`Turn exceeded iteration budget (${maxIterations}) for ${conversation.label}`
|
|
16405
|
+
);
|
|
15971
16406
|
}
|
|
15972
16407
|
return {
|
|
15973
16408
|
artifactDir,
|
|
16409
|
+
granular: options.granular,
|
|
15974
16410
|
openConversation,
|
|
15975
16411
|
closeConversation,
|
|
15976
16412
|
runTurn,
|
|
@@ -16006,12 +16442,11 @@ function createAgentTester(options) {
|
|
|
16006
16442
|
}
|
|
16007
16443
|
if ("connect" in options.target && !connectSeeded) {
|
|
16008
16444
|
connectSeeded = true;
|
|
16009
|
-
const environment = await granular.
|
|
16010
|
-
...options.target.connect
|
|
16011
|
-
clientId
|
|
16445
|
+
const environment = await granular.openEnvironment({
|
|
16446
|
+
...options.target.connect
|
|
16012
16447
|
});
|
|
16013
16448
|
resolvedEnvironmentId = environment.environmentId;
|
|
16014
|
-
return environment;
|
|
16449
|
+
return environment.sessions.create({ clientId });
|
|
16015
16450
|
}
|
|
16016
16451
|
if ("createEnvironment" in options.target && !resolvedEnvironmentId) {
|
|
16017
16452
|
const envData = await granular.environments.create(
|