@granular-software/sdk 0.4.30 → 0.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -20
- package/dist/agent-evals.d.mts +7 -6
- package/dist/agent-evals.d.ts +7 -6
- package/dist/agent-evals.js +681 -277
- package/dist/agent-evals.js.map +1 -1
- package/dist/agent-evals.mjs +681 -277
- package/dist/agent-evals.mjs.map +1 -1
- package/dist/cli/index.js +526 -271
- package/dist/{client-BVvOMfln.d.mts → client-CTH1hfwe.d.mts} +254 -123
- package/dist/{client-BVvOMfln.d.ts → client-CTH1hfwe.d.ts} +254 -123
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +439 -187
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +438 -188
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/agent-evals.js
CHANGED
|
@@ -4674,27 +4674,27 @@ var Session = class {
|
|
|
4674
4674
|
}
|
|
4675
4675
|
async publishTools(tools, revision = "1.0.0") {
|
|
4676
4676
|
throw new Error(
|
|
4677
|
-
"Environment-scoped effect publication was removed. Declare effects in the manifest and register live handlers with granular.
|
|
4677
|
+
"Environment-scoped effect publication was removed. Declare effects in the manifest and register live handlers with granular.ontology(environment.sandboxId).effects.registerMany(effects)."
|
|
4678
4678
|
);
|
|
4679
4679
|
}
|
|
4680
4680
|
async publishEffect(effect) {
|
|
4681
4681
|
throw new Error(
|
|
4682
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4682
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.register(effect)."
|
|
4683
4683
|
);
|
|
4684
4684
|
}
|
|
4685
4685
|
async publishEffects(effects) {
|
|
4686
4686
|
throw new Error(
|
|
4687
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4687
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.registerMany(effects)."
|
|
4688
4688
|
);
|
|
4689
4689
|
}
|
|
4690
4690
|
async unpublishEffect(name) {
|
|
4691
4691
|
throw new Error(
|
|
4692
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4692
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.unregister(effectName)."
|
|
4693
4693
|
);
|
|
4694
4694
|
}
|
|
4695
4695
|
async unpublishAllEffects() {
|
|
4696
4696
|
throw new Error(
|
|
4697
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4697
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.clear()."
|
|
4698
4698
|
);
|
|
4699
4699
|
}
|
|
4700
4700
|
/**
|
|
@@ -11718,6 +11718,22 @@ function normalizeHeapSnapshot(raw) {
|
|
|
11718
11718
|
updatedAt: typeof heap.updatedAt === "number" ? heap.updatedAt : Date.now()
|
|
11719
11719
|
};
|
|
11720
11720
|
}
|
|
11721
|
+
function deriveRuntimeBaseUrl(apiEndpoint) {
|
|
11722
|
+
try {
|
|
11723
|
+
const endpoint = new URL(apiEndpoint);
|
|
11724
|
+
const graphqlSuffix = "/orchestrator/graphql";
|
|
11725
|
+
if (endpoint.pathname.endsWith(graphqlSuffix)) {
|
|
11726
|
+
endpoint.pathname = endpoint.pathname.slice(0, -graphqlSuffix.length);
|
|
11727
|
+
} else if (endpoint.pathname.endsWith("/graphql")) {
|
|
11728
|
+
endpoint.pathname = endpoint.pathname.slice(0, -"/graphql".length);
|
|
11729
|
+
}
|
|
11730
|
+
endpoint.search = "";
|
|
11731
|
+
endpoint.hash = "";
|
|
11732
|
+
return endpoint.toString().replace(/\/$/, "");
|
|
11733
|
+
} catch {
|
|
11734
|
+
return apiEndpoint.replace(/\/orchestrator\/graphql$/, "").replace(/\/$/, "");
|
|
11735
|
+
}
|
|
11736
|
+
}
|
|
11721
11737
|
function normalizeSubject(subject) {
|
|
11722
11738
|
const granularId = subject.granularId || subject.subjectId;
|
|
11723
11739
|
const userId = subject.userId || subject.identityId || granularId;
|
|
@@ -11757,12 +11773,13 @@ function normalizeEnvironmentData(environment) {
|
|
|
11757
11773
|
tracking: environment.tracking || buildPolicy
|
|
11758
11774
|
};
|
|
11759
11775
|
}
|
|
11760
|
-
var Environment = class
|
|
11776
|
+
var Environment = class {
|
|
11777
|
+
granular;
|
|
11761
11778
|
envData;
|
|
11762
11779
|
_apiKey;
|
|
11763
11780
|
_apiEndpoint;
|
|
11764
|
-
constructor(
|
|
11765
|
-
|
|
11781
|
+
constructor(granular, envData, apiKey, apiEndpoint) {
|
|
11782
|
+
this.granular = granular;
|
|
11766
11783
|
this.envData = envData;
|
|
11767
11784
|
this._apiKey = apiKey;
|
|
11768
11785
|
this._apiEndpoint = apiEndpoint;
|
|
@@ -11803,35 +11820,126 @@ var Environment = class extends Session {
|
|
|
11803
11820
|
get permissionProfileId() {
|
|
11804
11821
|
return this.envData.permissionProfileId;
|
|
11805
11822
|
}
|
|
11823
|
+
/** The current build policy backing this environment */
|
|
11824
|
+
get buildPolicy() {
|
|
11825
|
+
return this.envData.buildPolicy;
|
|
11826
|
+
}
|
|
11827
|
+
/** The current update state relative to the followed tag */
|
|
11828
|
+
get updateState() {
|
|
11829
|
+
return this.envData.updateState;
|
|
11830
|
+
}
|
|
11831
|
+
/** Convenience flag for whether this environment trails the current tag target */
|
|
11832
|
+
get isOutdated() {
|
|
11833
|
+
return this.envData.updateState === "update_available";
|
|
11834
|
+
}
|
|
11835
|
+
/** The followed tag name when this environment is tag-tracked */
|
|
11836
|
+
get tag() {
|
|
11837
|
+
return this.envData.tag?.name || this.envData.buildPolicy.tagName || null;
|
|
11838
|
+
}
|
|
11806
11839
|
/** The GraphQL API endpoint URL */
|
|
11807
11840
|
get apiEndpoint() {
|
|
11808
11841
|
return this._apiEndpoint;
|
|
11809
11842
|
}
|
|
11843
|
+
/** Internal auth token used for control-plane and runtime fallback requests */
|
|
11844
|
+
get authToken() {
|
|
11845
|
+
return this._apiKey;
|
|
11846
|
+
}
|
|
11847
|
+
/** Base runtime URL derived from the GraphQL endpoint */
|
|
11848
|
+
get runtimeBaseUrl() {
|
|
11849
|
+
return this.getRuntimeBaseUrl();
|
|
11850
|
+
}
|
|
11851
|
+
get sessions() {
|
|
11852
|
+
return {
|
|
11853
|
+
list: async (options) => this.listSessions(options?.status || "active"),
|
|
11854
|
+
create: async (options) => this.createSession(options),
|
|
11855
|
+
connect: async (sessionId, options) => this.connectSession(sessionId, options),
|
|
11856
|
+
reopen: async (sessionId, options) => this.reopenSession(sessionId, options),
|
|
11857
|
+
close: async (sessionId, session) => this.closeSession(sessionId, session)
|
|
11858
|
+
};
|
|
11859
|
+
}
|
|
11860
|
+
get data() {
|
|
11861
|
+
return {
|
|
11862
|
+
record: async (record) => this.recordObject(record),
|
|
11863
|
+
recordMany: async (records, options) => this.recordObjects(records, options),
|
|
11864
|
+
import: async (records, options) => this.enqueueRecordImport(records, options),
|
|
11865
|
+
listImports: async (status) => this.listRecordImports(status),
|
|
11866
|
+
getImport: async (importId) => this.getRecordImport(importId),
|
|
11867
|
+
getImportSummary: async () => this.getRecordImportSummary(),
|
|
11868
|
+
cancelImport: async (importId) => this.cancelRecordImport(importId),
|
|
11869
|
+
getAwaitingCount: async () => this.getAwaitingRecordCount()
|
|
11870
|
+
};
|
|
11871
|
+
}
|
|
11872
|
+
get feedback() {
|
|
11873
|
+
return {
|
|
11874
|
+
list: async () => this.listFeedback()
|
|
11875
|
+
};
|
|
11876
|
+
}
|
|
11810
11877
|
/**
|
|
11811
|
-
*
|
|
11812
|
-
*
|
|
11813
|
-
*
|
|
11814
|
-
*
|
|
11878
|
+
* Sessionless environments do not own a live transport, so disconnecting the
|
|
11879
|
+
* environment handle itself is a no-op. This keeps the public surface
|
|
11880
|
+
* symmetric with `EnvironmentSession.disconnect()` and lets callers always
|
|
11881
|
+
* clean up safely without tracking whether they currently hold an environment
|
|
11882
|
+
* or a session.
|
|
11815
11883
|
*/
|
|
11816
|
-
|
|
11817
|
-
const doc = this.document;
|
|
11818
|
-
return normalizeHeapSnapshot(doc?.heap);
|
|
11884
|
+
async disconnect() {
|
|
11819
11885
|
}
|
|
11820
|
-
|
|
11821
|
-
|
|
11822
|
-
const
|
|
11823
|
-
|
|
11824
|
-
|
|
11825
|
-
|
|
11826
|
-
|
|
11827
|
-
|
|
11828
|
-
|
|
11829
|
-
|
|
11830
|
-
|
|
11831
|
-
|
|
11832
|
-
|
|
11833
|
-
|
|
11886
|
+
async listSessions(status = "active") {
|
|
11887
|
+
if (status === "all") {
|
|
11888
|
+
const [active, closed] = await Promise.all([
|
|
11889
|
+
this.granular.listOpenSessions({ environmentId: this.environmentId }),
|
|
11890
|
+
this.granular.listClosedSessions({ environmentId: this.environmentId })
|
|
11891
|
+
]);
|
|
11892
|
+
return [...active, ...closed].sort(
|
|
11893
|
+
(left, right) => Date.parse(right.lastSeenAt) - Date.parse(left.lastSeenAt)
|
|
11894
|
+
);
|
|
11895
|
+
}
|
|
11896
|
+
return status === "closed" ? this.granular.listClosedSessions({ environmentId: this.environmentId }) : this.granular.listOpenSessions({ environmentId: this.environmentId });
|
|
11897
|
+
}
|
|
11898
|
+
async createSession(options) {
|
|
11899
|
+
return this.granular.createSession({
|
|
11900
|
+
environmentId: this.environmentId,
|
|
11901
|
+
clientId: options?.clientId,
|
|
11902
|
+
initialHeap: options?.initialHeap
|
|
11903
|
+
});
|
|
11904
|
+
}
|
|
11905
|
+
async connectSession(sessionId, options) {
|
|
11906
|
+
const session = await this.granular["connectSession"]({
|
|
11907
|
+
sessionId,
|
|
11908
|
+
clientId: options?.clientId
|
|
11909
|
+
});
|
|
11910
|
+
if (session.environmentId !== this.environmentId) {
|
|
11911
|
+
await session.disconnect().catch(() => {
|
|
11912
|
+
session.disconnectTransport();
|
|
11913
|
+
});
|
|
11914
|
+
throw new Error(
|
|
11915
|
+
`Session ${sessionId} belongs to environment ${session.environmentId}, not ${this.environmentId}.`
|
|
11916
|
+
);
|
|
11917
|
+
}
|
|
11918
|
+
return session;
|
|
11919
|
+
}
|
|
11920
|
+
async reopenSession(sessionId, options) {
|
|
11921
|
+
const session = await this.granular.reopenSession(sessionId, {
|
|
11922
|
+
clientId: options?.clientId
|
|
11923
|
+
});
|
|
11924
|
+
if (session.environmentId !== this.environmentId) {
|
|
11925
|
+
await session.disconnect().catch(() => {
|
|
11926
|
+
session.disconnectTransport();
|
|
11927
|
+
});
|
|
11928
|
+
throw new Error(
|
|
11929
|
+
`Session ${sessionId} belongs to environment ${session.environmentId}, not ${this.environmentId}.`
|
|
11930
|
+
);
|
|
11834
11931
|
}
|
|
11932
|
+
return session;
|
|
11933
|
+
}
|
|
11934
|
+
async closeSession(sessionId, session) {
|
|
11935
|
+
await this.granular.closeSession(sessionId, session);
|
|
11936
|
+
}
|
|
11937
|
+
async listFeedback() {
|
|
11938
|
+
const response = await this.controlPlaneRequest(`/control/environments/${this.environmentId}/feedback`);
|
|
11939
|
+
return Array.isArray(response.items) ? response.items : [];
|
|
11940
|
+
}
|
|
11941
|
+
getRuntimeBaseUrl() {
|
|
11942
|
+
return deriveRuntimeBaseUrl(this._apiEndpoint);
|
|
11835
11943
|
}
|
|
11836
11944
|
async controlPlaneRequest(path2, options = {}) {
|
|
11837
11945
|
const runtimeBase = this.getRuntimeBaseUrl();
|
|
@@ -11852,95 +11960,6 @@ var Environment = class extends Session {
|
|
|
11852
11960
|
}
|
|
11853
11961
|
return response.json();
|
|
11854
11962
|
}
|
|
11855
|
-
/**
|
|
11856
|
-
* Close the session and disconnect from the sandbox.
|
|
11857
|
-
*
|
|
11858
|
-
* Sends `client.goodbye` over WebSocket first, then issues an HTTP fallback
|
|
11859
|
-
* to the runtime goodbye endpoint if no definitive WS-side runtime notify
|
|
11860
|
-
* acknowledgement was observed.
|
|
11861
|
-
*/
|
|
11862
|
-
async disconnect() {
|
|
11863
|
-
let wsNotifiedRuntime = false;
|
|
11864
|
-
try {
|
|
11865
|
-
const goodbye = await this.rpc(
|
|
11866
|
-
"client.goodbye",
|
|
11867
|
-
{
|
|
11868
|
-
timestamp: Date.now()
|
|
11869
|
-
}
|
|
11870
|
-
);
|
|
11871
|
-
wsNotifiedRuntime = Boolean(goodbye?.ok && goodbye?.via);
|
|
11872
|
-
} catch {
|
|
11873
|
-
wsNotifiedRuntime = false;
|
|
11874
|
-
}
|
|
11875
|
-
if (!wsNotifiedRuntime) {
|
|
11876
|
-
try {
|
|
11877
|
-
const runtimeBase = this.getRuntimeBaseUrl();
|
|
11878
|
-
await fetch(
|
|
11879
|
-
`${runtimeBase}/orchestrator/runtime/environments/${this.environmentId}/session-goodbye`,
|
|
11880
|
-
{
|
|
11881
|
-
method: "POST",
|
|
11882
|
-
headers: {
|
|
11883
|
-
"Content-Type": "application/json",
|
|
11884
|
-
Authorization: `Bearer ${this._apiKey}`,
|
|
11885
|
-
Connection: "close"
|
|
11886
|
-
},
|
|
11887
|
-
body: JSON.stringify({
|
|
11888
|
-
reason: "sdk_disconnect_http_fallback",
|
|
11889
|
-
sessionId: this.client.currentSessionId
|
|
11890
|
-
})
|
|
11891
|
-
}
|
|
11892
|
-
);
|
|
11893
|
-
} catch {
|
|
11894
|
-
}
|
|
11895
|
-
}
|
|
11896
|
-
this.client.disconnect();
|
|
11897
|
-
}
|
|
11898
|
-
/**
|
|
11899
|
-
* Close only the socket transport without sending `client.goodbye`.
|
|
11900
|
-
*
|
|
11901
|
-
* Use this when the caller intends to immediately reattach to the same
|
|
11902
|
-
* session after an unexpected disconnect.
|
|
11903
|
-
*/
|
|
11904
|
-
disconnectTransport() {
|
|
11905
|
-
this.client.disconnect();
|
|
11906
|
-
}
|
|
11907
|
-
// ==================== GRAPH CONTAINER READINESS ====================
|
|
11908
|
-
/** The last known graph container status, updated by checkReadiness() or on heartbeat */
|
|
11909
|
-
graphContainerStatus = null;
|
|
11910
|
-
/**
|
|
11911
|
-
* Check if the graph container is ready and warm.
|
|
11912
|
-
*
|
|
11913
|
-
* Sends a lightweight heartbeat RPC to the Session DO which internally
|
|
11914
|
-
* pings the FalkorDB container. The response includes `graphContainerStatus`,
|
|
11915
|
-
* which is stored locally and emitted as a `readiness` event.
|
|
11916
|
-
*
|
|
11917
|
-
* Use this method to proactively warm the graph container before any
|
|
11918
|
-
* GraphQL query that requires it, or to poll the container's state in
|
|
11919
|
-
* the background.
|
|
11920
|
-
*
|
|
11921
|
-
* @returns The current graph container status object
|
|
11922
|
-
*
|
|
11923
|
-
* @example
|
|
11924
|
-
* ```typescript
|
|
11925
|
-
* const status = await env.checkReadiness();
|
|
11926
|
-
* console.log(status.status); // 'hot' | 'warming' | 'unknown'
|
|
11927
|
-
*
|
|
11928
|
-
* // Or listen for live updates
|
|
11929
|
-
* env.on('readiness', (status) => {
|
|
11930
|
-
* console.log('Graph is now:', status.status);
|
|
11931
|
-
* });
|
|
11932
|
-
* ```
|
|
11933
|
-
*/
|
|
11934
|
-
async checkReadiness() {
|
|
11935
|
-
const result = await this.client.call("client.heartbeat", {});
|
|
11936
|
-
const containerStatus = result?.graphContainerStatus ?? {
|
|
11937
|
-
lastKeepAliveAt: Date.now(),
|
|
11938
|
-
status: "unknown"
|
|
11939
|
-
};
|
|
11940
|
-
this.graphContainerStatus = containerStatus;
|
|
11941
|
-
this.emit("readiness", containerStatus);
|
|
11942
|
-
return containerStatus;
|
|
11943
|
-
}
|
|
11944
11963
|
// ==================== ID ↔ GRAPH PATH MAPPING ====================
|
|
11945
11964
|
/**
|
|
11946
11965
|
* Convert a class name + real-world ID into a unique graph path.
|
|
@@ -12801,36 +12820,186 @@ var Environment = class extends Session {
|
|
|
12801
12820
|
}
|
|
12802
12821
|
);
|
|
12803
12822
|
}
|
|
12804
|
-
|
|
12823
|
+
};
|
|
12824
|
+
var EnvironmentSession = class extends Session {
|
|
12825
|
+
environment;
|
|
12826
|
+
/** The last known graph container status, updated by checkReadiness() or on heartbeat */
|
|
12827
|
+
graphContainerStatus = null;
|
|
12828
|
+
constructor(client, environment, clientId) {
|
|
12829
|
+
super(client, clientId);
|
|
12830
|
+
this.environment = environment;
|
|
12831
|
+
}
|
|
12832
|
+
get environmentId() {
|
|
12833
|
+
return this.environment.environmentId;
|
|
12834
|
+
}
|
|
12835
|
+
get sandboxId() {
|
|
12836
|
+
return this.environment.sandboxId;
|
|
12837
|
+
}
|
|
12838
|
+
get ontologyId() {
|
|
12839
|
+
return this.environment.ontologyId;
|
|
12840
|
+
}
|
|
12841
|
+
get subjectId() {
|
|
12842
|
+
return this.environment.subjectId;
|
|
12843
|
+
}
|
|
12844
|
+
get envName() {
|
|
12845
|
+
return this.environment.envName;
|
|
12846
|
+
}
|
|
12847
|
+
get versionId() {
|
|
12848
|
+
return this.environment.versionId;
|
|
12849
|
+
}
|
|
12850
|
+
get granularId() {
|
|
12851
|
+
return this.environment.granularId;
|
|
12852
|
+
}
|
|
12853
|
+
get permissionProfileId() {
|
|
12854
|
+
return this.environment.permissionProfileId;
|
|
12855
|
+
}
|
|
12856
|
+
get apiEndpoint() {
|
|
12857
|
+
return this.environment.apiEndpoint;
|
|
12858
|
+
}
|
|
12859
|
+
get data() {
|
|
12860
|
+
return this.environment.data;
|
|
12861
|
+
}
|
|
12862
|
+
get feedback() {
|
|
12863
|
+
return this.environment.feedback;
|
|
12864
|
+
}
|
|
12805
12865
|
/**
|
|
12806
|
-
*
|
|
12866
|
+
* Return a plain JS snapshot of the synced session heap.
|
|
12807
12867
|
*/
|
|
12808
|
-
|
|
12809
|
-
|
|
12868
|
+
getHeap() {
|
|
12869
|
+
const doc = this.document;
|
|
12870
|
+
return normalizeHeapSnapshot(doc?.heap);
|
|
12871
|
+
}
|
|
12872
|
+
async graphql(query, variables) {
|
|
12873
|
+
return this.environment.graphql(query, variables);
|
|
12874
|
+
}
|
|
12875
|
+
async defineRelationship(options) {
|
|
12876
|
+
return this.environment.defineRelationship(options);
|
|
12877
|
+
}
|
|
12878
|
+
async getRelationships(modelPath) {
|
|
12879
|
+
return this.environment.getRelationships(modelPath);
|
|
12880
|
+
}
|
|
12881
|
+
async attach(modelPath, submodelPath, targetPath) {
|
|
12882
|
+
return this.environment.attach(modelPath, submodelPath, targetPath);
|
|
12883
|
+
}
|
|
12884
|
+
async detach(modelPath, submodelPath, targetPath) {
|
|
12885
|
+
return this.environment.detach(modelPath, submodelPath, targetPath);
|
|
12886
|
+
}
|
|
12887
|
+
async listRelated(modelPath, submodelPath) {
|
|
12888
|
+
return this.environment.listRelated(modelPath, submodelPath);
|
|
12889
|
+
}
|
|
12890
|
+
async applyManifest(manifest) {
|
|
12891
|
+
return this.environment.applyManifest(manifest);
|
|
12892
|
+
}
|
|
12893
|
+
async recordObject(options) {
|
|
12894
|
+
return this.environment.recordObject(options);
|
|
12895
|
+
}
|
|
12896
|
+
async recordObjects(records, options) {
|
|
12897
|
+
return this.environment.recordObjects(records, options);
|
|
12898
|
+
}
|
|
12899
|
+
async enqueueRecordImport(records, options = {}) {
|
|
12900
|
+
return this.environment.enqueueRecordImport(records, options);
|
|
12901
|
+
}
|
|
12902
|
+
async listRecordImports(status) {
|
|
12903
|
+
return this.environment.listRecordImports(status);
|
|
12904
|
+
}
|
|
12905
|
+
async getRecordImportSummary() {
|
|
12906
|
+
return this.environment.getRecordImportSummary();
|
|
12907
|
+
}
|
|
12908
|
+
async getAwaitingRecordCount() {
|
|
12909
|
+
return this.environment.getAwaitingRecordCount();
|
|
12910
|
+
}
|
|
12911
|
+
async getRecordImport(importId) {
|
|
12912
|
+
return this.environment.getRecordImport(importId);
|
|
12913
|
+
}
|
|
12914
|
+
async cancelRecordImport(importId) {
|
|
12915
|
+
return this.environment.cancelRecordImport(importId);
|
|
12916
|
+
}
|
|
12917
|
+
async listFeedback() {
|
|
12918
|
+
return this.environment.listFeedback();
|
|
12810
12919
|
}
|
|
12811
12920
|
/**
|
|
12812
|
-
*
|
|
12921
|
+
* Close the session and disconnect from the sandbox.
|
|
12922
|
+
*
|
|
12923
|
+
* Sends `client.goodbye` over WebSocket first, then issues an HTTP fallback
|
|
12924
|
+
* to the runtime goodbye endpoint if no definitive WS-side runtime notify
|
|
12925
|
+
* acknowledgement was observed.
|
|
12813
12926
|
*/
|
|
12814
|
-
async
|
|
12815
|
-
|
|
12927
|
+
async disconnect() {
|
|
12928
|
+
let wsNotifiedRuntime = false;
|
|
12929
|
+
try {
|
|
12930
|
+
const goodbye = await this.rpc(
|
|
12931
|
+
"client.goodbye",
|
|
12932
|
+
{
|
|
12933
|
+
timestamp: Date.now()
|
|
12934
|
+
}
|
|
12935
|
+
);
|
|
12936
|
+
wsNotifiedRuntime = Boolean(goodbye?.ok && goodbye?.via);
|
|
12937
|
+
} catch {
|
|
12938
|
+
wsNotifiedRuntime = false;
|
|
12939
|
+
}
|
|
12940
|
+
if (!wsNotifiedRuntime) {
|
|
12941
|
+
try {
|
|
12942
|
+
await fetch(
|
|
12943
|
+
`${this.environment.runtimeBaseUrl}/orchestrator/runtime/environments/${this.environmentId}/session-goodbye`,
|
|
12944
|
+
{
|
|
12945
|
+
method: "POST",
|
|
12946
|
+
headers: {
|
|
12947
|
+
"Content-Type": "application/json",
|
|
12948
|
+
Authorization: `Bearer ${this.environment.authToken}`,
|
|
12949
|
+
Connection: "close"
|
|
12950
|
+
},
|
|
12951
|
+
body: JSON.stringify({
|
|
12952
|
+
reason: "sdk_disconnect_http_fallback",
|
|
12953
|
+
sessionId: this.client.currentSessionId
|
|
12954
|
+
})
|
|
12955
|
+
}
|
|
12956
|
+
);
|
|
12957
|
+
} catch {
|
|
12958
|
+
}
|
|
12959
|
+
}
|
|
12960
|
+
this.client.disconnect();
|
|
12816
12961
|
}
|
|
12817
12962
|
/**
|
|
12818
|
-
*
|
|
12963
|
+
* Close only the socket transport without sending `client.goodbye`.
|
|
12819
12964
|
*/
|
|
12820
|
-
|
|
12821
|
-
|
|
12965
|
+
disconnectTransport() {
|
|
12966
|
+
this.client.disconnect();
|
|
12822
12967
|
}
|
|
12823
12968
|
/**
|
|
12824
|
-
*
|
|
12969
|
+
* Backwards-compatible alias for `disconnect()`.
|
|
12825
12970
|
*/
|
|
12826
|
-
async
|
|
12827
|
-
|
|
12971
|
+
async close() {
|
|
12972
|
+
await this.disconnect();
|
|
12828
12973
|
}
|
|
12829
12974
|
/**
|
|
12830
|
-
*
|
|
12975
|
+
* Check if the graph container is ready and warm.
|
|
12831
12976
|
*/
|
|
12832
|
-
async
|
|
12833
|
-
|
|
12977
|
+
async checkReadiness() {
|
|
12978
|
+
const result = await this.client.call("client.heartbeat", {});
|
|
12979
|
+
const containerStatus = result?.graphContainerStatus ?? {
|
|
12980
|
+
lastKeepAliveAt: Date.now(),
|
|
12981
|
+
status: "unknown"
|
|
12982
|
+
};
|
|
12983
|
+
this.graphContainerStatus = containerStatus;
|
|
12984
|
+
this.emit("readiness", containerStatus);
|
|
12985
|
+
return containerStatus;
|
|
12986
|
+
}
|
|
12987
|
+
};
|
|
12988
|
+
var OntologyHandle = class {
|
|
12989
|
+
granular;
|
|
12990
|
+
ontologyNameOrId;
|
|
12991
|
+
constructor(granular, ontologyNameOrId) {
|
|
12992
|
+
this.granular = granular;
|
|
12993
|
+
this.ontologyNameOrId = ontologyNameOrId;
|
|
12994
|
+
}
|
|
12995
|
+
get effects() {
|
|
12996
|
+
return {
|
|
12997
|
+
register: async (effect) => this.granular.registerEffect(this.ontologyNameOrId, effect),
|
|
12998
|
+
registerMany: async (effects) => this.granular.registerEffects(this.ontologyNameOrId, effects),
|
|
12999
|
+
unregister: async (name) => this.granular.unregisterEffect(this.ontologyNameOrId, name),
|
|
13000
|
+
clear: async () => this.granular.unregisterAllEffects(this.ontologyNameOrId),
|
|
13001
|
+
disconnect: async () => this.granular.disconnectEffects(this.ontologyNameOrId)
|
|
13002
|
+
};
|
|
12834
13003
|
}
|
|
12835
13004
|
};
|
|
12836
13005
|
var Granular = class _Granular {
|
|
@@ -12867,6 +13036,12 @@ var Granular = class _Granular {
|
|
|
12867
13036
|
this.onReconnectError = options.onReconnectError;
|
|
12868
13037
|
this.httpUrl = this.apiUrl.replace(/^wss:\/\//, "https://").replace(/^ws:\/\//, "http://").replace(/\/ws$/, "");
|
|
12869
13038
|
}
|
|
13039
|
+
/**
|
|
13040
|
+
* Return an ontology-scoped handle for effects and other ontology-level APIs.
|
|
13041
|
+
*/
|
|
13042
|
+
ontology(ontologyNameOrId) {
|
|
13043
|
+
return new OntologyHandle(this, ontologyNameOrId);
|
|
13044
|
+
}
|
|
12870
13045
|
/**
|
|
12871
13046
|
* Records/upserts a user and prepares them for sandbox connections
|
|
12872
13047
|
*
|
|
@@ -12903,7 +13078,23 @@ var Granular = class _Granular {
|
|
|
12903
13078
|
permissions: options.permissions || []
|
|
12904
13079
|
});
|
|
12905
13080
|
}
|
|
13081
|
+
/**
|
|
13082
|
+
* Alias for `recordUser()` with user-facing naming that matches upsert semantics.
|
|
13083
|
+
*/
|
|
13084
|
+
async upsertUser(options) {
|
|
13085
|
+
return this.recordUser(options);
|
|
13086
|
+
}
|
|
12906
13087
|
async resolveConnectUser(options) {
|
|
13088
|
+
const providedIdentityCount = [
|
|
13089
|
+
Boolean(options.user),
|
|
13090
|
+
Boolean(options.userId),
|
|
13091
|
+
Boolean(options.granularId)
|
|
13092
|
+
].filter(Boolean).length;
|
|
13093
|
+
if (providedIdentityCount !== 1) {
|
|
13094
|
+
throw new Error(
|
|
13095
|
+
"openEnvironment() requires exactly one of userId, granularId, or a user object returned by recordUser()."
|
|
13096
|
+
);
|
|
13097
|
+
}
|
|
12907
13098
|
if (options.user) {
|
|
12908
13099
|
const user = normalizeUser(options.user);
|
|
12909
13100
|
return {
|
|
@@ -12940,56 +13131,85 @@ var Granular = class _Granular {
|
|
|
12940
13131
|
};
|
|
12941
13132
|
}
|
|
12942
13133
|
throw new Error(
|
|
12943
|
-
"
|
|
13134
|
+
"openEnvironment() requires either userId, granularId, or a user object returned by recordUser()."
|
|
12944
13135
|
);
|
|
12945
13136
|
}
|
|
12946
13137
|
/**
|
|
12947
|
-
*
|
|
12948
|
-
*
|
|
12949
|
-
* Effects are registered at the sandbox level via `granular.registerEffect()`
|
|
12950
|
-
* or `granular.registerEffects()`. Sessions pick up live availability from
|
|
12951
|
-
* the sandbox registry automatically.
|
|
12952
|
-
*
|
|
12953
|
-
* @param options - Connection options
|
|
12954
|
-
* @returns An active environment session
|
|
13138
|
+
* Open or resolve an ontology environment for one user without opening a session.
|
|
12955
13139
|
*
|
|
12956
13140
|
* @example
|
|
12957
13141
|
* ```typescript
|
|
12958
|
-
* const environment = await granular.
|
|
13142
|
+
* const environment = await granular.openEnvironment({
|
|
12959
13143
|
* ontology: 'my-ontology',
|
|
12960
|
-
*
|
|
13144
|
+
* tag: 'dev',
|
|
12961
13145
|
* userId: 'user_123',
|
|
12962
13146
|
* permissions: ['agent'],
|
|
12963
13147
|
* });
|
|
12964
13148
|
*
|
|
12965
|
-
* await
|
|
12966
|
-
*
|
|
12967
|
-
*
|
|
12968
|
-
*
|
|
12969
|
-
* handler: async () => 'Hello!',
|
|
13149
|
+
* await environment.data.record({
|
|
13150
|
+
* className: 'customer',
|
|
13151
|
+
* id: 'acme',
|
|
13152
|
+
* fields: { name: 'Acme' },
|
|
12970
13153
|
* });
|
|
12971
13154
|
*
|
|
12972
|
-
*
|
|
12973
|
-
* const job = await
|
|
12974
|
-
*
|
|
12975
|
-
* return await tools.greet({});
|
|
12976
|
-
* `);
|
|
12977
|
-
*
|
|
12978
|
-
* console.log(await job.result); // 'Hello!'
|
|
13155
|
+
* const session = await environment.sessions.create();
|
|
13156
|
+
* const job = await session.submitJob(`return "hello";`);
|
|
13157
|
+
* console.log(await job.result);
|
|
12979
13158
|
* ```
|
|
12980
13159
|
*/
|
|
13160
|
+
async openEnvironment(options) {
|
|
13161
|
+
const envData = await this.resolveOpenEnvironmentData(
|
|
13162
|
+
options,
|
|
13163
|
+
"openEnvironment"
|
|
13164
|
+
);
|
|
13165
|
+
return this.bindEnvironmentHandle(envData);
|
|
13166
|
+
}
|
|
13167
|
+
/**
|
|
13168
|
+
* Deprecated compatibility alias for `openEnvironment()`.
|
|
13169
|
+
*
|
|
13170
|
+
* `connect()` no longer opens a runtime session automatically.
|
|
13171
|
+
*/
|
|
12981
13172
|
async connect(options) {
|
|
12982
|
-
|
|
13173
|
+
return this.openEnvironment({
|
|
13174
|
+
...options,
|
|
13175
|
+
tag: this.resolveRequestedTag(options, "connect"),
|
|
13176
|
+
permissions: options.permissions || options.user?.permissions || []
|
|
13177
|
+
});
|
|
13178
|
+
}
|
|
13179
|
+
resolveRequestedTag(options, methodName) {
|
|
13180
|
+
const tag = options.tag?.trim() || options.tagName?.trim() || options.environment?.trim();
|
|
13181
|
+
if (!tag) {
|
|
13182
|
+
throw new Error(`${methodName}() requires \`tag\`.`);
|
|
13183
|
+
}
|
|
13184
|
+
return tag;
|
|
13185
|
+
}
|
|
13186
|
+
buildManagedEnvironmentName(tag, versionId) {
|
|
13187
|
+
return `__sdk__${tag}__${versionId}`;
|
|
13188
|
+
}
|
|
13189
|
+
matchesTagTrackedEnvironment(environment, tagName, tagId) {
|
|
13190
|
+
const environmentTagName = environment.tag?.name || environment.buildPolicy.tagName || null;
|
|
13191
|
+
return environment.tagId === tagId || environmentTagName === tagName || environment.environment === tagName || environment.envName === tagName || environment.environment === this.buildManagedEnvironmentName(tagName, environment.versionId) || environment.envName === this.buildManagedEnvironmentName(tagName, environment.versionId);
|
|
13192
|
+
}
|
|
13193
|
+
sortEnvironmentsByRecency(environments) {
|
|
13194
|
+
return [...environments].sort(
|
|
13195
|
+
(left, right) => right.updatedAt - left.updatedAt
|
|
13196
|
+
);
|
|
13197
|
+
}
|
|
13198
|
+
async resolveOpenEnvironmentData(options, methodName) {
|
|
12983
13199
|
const ontology = options.ontology;
|
|
12984
13200
|
if (!ontology) {
|
|
12985
|
-
throw new Error(
|
|
13201
|
+
throw new Error(`${methodName}() requires \`ontology\`.`);
|
|
12986
13202
|
}
|
|
12987
|
-
const
|
|
12988
|
-
if (!
|
|
12989
|
-
throw new Error(
|
|
13203
|
+
const tagName = options.tag?.trim();
|
|
13204
|
+
if (!tagName) {
|
|
13205
|
+
throw new Error(`${methodName}() requires \`tag\`.`);
|
|
12990
13206
|
}
|
|
12991
|
-
const tagName = options.tagName?.trim() || void 0;
|
|
12992
13207
|
const user = await this.resolveConnectUser(options);
|
|
13208
|
+
if (!Array.isArray(user.permissions) || user.permissions.length === 0) {
|
|
13209
|
+
throw new Error(
|
|
13210
|
+
`${methodName}() requires at least one permission so the SDK can ensure assignments for new users.`
|
|
13211
|
+
);
|
|
13212
|
+
}
|
|
12993
13213
|
const sandbox = await this.findOrCreateSandbox(ontology);
|
|
12994
13214
|
for (const profileName of user.permissions) {
|
|
12995
13215
|
const profileId = await this.ensurePermissionProfile(
|
|
@@ -13002,22 +13222,49 @@ var Granular = class _Granular {
|
|
|
13002
13222
|
profileId
|
|
13003
13223
|
);
|
|
13004
13224
|
}
|
|
13005
|
-
const
|
|
13225
|
+
const tags = await this.request(
|
|
13226
|
+
`/control/sandboxes/${sandbox.sandboxId}/tags`
|
|
13227
|
+
);
|
|
13228
|
+
const tag = (tags.items || []).find(
|
|
13229
|
+
(candidate) => Boolean(candidate?.name === tagName)
|
|
13230
|
+
);
|
|
13231
|
+
if (!tag) {
|
|
13232
|
+
throw new Error(
|
|
13233
|
+
`Tag "${tagName}" was not found for ontology ${sandbox.sandboxId}.`
|
|
13234
|
+
);
|
|
13235
|
+
}
|
|
13236
|
+
const targetVersionId = tag.targetVersionId || tag.targetBuildId;
|
|
13237
|
+
if (!targetVersionId) {
|
|
13238
|
+
throw new Error(
|
|
13239
|
+
`Tag "${tagName}" does not currently point to a build/version.`
|
|
13240
|
+
);
|
|
13241
|
+
}
|
|
13242
|
+
const allEnvironments = await this.environments.list(sandbox.sandboxId);
|
|
13243
|
+
const userEnvironments = allEnvironments.filter(
|
|
13244
|
+
(environment) => environment.subjectId === user.granularId
|
|
13245
|
+
);
|
|
13246
|
+
const currentMatches = this.sortEnvironmentsByRecency(
|
|
13247
|
+
userEnvironments.filter(
|
|
13248
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId) && environment.versionId === targetVersionId
|
|
13249
|
+
)
|
|
13250
|
+
);
|
|
13251
|
+
if (currentMatches.length > 0) {
|
|
13252
|
+
return currentMatches[0];
|
|
13253
|
+
}
|
|
13254
|
+
const outdatedMatches = this.sortEnvironmentsByRecency(
|
|
13255
|
+
userEnvironments.filter(
|
|
13256
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId)
|
|
13257
|
+
)
|
|
13258
|
+
);
|
|
13259
|
+
if (outdatedMatches.length > 0 && options.createFreshIfOutdated !== true) {
|
|
13260
|
+
return outdatedMatches[0];
|
|
13261
|
+
}
|
|
13262
|
+
return this.environments.create(sandbox.sandboxId, {
|
|
13006
13263
|
subjectId: user.granularId,
|
|
13007
|
-
environment:
|
|
13008
|
-
|
|
13264
|
+
environment: this.buildManagedEnvironmentName(tagName, targetVersionId),
|
|
13265
|
+
tagId: tag.tagId,
|
|
13009
13266
|
permissionProfileId: null
|
|
13010
13267
|
});
|
|
13011
|
-
await this.activateEnvironment(envData.environmentId);
|
|
13012
|
-
const session = await this.request("/ws/sessions", {
|
|
13013
|
-
method: "POST",
|
|
13014
|
-
body: JSON.stringify({
|
|
13015
|
-
environmentId: envData.environmentId,
|
|
13016
|
-
clientId,
|
|
13017
|
-
initialHeap: options.initialHeap
|
|
13018
|
-
})
|
|
13019
|
-
});
|
|
13020
|
-
return this.bindWebSocketEnvironment(envData, clientId, session);
|
|
13021
13268
|
}
|
|
13022
13269
|
/**
|
|
13023
13270
|
* List active (open) sessions for an environment — each session is one agent conversation thread.
|
|
@@ -13080,6 +13327,7 @@ var Granular = class _Granular {
|
|
|
13080
13327
|
const clientId = options.clientId || `client_${Date.now()}`;
|
|
13081
13328
|
await this.activateEnvironment(options.environmentId);
|
|
13082
13329
|
const envData = await this.environments.get(options.environmentId);
|
|
13330
|
+
const environment = this.bindEnvironmentHandle(envData);
|
|
13083
13331
|
const session = await this.request("/ws/sessions", {
|
|
13084
13332
|
method: "POST",
|
|
13085
13333
|
body: JSON.stringify({
|
|
@@ -13088,7 +13336,7 @@ var Granular = class _Granular {
|
|
|
13088
13336
|
initialHeap: options.initialHeap
|
|
13089
13337
|
})
|
|
13090
13338
|
});
|
|
13091
|
-
return this.
|
|
13339
|
+
return this.bindWebSocketEnvironmentSession(environment, clientId, session);
|
|
13092
13340
|
}
|
|
13093
13341
|
/**
|
|
13094
13342
|
* Connect to an existing open session (same conversation thread) using a freshly minted WebSocket token.
|
|
@@ -13100,7 +13348,8 @@ var Granular = class _Granular {
|
|
|
13100
13348
|
body: JSON.stringify({})
|
|
13101
13349
|
});
|
|
13102
13350
|
const envData = await this.environments.get(minted.environmentId);
|
|
13103
|
-
|
|
13351
|
+
const environment = this.bindEnvironmentHandle(envData);
|
|
13352
|
+
return this.bindWebSocketEnvironmentSession(environment, clientId, minted);
|
|
13104
13353
|
}
|
|
13105
13354
|
/**
|
|
13106
13355
|
* Mark a session closed in the control plane. If `environment` is the connected handle for that
|
|
@@ -13132,7 +13381,11 @@ var Granular = class _Granular {
|
|
|
13132
13381
|
});
|
|
13133
13382
|
return this.connectSession({ sessionId, clientId: options?.clientId });
|
|
13134
13383
|
}
|
|
13135
|
-
|
|
13384
|
+
bindEnvironmentHandle(envData) {
|
|
13385
|
+
const graphqlEndpoint = `${this.httpUrl}/orchestrator/graphql`;
|
|
13386
|
+
return new Environment(this, envData, this.apiKey, graphqlEndpoint);
|
|
13387
|
+
}
|
|
13388
|
+
async bindWebSocketEnvironmentSession(environment, clientId, session) {
|
|
13136
13389
|
const client = new WSClient({
|
|
13137
13390
|
url: session.wsUrl,
|
|
13138
13391
|
sessionId: session.sessionId,
|
|
@@ -13143,16 +13396,13 @@ var Granular = class _Granular {
|
|
|
13143
13396
|
onReconnectError: this.onReconnectError
|
|
13144
13397
|
});
|
|
13145
13398
|
await client.connect();
|
|
13146
|
-
const
|
|
13147
|
-
const environment = new Environment(
|
|
13399
|
+
const environmentSession = new EnvironmentSession(
|
|
13148
13400
|
client,
|
|
13149
|
-
|
|
13150
|
-
clientId
|
|
13151
|
-
this.apiKey,
|
|
13152
|
-
graphqlEndpoint
|
|
13401
|
+
environment,
|
|
13402
|
+
clientId
|
|
13153
13403
|
);
|
|
13154
|
-
await
|
|
13155
|
-
return
|
|
13404
|
+
await environmentSession.hello();
|
|
13405
|
+
return environmentSession;
|
|
13156
13406
|
}
|
|
13157
13407
|
async activateEnvironment(environmentId) {
|
|
13158
13408
|
await this.request(`/orchestrator/runtime/environments/${environmentId}/activate`, {
|
|
@@ -15091,19 +15341,27 @@ function matchesPattern(text, matcher) {
|
|
|
15091
15341
|
function assertMatches(label, text, includes = [], excludes = []) {
|
|
15092
15342
|
for (const matcher of includes) {
|
|
15093
15343
|
if (!matchesPattern(text, matcher)) {
|
|
15094
|
-
throw new Error(
|
|
15095
|
-
${
|
|
15344
|
+
throw new Error(
|
|
15345
|
+
`${label} did not match ${matcherToString(matcher)}:
|
|
15346
|
+
${text}`
|
|
15347
|
+
);
|
|
15096
15348
|
}
|
|
15097
15349
|
}
|
|
15098
15350
|
for (const matcher of excludes) {
|
|
15099
15351
|
if (matchesPattern(text, matcher)) {
|
|
15100
|
-
throw new Error(
|
|
15101
|
-
${
|
|
15352
|
+
throw new Error(
|
|
15353
|
+
`${label} matched forbidden ${matcherToString(matcher)}:
|
|
15354
|
+
${text}`
|
|
15355
|
+
);
|
|
15102
15356
|
}
|
|
15103
15357
|
}
|
|
15104
15358
|
}
|
|
15105
15359
|
function buildArtifactDir(baseDir, suiteName = "granular-agent-evals") {
|
|
15106
|
-
return path__default.default.join(
|
|
15360
|
+
return path__default.default.join(
|
|
15361
|
+
baseDir || path__default.default.join(process.cwd(), "test-artifacts"),
|
|
15362
|
+
suiteName,
|
|
15363
|
+
timestampId()
|
|
15364
|
+
);
|
|
15107
15365
|
}
|
|
15108
15366
|
function createTimestampedArtifactDirectory(options) {
|
|
15109
15367
|
return buildArtifactDir(options?.baseDir, options?.suiteName);
|
|
@@ -15117,17 +15375,16 @@ function buildScenarioSteps(scenario) {
|
|
|
15117
15375
|
return scenario.steps;
|
|
15118
15376
|
}
|
|
15119
15377
|
if (!scenario.request) {
|
|
15120
|
-
throw new Error(
|
|
15378
|
+
throw new Error(
|
|
15379
|
+
`Scenario ${scenario.id} must provide either request or steps`
|
|
15380
|
+
);
|
|
15121
15381
|
}
|
|
15122
15382
|
const compatibilityStep = {
|
|
15123
15383
|
id: scenario.id,
|
|
15124
15384
|
request: scenario.request,
|
|
15125
15385
|
human: scenario.human,
|
|
15126
15386
|
expect: scenario.expect,
|
|
15127
|
-
inspect: [
|
|
15128
|
-
...asArray2(scenario.inspect),
|
|
15129
|
-
...asArray2(scenario.verify)
|
|
15130
|
-
],
|
|
15387
|
+
inspect: [...asArray2(scenario.inspect), ...asArray2(scenario.verify)],
|
|
15131
15388
|
check: scenario.check,
|
|
15132
15389
|
maxIterations: scenario.maxIterations,
|
|
15133
15390
|
setup: {
|
|
@@ -15145,22 +15402,27 @@ function buildAssistantHistoryContent(entry) {
|
|
|
15145
15402
|
${entry.content}`);
|
|
15146
15403
|
if (entry.jobStatus) parts.push(`[Job status]
|
|
15147
15404
|
${entry.jobStatus}`);
|
|
15148
|
-
if (entry.jobResultPreview)
|
|
15405
|
+
if (entry.jobResultPreview)
|
|
15406
|
+
parts.push(`[Job result]
|
|
15149
15407
|
${entry.jobResultPreview}`);
|
|
15150
15408
|
if (entry.error) parts.push(`[Job error]
|
|
15151
15409
|
${entry.error}`);
|
|
15152
15410
|
return parts.join("\n\n") || entry.content;
|
|
15153
15411
|
}
|
|
15154
15412
|
function buildHistory(entries) {
|
|
15155
|
-
return entries.reduce(
|
|
15156
|
-
|
|
15157
|
-
if (entry.
|
|
15413
|
+
return entries.reduce(
|
|
15414
|
+
(history, entry) => {
|
|
15415
|
+
if (entry.role === "user") {
|
|
15416
|
+
if (entry.content.trim())
|
|
15417
|
+
history.push({ role: "user", content: entry.content });
|
|
15418
|
+
return history;
|
|
15419
|
+
}
|
|
15420
|
+
const content = buildAssistantHistoryContent(entry).trim();
|
|
15421
|
+
if (content) history.push({ role: "assistant", content });
|
|
15158
15422
|
return history;
|
|
15159
|
-
}
|
|
15160
|
-
|
|
15161
|
-
|
|
15162
|
-
return history;
|
|
15163
|
-
}, []);
|
|
15423
|
+
},
|
|
15424
|
+
[]
|
|
15425
|
+
);
|
|
15164
15426
|
}
|
|
15165
15427
|
function getOpenPromptsFromDoc(liveDoc) {
|
|
15166
15428
|
const jobsById = asRecord4(asRecord4(liveDoc?.jobs)?.byId) || {};
|
|
@@ -15169,7 +15431,8 @@ function getOpenPromptsFromDoc(liveDoc) {
|
|
|
15169
15431
|
const promptRecords = asRecord4(asRecord4(job)?.prompts) || {};
|
|
15170
15432
|
for (const raw of Object.values(promptRecords)) {
|
|
15171
15433
|
const record = asRecord4(raw);
|
|
15172
|
-
if (!record || record.status !== "open" || typeof record.promptId !== "string")
|
|
15434
|
+
if (!record || record.status !== "open" || typeof record.promptId !== "string")
|
|
15435
|
+
continue;
|
|
15173
15436
|
const prompt = normalizePrompt({
|
|
15174
15437
|
promptId: record.promptId,
|
|
15175
15438
|
kind: record.kind,
|
|
@@ -15213,7 +15476,9 @@ ${prompt.message || ""}`;
|
|
|
15213
15476
|
return resolvePromptAnswer(prompt, rawAnswer);
|
|
15214
15477
|
}
|
|
15215
15478
|
if (fallback) return fallback({ prompt, history });
|
|
15216
|
-
throw new Error(
|
|
15479
|
+
throw new Error(
|
|
15480
|
+
`No scripted prompt responder matched prompt ${prompt.id}: ${promptText}`
|
|
15481
|
+
);
|
|
15217
15482
|
};
|
|
15218
15483
|
}
|
|
15219
15484
|
function extractJsonObject(text) {
|
|
@@ -15237,13 +15502,19 @@ function modelOutputInstruction() {
|
|
|
15237
15502
|
].join("\n");
|
|
15238
15503
|
}
|
|
15239
15504
|
function createOpenAIChatTurnGenerator(options) {
|
|
15240
|
-
const baseUrl = (options.baseUrl || "https://api.openai.com/v1").replace(
|
|
15505
|
+
const baseUrl = (options.baseUrl || "https://api.openai.com/v1").replace(
|
|
15506
|
+
/\/$/,
|
|
15507
|
+
""
|
|
15508
|
+
);
|
|
15241
15509
|
const model = options.model || "gpt-5-mini";
|
|
15242
15510
|
return async (input) => {
|
|
15243
15511
|
const messages = [
|
|
15244
|
-
{
|
|
15512
|
+
{
|
|
15513
|
+
role: "system",
|
|
15514
|
+
content: `${input.systemPrompt}
|
|
15245
15515
|
|
|
15246
|
-
${modelOutputInstruction()}`
|
|
15516
|
+
${modelOutputInstruction()}`
|
|
15517
|
+
},
|
|
15247
15518
|
...input.history,
|
|
15248
15519
|
{ role: "user", content: input.request }
|
|
15249
15520
|
];
|
|
@@ -15272,10 +15543,14 @@ ${modelOutputInstruction()}` },
|
|
|
15272
15543
|
await sleep2(500 * attempt);
|
|
15273
15544
|
continue;
|
|
15274
15545
|
}
|
|
15275
|
-
throw new Error(
|
|
15546
|
+
throw new Error(
|
|
15547
|
+
`OpenAI chat generation failed: ${response.status} ${errorText}`
|
|
15548
|
+
);
|
|
15276
15549
|
}
|
|
15277
15550
|
const raw = await response.json();
|
|
15278
|
-
const content = asRecord4(
|
|
15551
|
+
const content = asRecord4(
|
|
15552
|
+
asRecord4(raw.choices?.[0])?.message
|
|
15553
|
+
)?.content;
|
|
15279
15554
|
const text = typeof content === "string" ? content : Array.isArray(content) ? content.map((part) => asRecord4(part)?.text || "").join("") : "";
|
|
15280
15555
|
const parsed = extractJsonObject(text);
|
|
15281
15556
|
if (!parsed) {
|
|
@@ -15293,7 +15568,9 @@ ${text}`);
|
|
|
15293
15568
|
};
|
|
15294
15569
|
} catch (error) {
|
|
15295
15570
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
15296
|
-
if (attempt < 3 && /socket connection was closed unexpectedly|ECONNRESET|network|timed out/i.test(
|
|
15571
|
+
if (attempt < 3 && /socket connection was closed unexpectedly|ECONNRESET|network|timed out/i.test(
|
|
15572
|
+
lastError.message
|
|
15573
|
+
)) {
|
|
15297
15574
|
await sleep2(500 * attempt);
|
|
15298
15575
|
continue;
|
|
15299
15576
|
}
|
|
@@ -15315,7 +15592,10 @@ async function withTimeout(promise, ms, label) {
|
|
|
15315
15592
|
return await Promise.race([
|
|
15316
15593
|
promise,
|
|
15317
15594
|
new Promise((_, reject) => {
|
|
15318
|
-
timeoutId = setTimeout(
|
|
15595
|
+
timeoutId = setTimeout(
|
|
15596
|
+
() => reject(new Error(`${label} timed out after ${ms}ms`)),
|
|
15597
|
+
ms
|
|
15598
|
+
);
|
|
15319
15599
|
})
|
|
15320
15600
|
]);
|
|
15321
15601
|
} finally {
|
|
@@ -15325,7 +15605,9 @@ async function withTimeout(promise, ms, label) {
|
|
|
15325
15605
|
function getActionSummary(liveDoc, jobId) {
|
|
15326
15606
|
const jobsById = asRecord4(asRecord4(liveDoc?.jobs)?.byId) || {};
|
|
15327
15607
|
const job = asRecord4(jobsById[jobId]);
|
|
15328
|
-
return Array.isArray(job?.actionSummary) ? job.actionSummary.filter(
|
|
15608
|
+
return Array.isArray(job?.actionSummary) ? job.actionSummary.filter(
|
|
15609
|
+
(line) => typeof line === "string"
|
|
15610
|
+
) : [];
|
|
15329
15611
|
}
|
|
15330
15612
|
function normalizeHeapSnapshot2(heap) {
|
|
15331
15613
|
return {
|
|
@@ -15355,12 +15637,20 @@ async function waitForJobOutcome(input) {
|
|
|
15355
15637
|
const startedAt = Date.now();
|
|
15356
15638
|
while (Date.now() - startedAt < input.timeoutMs) {
|
|
15357
15639
|
const liveDoc = cloneJson(input.environment.document);
|
|
15358
|
-
const prompts = filterPromptsByBoundary(
|
|
15640
|
+
const prompts = filterPromptsByBoundary(
|
|
15641
|
+
liveDoc,
|
|
15642
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
15643
|
+
input.boundaryTimestamp
|
|
15644
|
+
);
|
|
15359
15645
|
if (prompts.length > 0) {
|
|
15360
15646
|
return { kind: "prompt", prompts, liveDoc, stdout, stderr };
|
|
15361
15647
|
}
|
|
15362
15648
|
try {
|
|
15363
|
-
const result = await withTimeout(
|
|
15649
|
+
const result = await withTimeout(
|
|
15650
|
+
input.job.result,
|
|
15651
|
+
input.pollIntervalMs,
|
|
15652
|
+
`job ${input.job.id} tick`
|
|
15653
|
+
);
|
|
15364
15654
|
return { kind: "completed", result, liveDoc, stdout, stderr };
|
|
15365
15655
|
} catch (error) {
|
|
15366
15656
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -15426,7 +15716,9 @@ function buildResultReport(result) {
|
|
|
15426
15716
|
...result.actionSummary.length ? result.actionSummary.map((line) => `- ${line}`) : ["- None"],
|
|
15427
15717
|
"",
|
|
15428
15718
|
"## Prompt Interactions",
|
|
15429
|
-
...result.promptInteractions.length ? result.promptInteractions.map(
|
|
15719
|
+
...result.promptInteractions.length ? result.promptInteractions.map(
|
|
15720
|
+
(interaction) => `- [${interaction.type}] ${interaction.message || interaction.title} -> ${JSON.stringify(interaction.answer)}`
|
|
15721
|
+
) : ["- None"],
|
|
15430
15722
|
"",
|
|
15431
15723
|
...stepSection,
|
|
15432
15724
|
"## Raw Files",
|
|
@@ -15441,7 +15733,9 @@ function buildSuiteIndex(results) {
|
|
|
15441
15733
|
const lines = [
|
|
15442
15734
|
"# Agent Eval Report Index",
|
|
15443
15735
|
"",
|
|
15444
|
-
...results.map(
|
|
15736
|
+
...results.map(
|
|
15737
|
+
(result) => `- [${result.scenario.id}](./${result.scenario.id}/REPORT.md) - ${result.status}`
|
|
15738
|
+
)
|
|
15445
15739
|
];
|
|
15446
15740
|
return `${lines.join("\n")}
|
|
15447
15741
|
`;
|
|
@@ -15455,7 +15749,7 @@ async function applySetup(setup, context) {
|
|
|
15455
15749
|
await context.environment.recordObjects(setup.records);
|
|
15456
15750
|
}
|
|
15457
15751
|
if (setup.effects?.length) {
|
|
15458
|
-
await context.environment.
|
|
15752
|
+
await context.granular.ontology(context.environment.sandboxId).effects.registerMany(setup.effects);
|
|
15459
15753
|
}
|
|
15460
15754
|
if (setup.run) {
|
|
15461
15755
|
await setup.run(context);
|
|
@@ -15477,6 +15771,7 @@ async function runAgentEvalSuite(options) {
|
|
|
15477
15771
|
);
|
|
15478
15772
|
try {
|
|
15479
15773
|
await applySetup(scenario.setup, {
|
|
15774
|
+
granular: options.harness.granular,
|
|
15480
15775
|
conversation,
|
|
15481
15776
|
environment: conversation.environment,
|
|
15482
15777
|
turnDir: conversation.artifactDir
|
|
@@ -15489,14 +15784,19 @@ async function runAgentEvalSuite(options) {
|
|
|
15489
15784
|
conversation,
|
|
15490
15785
|
request: step.request,
|
|
15491
15786
|
prepare: async (ctx) => {
|
|
15492
|
-
await applySetup(step.setup,
|
|
15787
|
+
await applySetup(step.setup, {
|
|
15788
|
+
granular: options.harness.granular,
|
|
15789
|
+
...ctx
|
|
15790
|
+
});
|
|
15493
15791
|
},
|
|
15494
15792
|
human: step.human,
|
|
15495
15793
|
maxIterations: step.maxIterations,
|
|
15496
15794
|
autoAnswerPrompts: step.autoAnswerPrompts
|
|
15497
15795
|
});
|
|
15498
15796
|
if ("prompts" in completed) {
|
|
15499
|
-
throw new Error(
|
|
15797
|
+
throw new Error(
|
|
15798
|
+
`Scenario ${scenario.id} step ${index + 1} paused for human input instead of completing automatically`
|
|
15799
|
+
);
|
|
15500
15800
|
}
|
|
15501
15801
|
const inspectionResults = [];
|
|
15502
15802
|
const stepChecks = asArray2(step.check);
|
|
@@ -15512,12 +15812,23 @@ async function runAgentEvalSuite(options) {
|
|
|
15512
15812
|
actionSummary: completed.actionSummary,
|
|
15513
15813
|
promptInteractions: completed.promptInteractions,
|
|
15514
15814
|
result: completed.result,
|
|
15515
|
-
heap: normalizeHeapSnapshot2(
|
|
15516
|
-
|
|
15815
|
+
heap: normalizeHeapSnapshot2(
|
|
15816
|
+
asRecord4(
|
|
15817
|
+
cloneJson(conversation.environment.document)?.heap
|
|
15818
|
+
)
|
|
15819
|
+
),
|
|
15820
|
+
openPrompts: getOpenPromptsFromDoc(
|
|
15821
|
+
cloneJson(conversation.environment.document)
|
|
15822
|
+
),
|
|
15517
15823
|
liveDoc: cloneJson(conversation.environment.document),
|
|
15518
15824
|
inspect: async (code) => {
|
|
15519
|
-
const
|
|
15520
|
-
|
|
15825
|
+
const session = conversation.environment;
|
|
15826
|
+
const job = await session.submitJob(code);
|
|
15827
|
+
return withTimeout(
|
|
15828
|
+
job.result,
|
|
15829
|
+
9e4,
|
|
15830
|
+
`inspection job ${job.id}`
|
|
15831
|
+
);
|
|
15521
15832
|
},
|
|
15522
15833
|
assertMatches
|
|
15523
15834
|
};
|
|
@@ -15575,7 +15886,9 @@ async function runAgentEvalSuite(options) {
|
|
|
15575
15886
|
}
|
|
15576
15887
|
const lastStep = stepResults[stepResults.length - 1];
|
|
15577
15888
|
if (!lastStep) {
|
|
15578
|
-
throw new Error(
|
|
15889
|
+
throw new Error(
|
|
15890
|
+
`Scenario ${scenario.id} produced no completed steps`
|
|
15891
|
+
);
|
|
15579
15892
|
}
|
|
15580
15893
|
const result = {
|
|
15581
15894
|
scenario,
|
|
@@ -15589,9 +15902,18 @@ async function runAgentEvalSuite(options) {
|
|
|
15589
15902
|
steps: stepResults,
|
|
15590
15903
|
turnDir: conversation.artifactDir
|
|
15591
15904
|
};
|
|
15592
|
-
await writeJson(
|
|
15593
|
-
|
|
15594
|
-
|
|
15905
|
+
await writeJson(
|
|
15906
|
+
path__default.default.join(conversation.artifactDir, "result.json"),
|
|
15907
|
+
result
|
|
15908
|
+
);
|
|
15909
|
+
await writeJson(
|
|
15910
|
+
path__default.default.join(conversation.artifactDir, "report.json"),
|
|
15911
|
+
result
|
|
15912
|
+
);
|
|
15913
|
+
await promises.writeFile(
|
|
15914
|
+
path__default.default.join(conversation.artifactDir, "REPORT.md"),
|
|
15915
|
+
buildResultReport(result)
|
|
15916
|
+
);
|
|
15595
15917
|
finalResult = result;
|
|
15596
15918
|
} catch (error) {
|
|
15597
15919
|
const failureMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -15613,7 +15935,10 @@ async function runAgentEvalSuite(options) {
|
|
|
15613
15935
|
await ensureDir(failed.turnDir);
|
|
15614
15936
|
await writeJson(path__default.default.join(failed.turnDir, "result.json"), failed);
|
|
15615
15937
|
await writeJson(path__default.default.join(failed.turnDir, "report.json"), failed);
|
|
15616
|
-
await promises.writeFile(
|
|
15938
|
+
await promises.writeFile(
|
|
15939
|
+
path__default.default.join(failed.turnDir, "REPORT.md"),
|
|
15940
|
+
buildResultReport(failed)
|
|
15941
|
+
);
|
|
15617
15942
|
finalResult = failed;
|
|
15618
15943
|
} finally {
|
|
15619
15944
|
await options.harness.closeConversation(conversation);
|
|
@@ -15634,12 +15959,21 @@ async function runAgentEvalSuite(options) {
|
|
|
15634
15959
|
}
|
|
15635
15960
|
results.push(finalResult);
|
|
15636
15961
|
}
|
|
15637
|
-
await writeJson(
|
|
15638
|
-
|
|
15962
|
+
await writeJson(
|
|
15963
|
+
path__default.default.join(options.harness.artifactDir, "summary.json"),
|
|
15964
|
+
results
|
|
15965
|
+
);
|
|
15966
|
+
await promises.writeFile(
|
|
15967
|
+
path__default.default.join(options.harness.artifactDir, "REPORT_INDEX.md"),
|
|
15968
|
+
buildSuiteIndex(results)
|
|
15969
|
+
);
|
|
15639
15970
|
return { artifactDir: options.harness.artifactDir, results };
|
|
15640
15971
|
}
|
|
15641
15972
|
function createAgentEvalHarness(options) {
|
|
15642
|
-
const artifactDir = buildArtifactDir(
|
|
15973
|
+
const artifactDir = buildArtifactDir(
|
|
15974
|
+
options.artifactBaseDir,
|
|
15975
|
+
options.suiteName
|
|
15976
|
+
);
|
|
15643
15977
|
const controllerBudgets = {
|
|
15644
15978
|
...DEFAULT_CONTROLLER_BUDGETS,
|
|
15645
15979
|
...options.controllerBudgets || {}
|
|
@@ -15651,7 +15985,9 @@ function createAgentEvalHarness(options) {
|
|
|
15651
15985
|
await ensureDir(artifactDir);
|
|
15652
15986
|
const clientId = `${slugify(label)}-${Date.now()}`;
|
|
15653
15987
|
if (!options.openEnvironment && !options.environmentId) {
|
|
15654
|
-
throw new Error(
|
|
15988
|
+
throw new Error(
|
|
15989
|
+
"createAgentEvalHarness requires either environmentId or openEnvironment"
|
|
15990
|
+
);
|
|
15655
15991
|
}
|
|
15656
15992
|
const environment = options.openEnvironment ? await options.openEnvironment({ label, clientId }) : await options.granular.createSession({
|
|
15657
15993
|
environmentId: options.environmentId,
|
|
@@ -15675,12 +16011,15 @@ function createAgentEvalHarness(options) {
|
|
|
15675
16011
|
}
|
|
15676
16012
|
async function closeConversation(conversation) {
|
|
15677
16013
|
try {
|
|
15678
|
-
await options.granular.closeSession(
|
|
16014
|
+
await options.granular.closeSession(
|
|
16015
|
+
conversation.environment.sessionId,
|
|
16016
|
+
conversation.environment
|
|
16017
|
+
);
|
|
15679
16018
|
} catch {
|
|
15680
16019
|
}
|
|
15681
16020
|
}
|
|
15682
|
-
async function runCheckJob(code,
|
|
15683
|
-
const job = await
|
|
16021
|
+
async function runCheckJob(code, session) {
|
|
16022
|
+
const job = await session.submitJob(code);
|
|
15684
16023
|
return withTimeout(job.result, jobTimeoutMs, `check job ${job.id}`);
|
|
15685
16024
|
}
|
|
15686
16025
|
function buildCheckContext(conversation, completed, turnDir) {
|
|
@@ -15725,8 +16064,12 @@ function createAgentEvalHarness(options) {
|
|
|
15725
16064
|
async function resumePendingTurn(pending, responder) {
|
|
15726
16065
|
const prompt = pending.prompts[0];
|
|
15727
16066
|
if (!prompt) throw new Error("Pending turn has no prompts to answer");
|
|
15728
|
-
const answer = await responder({
|
|
15729
|
-
|
|
16067
|
+
const answer = await responder({
|
|
16068
|
+
prompt,
|
|
16069
|
+
history: pending.promptInteractions
|
|
16070
|
+
});
|
|
16071
|
+
const session = pending.conversation.environment;
|
|
16072
|
+
await session.answerPrompt(prompt.id, answer);
|
|
15730
16073
|
pending.promptInteractions.push({
|
|
15731
16074
|
promptId: prompt.id,
|
|
15732
16075
|
type: prompt.type,
|
|
@@ -15750,7 +16093,9 @@ function createAgentEvalHarness(options) {
|
|
|
15750
16093
|
};
|
|
15751
16094
|
}
|
|
15752
16095
|
await sleep2(350);
|
|
15753
|
-
const liveDoc = cloneJson(
|
|
16096
|
+
const liveDoc = cloneJson(
|
|
16097
|
+
pending.conversation.environment.document
|
|
16098
|
+
);
|
|
15754
16099
|
const presentation = resolveJobPresentation({
|
|
15755
16100
|
jobId: pending.job.id,
|
|
15756
16101
|
result: resumed.result,
|
|
@@ -15795,25 +16140,40 @@ function createAgentEvalHarness(options) {
|
|
|
15795
16140
|
await conversation.environment.recordObjects(input.prepareRecords);
|
|
15796
16141
|
}
|
|
15797
16142
|
if (input.prepareTools?.length) {
|
|
15798
|
-
await conversation.environment.
|
|
16143
|
+
await options.granular.ontology(conversation.environment.sandboxId).effects.registerMany(input.prepareTools);
|
|
15799
16144
|
}
|
|
15800
16145
|
if (input.prepare) {
|
|
15801
|
-
await input.prepare({
|
|
16146
|
+
await input.prepare({
|
|
16147
|
+
conversation,
|
|
16148
|
+
environment: conversation.environment,
|
|
16149
|
+
turnDir
|
|
16150
|
+
});
|
|
15802
16151
|
}
|
|
15803
16152
|
const boundaryTimestamp = Date.now();
|
|
15804
16153
|
conversation.history.push({ role: "user", content: input.request });
|
|
15805
|
-
await writeJson(path__default.default.join(turnDir, "request.json"), {
|
|
16154
|
+
await writeJson(path__default.default.join(turnDir, "request.json"), {
|
|
16155
|
+
request: input.request,
|
|
16156
|
+
boundaryTimestamp
|
|
16157
|
+
});
|
|
15806
16158
|
let iteration = 0;
|
|
15807
16159
|
let noProgressCount = 0;
|
|
15808
16160
|
let previousSnapshot = null;
|
|
15809
16161
|
let latestCheckpoint = null;
|
|
15810
16162
|
const maxIterations = input.maxIterations || controllerBudgets.maxIterations;
|
|
15811
16163
|
const autoAnswerPrompts = input.autoAnswerPrompts ?? true;
|
|
15812
|
-
const baselineClosureId = getCurrentClosureId(
|
|
16164
|
+
const baselineClosureId = getCurrentClosureId(
|
|
16165
|
+
cloneJson(conversation.environment.document)
|
|
16166
|
+
);
|
|
15813
16167
|
while (iteration < maxIterations) {
|
|
15814
16168
|
const liveDoc = cloneJson(conversation.environment.document);
|
|
15815
|
-
const pendingPrompts = filterPromptsByBoundary(
|
|
15816
|
-
|
|
16169
|
+
const pendingPrompts = filterPromptsByBoundary(
|
|
16170
|
+
liveDoc,
|
|
16171
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
16172
|
+
boundaryTimestamp
|
|
16173
|
+
);
|
|
16174
|
+
const workflowFocus = projectWorkflowFocus(liveDoc, pendingPrompts, {
|
|
16175
|
+
boundaryTimestamp
|
|
16176
|
+
});
|
|
15817
16177
|
const systemPrompt = buildGranularAgentSystemPrompt({
|
|
15818
16178
|
domainDocumentation: await conversation.environment.getDomainDocumentation(),
|
|
15819
16179
|
sessionContext: {
|
|
@@ -15824,8 +16184,12 @@ function createAgentEvalHarness(options) {
|
|
|
15824
16184
|
heapSummary: projectHeapSummary(liveDoc, {
|
|
15825
16185
|
focus: workflowFocus
|
|
15826
16186
|
}),
|
|
15827
|
-
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
15828
|
-
|
|
16187
|
+
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
16188
|
+
boundaryTimestamp
|
|
16189
|
+
}),
|
|
16190
|
+
workflowSummary: projectWorkflowSummary(liveDoc, pendingPrompts, {
|
|
16191
|
+
boundaryTimestamp
|
|
16192
|
+
}),
|
|
15829
16193
|
tools: conversation.environment.getEffects().map((tool) => ({
|
|
15830
16194
|
name: tool.name,
|
|
15831
16195
|
description: tool.description,
|
|
@@ -15835,14 +16199,23 @@ function createAgentEvalHarness(options) {
|
|
|
15835
16199
|
})),
|
|
15836
16200
|
checkpoint: latestCheckpoint
|
|
15837
16201
|
});
|
|
15838
|
-
const request = iteration === 0 ? input.request : buildContinuationInstruction(
|
|
15839
|
-
|
|
15840
|
-
|
|
15841
|
-
|
|
15842
|
-
|
|
15843
|
-
|
|
15844
|
-
|
|
15845
|
-
|
|
16202
|
+
const request = iteration === 0 ? input.request : buildContinuationInstruction(
|
|
16203
|
+
buildContinuationPreview(latestCheckpoint, noProgressCount)
|
|
16204
|
+
);
|
|
16205
|
+
const generation = await withTimeout(
|
|
16206
|
+
generateTurnWithRepair(options.generator, {
|
|
16207
|
+
systemPrompt,
|
|
16208
|
+
history: buildHistory(conversation.history),
|
|
16209
|
+
request,
|
|
16210
|
+
attempt: 1
|
|
16211
|
+
}),
|
|
16212
|
+
chatTimeoutMs,
|
|
16213
|
+
`chat generation for ${conversation.label} iteration ${iteration + 1}`
|
|
16214
|
+
);
|
|
16215
|
+
await writeJson(
|
|
16216
|
+
path__default.default.join(turnDir, `iteration-${iteration + 1}-generation.json`),
|
|
16217
|
+
generation
|
|
16218
|
+
);
|
|
15846
16219
|
if (!generation.code) {
|
|
15847
16220
|
const responseText2 = generation.reply?.trim() || "Done.";
|
|
15848
16221
|
conversation.history.push({ role: "assistant", content: responseText2 });
|
|
@@ -15858,12 +16231,18 @@ function createAgentEvalHarness(options) {
|
|
|
15858
16231
|
result: generation.reply?.trim() || responseText2
|
|
15859
16232
|
};
|
|
15860
16233
|
if (input.verification) {
|
|
15861
|
-
completed.verification = await runInspection(
|
|
16234
|
+
completed.verification = await runInspection(
|
|
16235
|
+
conversation,
|
|
16236
|
+
input.verification,
|
|
16237
|
+
completed,
|
|
16238
|
+
turnDir
|
|
16239
|
+
);
|
|
15862
16240
|
}
|
|
15863
16241
|
await writeJson(path__default.default.join(turnDir, "result.json"), completed);
|
|
15864
16242
|
return completed;
|
|
15865
16243
|
}
|
|
15866
|
-
const
|
|
16244
|
+
const session = conversation.environment;
|
|
16245
|
+
const job = await session.submitJob(generation.code);
|
|
15867
16246
|
const outcome = await waitForJobOutcome({
|
|
15868
16247
|
environment: conversation.environment,
|
|
15869
16248
|
job,
|
|
@@ -15888,7 +16267,9 @@ function createAgentEvalHarness(options) {
|
|
|
15888
16267
|
};
|
|
15889
16268
|
}
|
|
15890
16269
|
if (!input.human) {
|
|
15891
|
-
throw new Error(
|
|
16270
|
+
throw new Error(
|
|
16271
|
+
"This turn reached a human prompt but no responder was provided"
|
|
16272
|
+
);
|
|
15892
16273
|
}
|
|
15893
16274
|
let pending = {
|
|
15894
16275
|
conversation,
|
|
@@ -15910,16 +16291,25 @@ function createAgentEvalHarness(options) {
|
|
|
15910
16291
|
continue;
|
|
15911
16292
|
}
|
|
15912
16293
|
if (input.verification) {
|
|
15913
|
-
resumed.verification = await runInspection(
|
|
16294
|
+
resumed.verification = await runInspection(
|
|
16295
|
+
conversation,
|
|
16296
|
+
input.verification,
|
|
16297
|
+
resumed,
|
|
16298
|
+
turnDir
|
|
16299
|
+
);
|
|
15914
16300
|
}
|
|
15915
16301
|
return resumed;
|
|
15916
16302
|
}
|
|
15917
16303
|
}
|
|
15918
16304
|
if (outcome.kind !== "completed") {
|
|
15919
|
-
throw new Error(
|
|
16305
|
+
throw new Error(
|
|
16306
|
+
"Unexpected non-completed outcome after prompt handling"
|
|
16307
|
+
);
|
|
15920
16308
|
}
|
|
15921
16309
|
await sleep2(350);
|
|
15922
|
-
const settledLiveDoc = cloneJson(
|
|
16310
|
+
const settledLiveDoc = cloneJson(
|
|
16311
|
+
conversation.environment.document
|
|
16312
|
+
);
|
|
15923
16313
|
const sessionHeap = normalizeHeapSnapshot2(asRecord4(settledLiveDoc?.heap));
|
|
15924
16314
|
const presentation = resolveJobPresentation({
|
|
15925
16315
|
jobId: job.id,
|
|
@@ -15940,7 +16330,11 @@ function createAgentEvalHarness(options) {
|
|
|
15940
16330
|
baselineClosureId,
|
|
15941
16331
|
currentClosureId: getCurrentClosureId(settledLiveDoc),
|
|
15942
16332
|
liveDoc: settledLiveDoc,
|
|
15943
|
-
pendingPrompts: filterPromptsByBoundary(
|
|
16333
|
+
pendingPrompts: filterPromptsByBoundary(
|
|
16334
|
+
settledLiveDoc,
|
|
16335
|
+
getOpenPromptsFromDoc(settledLiveDoc),
|
|
16336
|
+
boundaryTimestamp
|
|
16337
|
+
),
|
|
15944
16338
|
projectionOptions: { boundaryTimestamp },
|
|
15945
16339
|
latestResponseText: responseText,
|
|
15946
16340
|
previousSnapshot,
|
|
@@ -15965,12 +16359,15 @@ function createAgentEvalHarness(options) {
|
|
|
15965
16359
|
jobStatus: "succeeded",
|
|
15966
16360
|
jobResultPreview: JSON.stringify(outcome.result, null, 2)
|
|
15967
16361
|
});
|
|
15968
|
-
await writeJson(
|
|
15969
|
-
|
|
15970
|
-
|
|
15971
|
-
|
|
15972
|
-
|
|
15973
|
-
|
|
16362
|
+
await writeJson(
|
|
16363
|
+
path__default.default.join(turnDir, `iteration-${iteration + 1}-result.json`),
|
|
16364
|
+
{
|
|
16365
|
+
responseText,
|
|
16366
|
+
continuation,
|
|
16367
|
+
actionSummary: latestCheckpoint.latestActionSummary,
|
|
16368
|
+
result: outcome.result
|
|
16369
|
+
}
|
|
16370
|
+
);
|
|
15974
16371
|
if (!continuation.shouldContinue) {
|
|
15975
16372
|
const completed = {
|
|
15976
16373
|
conversation,
|
|
@@ -15985,17 +16382,25 @@ function createAgentEvalHarness(options) {
|
|
|
15985
16382
|
result: outcome.result
|
|
15986
16383
|
};
|
|
15987
16384
|
if (input.verification) {
|
|
15988
|
-
completed.verification = await runInspection(
|
|
16385
|
+
completed.verification = await runInspection(
|
|
16386
|
+
conversation,
|
|
16387
|
+
input.verification,
|
|
16388
|
+
completed,
|
|
16389
|
+
turnDir
|
|
16390
|
+
);
|
|
15989
16391
|
}
|
|
15990
16392
|
await writeJson(path__default.default.join(turnDir, "result.json"), completed);
|
|
15991
16393
|
return completed;
|
|
15992
16394
|
}
|
|
15993
16395
|
iteration += 1;
|
|
15994
16396
|
}
|
|
15995
|
-
throw new Error(
|
|
16397
|
+
throw new Error(
|
|
16398
|
+
`Turn exceeded iteration budget (${maxIterations}) for ${conversation.label}`
|
|
16399
|
+
);
|
|
15996
16400
|
}
|
|
15997
16401
|
return {
|
|
15998
16402
|
artifactDir,
|
|
16403
|
+
granular: options.granular,
|
|
15999
16404
|
openConversation,
|
|
16000
16405
|
closeConversation,
|
|
16001
16406
|
runTurn,
|
|
@@ -16031,12 +16436,11 @@ function createAgentTester(options) {
|
|
|
16031
16436
|
}
|
|
16032
16437
|
if ("connect" in options.target && !connectSeeded) {
|
|
16033
16438
|
connectSeeded = true;
|
|
16034
|
-
const environment = await granular.
|
|
16035
|
-
...options.target.connect
|
|
16036
|
-
clientId
|
|
16439
|
+
const environment = await granular.openEnvironment({
|
|
16440
|
+
...options.target.connect
|
|
16037
16441
|
});
|
|
16038
16442
|
resolvedEnvironmentId = environment.environmentId;
|
|
16039
|
-
return environment;
|
|
16443
|
+
return environment.sessions.create({ clientId });
|
|
16040
16444
|
}
|
|
16041
16445
|
if ("createEnvironment" in options.target && !resolvedEnvironmentId) {
|
|
16042
16446
|
const envData = await granular.environments.create(
|