@granular-software/sdk 0.4.30 → 0.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -20
- package/dist/agent-evals.d.mts +7 -6
- package/dist/agent-evals.d.ts +7 -6
- package/dist/agent-evals.js +681 -277
- package/dist/agent-evals.js.map +1 -1
- package/dist/agent-evals.mjs +681 -277
- package/dist/agent-evals.mjs.map +1 -1
- package/dist/cli/index.js +526 -271
- package/dist/{client-BVvOMfln.d.mts → client-CTH1hfwe.d.mts} +254 -123
- package/dist/{client-BVvOMfln.d.ts → client-CTH1hfwe.d.ts} +254 -123
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +439 -187
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +438 -188
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/agent-evals.mjs
CHANGED
|
@@ -4649,27 +4649,27 @@ var Session = class {
|
|
|
4649
4649
|
}
|
|
4650
4650
|
async publishTools(tools, revision = "1.0.0") {
|
|
4651
4651
|
throw new Error(
|
|
4652
|
-
"Environment-scoped effect publication was removed. Declare effects in the manifest and register live handlers with granular.
|
|
4652
|
+
"Environment-scoped effect publication was removed. Declare effects in the manifest and register live handlers with granular.ontology(environment.sandboxId).effects.registerMany(effects)."
|
|
4653
4653
|
);
|
|
4654
4654
|
}
|
|
4655
4655
|
async publishEffect(effect) {
|
|
4656
4656
|
throw new Error(
|
|
4657
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4657
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.register(effect)."
|
|
4658
4658
|
);
|
|
4659
4659
|
}
|
|
4660
4660
|
async publishEffects(effects) {
|
|
4661
4661
|
throw new Error(
|
|
4662
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4662
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.registerMany(effects)."
|
|
4663
4663
|
);
|
|
4664
4664
|
}
|
|
4665
4665
|
async unpublishEffect(name) {
|
|
4666
4666
|
throw new Error(
|
|
4667
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4667
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.unregister(effectName)."
|
|
4668
4668
|
);
|
|
4669
4669
|
}
|
|
4670
4670
|
async unpublishAllEffects() {
|
|
4671
4671
|
throw new Error(
|
|
4672
|
-
"Environment-scoped effect publication was removed. Use granular.
|
|
4672
|
+
"Environment-scoped effect publication was removed. Use granular.ontology(environment.sandboxId).effects.clear()."
|
|
4673
4673
|
);
|
|
4674
4674
|
}
|
|
4675
4675
|
/**
|
|
@@ -11693,6 +11693,22 @@ function normalizeHeapSnapshot(raw) {
|
|
|
11693
11693
|
updatedAt: typeof heap.updatedAt === "number" ? heap.updatedAt : Date.now()
|
|
11694
11694
|
};
|
|
11695
11695
|
}
|
|
11696
|
+
function deriveRuntimeBaseUrl(apiEndpoint) {
|
|
11697
|
+
try {
|
|
11698
|
+
const endpoint = new URL(apiEndpoint);
|
|
11699
|
+
const graphqlSuffix = "/orchestrator/graphql";
|
|
11700
|
+
if (endpoint.pathname.endsWith(graphqlSuffix)) {
|
|
11701
|
+
endpoint.pathname = endpoint.pathname.slice(0, -graphqlSuffix.length);
|
|
11702
|
+
} else if (endpoint.pathname.endsWith("/graphql")) {
|
|
11703
|
+
endpoint.pathname = endpoint.pathname.slice(0, -"/graphql".length);
|
|
11704
|
+
}
|
|
11705
|
+
endpoint.search = "";
|
|
11706
|
+
endpoint.hash = "";
|
|
11707
|
+
return endpoint.toString().replace(/\/$/, "");
|
|
11708
|
+
} catch {
|
|
11709
|
+
return apiEndpoint.replace(/\/orchestrator\/graphql$/, "").replace(/\/$/, "");
|
|
11710
|
+
}
|
|
11711
|
+
}
|
|
11696
11712
|
function normalizeSubject(subject) {
|
|
11697
11713
|
const granularId = subject.granularId || subject.subjectId;
|
|
11698
11714
|
const userId = subject.userId || subject.identityId || granularId;
|
|
@@ -11732,12 +11748,13 @@ function normalizeEnvironmentData(environment) {
|
|
|
11732
11748
|
tracking: environment.tracking || buildPolicy
|
|
11733
11749
|
};
|
|
11734
11750
|
}
|
|
11735
|
-
var Environment = class
|
|
11751
|
+
var Environment = class {
|
|
11752
|
+
granular;
|
|
11736
11753
|
envData;
|
|
11737
11754
|
_apiKey;
|
|
11738
11755
|
_apiEndpoint;
|
|
11739
|
-
constructor(
|
|
11740
|
-
|
|
11756
|
+
constructor(granular, envData, apiKey, apiEndpoint) {
|
|
11757
|
+
this.granular = granular;
|
|
11741
11758
|
this.envData = envData;
|
|
11742
11759
|
this._apiKey = apiKey;
|
|
11743
11760
|
this._apiEndpoint = apiEndpoint;
|
|
@@ -11778,35 +11795,126 @@ var Environment = class extends Session {
|
|
|
11778
11795
|
get permissionProfileId() {
|
|
11779
11796
|
return this.envData.permissionProfileId;
|
|
11780
11797
|
}
|
|
11798
|
+
/** The current build policy backing this environment */
|
|
11799
|
+
get buildPolicy() {
|
|
11800
|
+
return this.envData.buildPolicy;
|
|
11801
|
+
}
|
|
11802
|
+
/** The current update state relative to the followed tag */
|
|
11803
|
+
get updateState() {
|
|
11804
|
+
return this.envData.updateState;
|
|
11805
|
+
}
|
|
11806
|
+
/** Convenience flag for whether this environment trails the current tag target */
|
|
11807
|
+
get isOutdated() {
|
|
11808
|
+
return this.envData.updateState === "update_available";
|
|
11809
|
+
}
|
|
11810
|
+
/** The followed tag name when this environment is tag-tracked */
|
|
11811
|
+
get tag() {
|
|
11812
|
+
return this.envData.tag?.name || this.envData.buildPolicy.tagName || null;
|
|
11813
|
+
}
|
|
11781
11814
|
/** The GraphQL API endpoint URL */
|
|
11782
11815
|
get apiEndpoint() {
|
|
11783
11816
|
return this._apiEndpoint;
|
|
11784
11817
|
}
|
|
11818
|
+
/** Internal auth token used for control-plane and runtime fallback requests */
|
|
11819
|
+
get authToken() {
|
|
11820
|
+
return this._apiKey;
|
|
11821
|
+
}
|
|
11822
|
+
/** Base runtime URL derived from the GraphQL endpoint */
|
|
11823
|
+
get runtimeBaseUrl() {
|
|
11824
|
+
return this.getRuntimeBaseUrl();
|
|
11825
|
+
}
|
|
11826
|
+
get sessions() {
|
|
11827
|
+
return {
|
|
11828
|
+
list: async (options) => this.listSessions(options?.status || "active"),
|
|
11829
|
+
create: async (options) => this.createSession(options),
|
|
11830
|
+
connect: async (sessionId, options) => this.connectSession(sessionId, options),
|
|
11831
|
+
reopen: async (sessionId, options) => this.reopenSession(sessionId, options),
|
|
11832
|
+
close: async (sessionId, session) => this.closeSession(sessionId, session)
|
|
11833
|
+
};
|
|
11834
|
+
}
|
|
11835
|
+
get data() {
|
|
11836
|
+
return {
|
|
11837
|
+
record: async (record) => this.recordObject(record),
|
|
11838
|
+
recordMany: async (records, options) => this.recordObjects(records, options),
|
|
11839
|
+
import: async (records, options) => this.enqueueRecordImport(records, options),
|
|
11840
|
+
listImports: async (status) => this.listRecordImports(status),
|
|
11841
|
+
getImport: async (importId) => this.getRecordImport(importId),
|
|
11842
|
+
getImportSummary: async () => this.getRecordImportSummary(),
|
|
11843
|
+
cancelImport: async (importId) => this.cancelRecordImport(importId),
|
|
11844
|
+
getAwaitingCount: async () => this.getAwaitingRecordCount()
|
|
11845
|
+
};
|
|
11846
|
+
}
|
|
11847
|
+
get feedback() {
|
|
11848
|
+
return {
|
|
11849
|
+
list: async () => this.listFeedback()
|
|
11850
|
+
};
|
|
11851
|
+
}
|
|
11785
11852
|
/**
|
|
11786
|
-
*
|
|
11787
|
-
*
|
|
11788
|
-
*
|
|
11789
|
-
*
|
|
11853
|
+
* Sessionless environments do not own a live transport, so disconnecting the
|
|
11854
|
+
* environment handle itself is a no-op. This keeps the public surface
|
|
11855
|
+
* symmetric with `EnvironmentSession.disconnect()` and lets callers always
|
|
11856
|
+
* clean up safely without tracking whether they currently hold an environment
|
|
11857
|
+
* or a session.
|
|
11790
11858
|
*/
|
|
11791
|
-
|
|
11792
|
-
const doc = this.document;
|
|
11793
|
-
return normalizeHeapSnapshot(doc?.heap);
|
|
11859
|
+
async disconnect() {
|
|
11794
11860
|
}
|
|
11795
|
-
|
|
11796
|
-
|
|
11797
|
-
const
|
|
11798
|
-
|
|
11799
|
-
|
|
11800
|
-
|
|
11801
|
-
|
|
11802
|
-
|
|
11803
|
-
|
|
11804
|
-
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
|
|
11861
|
+
async listSessions(status = "active") {
|
|
11862
|
+
if (status === "all") {
|
|
11863
|
+
const [active, closed] = await Promise.all([
|
|
11864
|
+
this.granular.listOpenSessions({ environmentId: this.environmentId }),
|
|
11865
|
+
this.granular.listClosedSessions({ environmentId: this.environmentId })
|
|
11866
|
+
]);
|
|
11867
|
+
return [...active, ...closed].sort(
|
|
11868
|
+
(left, right) => Date.parse(right.lastSeenAt) - Date.parse(left.lastSeenAt)
|
|
11869
|
+
);
|
|
11870
|
+
}
|
|
11871
|
+
return status === "closed" ? this.granular.listClosedSessions({ environmentId: this.environmentId }) : this.granular.listOpenSessions({ environmentId: this.environmentId });
|
|
11872
|
+
}
|
|
11873
|
+
async createSession(options) {
|
|
11874
|
+
return this.granular.createSession({
|
|
11875
|
+
environmentId: this.environmentId,
|
|
11876
|
+
clientId: options?.clientId,
|
|
11877
|
+
initialHeap: options?.initialHeap
|
|
11878
|
+
});
|
|
11879
|
+
}
|
|
11880
|
+
async connectSession(sessionId, options) {
|
|
11881
|
+
const session = await this.granular["connectSession"]({
|
|
11882
|
+
sessionId,
|
|
11883
|
+
clientId: options?.clientId
|
|
11884
|
+
});
|
|
11885
|
+
if (session.environmentId !== this.environmentId) {
|
|
11886
|
+
await session.disconnect().catch(() => {
|
|
11887
|
+
session.disconnectTransport();
|
|
11888
|
+
});
|
|
11889
|
+
throw new Error(
|
|
11890
|
+
`Session ${sessionId} belongs to environment ${session.environmentId}, not ${this.environmentId}.`
|
|
11891
|
+
);
|
|
11892
|
+
}
|
|
11893
|
+
return session;
|
|
11894
|
+
}
|
|
11895
|
+
async reopenSession(sessionId, options) {
|
|
11896
|
+
const session = await this.granular.reopenSession(sessionId, {
|
|
11897
|
+
clientId: options?.clientId
|
|
11898
|
+
});
|
|
11899
|
+
if (session.environmentId !== this.environmentId) {
|
|
11900
|
+
await session.disconnect().catch(() => {
|
|
11901
|
+
session.disconnectTransport();
|
|
11902
|
+
});
|
|
11903
|
+
throw new Error(
|
|
11904
|
+
`Session ${sessionId} belongs to environment ${session.environmentId}, not ${this.environmentId}.`
|
|
11905
|
+
);
|
|
11809
11906
|
}
|
|
11907
|
+
return session;
|
|
11908
|
+
}
|
|
11909
|
+
async closeSession(sessionId, session) {
|
|
11910
|
+
await this.granular.closeSession(sessionId, session);
|
|
11911
|
+
}
|
|
11912
|
+
async listFeedback() {
|
|
11913
|
+
const response = await this.controlPlaneRequest(`/control/environments/${this.environmentId}/feedback`);
|
|
11914
|
+
return Array.isArray(response.items) ? response.items : [];
|
|
11915
|
+
}
|
|
11916
|
+
getRuntimeBaseUrl() {
|
|
11917
|
+
return deriveRuntimeBaseUrl(this._apiEndpoint);
|
|
11810
11918
|
}
|
|
11811
11919
|
async controlPlaneRequest(path2, options = {}) {
|
|
11812
11920
|
const runtimeBase = this.getRuntimeBaseUrl();
|
|
@@ -11827,95 +11935,6 @@ var Environment = class extends Session {
|
|
|
11827
11935
|
}
|
|
11828
11936
|
return response.json();
|
|
11829
11937
|
}
|
|
11830
|
-
/**
|
|
11831
|
-
* Close the session and disconnect from the sandbox.
|
|
11832
|
-
*
|
|
11833
|
-
* Sends `client.goodbye` over WebSocket first, then issues an HTTP fallback
|
|
11834
|
-
* to the runtime goodbye endpoint if no definitive WS-side runtime notify
|
|
11835
|
-
* acknowledgement was observed.
|
|
11836
|
-
*/
|
|
11837
|
-
async disconnect() {
|
|
11838
|
-
let wsNotifiedRuntime = false;
|
|
11839
|
-
try {
|
|
11840
|
-
const goodbye = await this.rpc(
|
|
11841
|
-
"client.goodbye",
|
|
11842
|
-
{
|
|
11843
|
-
timestamp: Date.now()
|
|
11844
|
-
}
|
|
11845
|
-
);
|
|
11846
|
-
wsNotifiedRuntime = Boolean(goodbye?.ok && goodbye?.via);
|
|
11847
|
-
} catch {
|
|
11848
|
-
wsNotifiedRuntime = false;
|
|
11849
|
-
}
|
|
11850
|
-
if (!wsNotifiedRuntime) {
|
|
11851
|
-
try {
|
|
11852
|
-
const runtimeBase = this.getRuntimeBaseUrl();
|
|
11853
|
-
await fetch(
|
|
11854
|
-
`${runtimeBase}/orchestrator/runtime/environments/${this.environmentId}/session-goodbye`,
|
|
11855
|
-
{
|
|
11856
|
-
method: "POST",
|
|
11857
|
-
headers: {
|
|
11858
|
-
"Content-Type": "application/json",
|
|
11859
|
-
Authorization: `Bearer ${this._apiKey}`,
|
|
11860
|
-
Connection: "close"
|
|
11861
|
-
},
|
|
11862
|
-
body: JSON.stringify({
|
|
11863
|
-
reason: "sdk_disconnect_http_fallback",
|
|
11864
|
-
sessionId: this.client.currentSessionId
|
|
11865
|
-
})
|
|
11866
|
-
}
|
|
11867
|
-
);
|
|
11868
|
-
} catch {
|
|
11869
|
-
}
|
|
11870
|
-
}
|
|
11871
|
-
this.client.disconnect();
|
|
11872
|
-
}
|
|
11873
|
-
/**
|
|
11874
|
-
* Close only the socket transport without sending `client.goodbye`.
|
|
11875
|
-
*
|
|
11876
|
-
* Use this when the caller intends to immediately reattach to the same
|
|
11877
|
-
* session after an unexpected disconnect.
|
|
11878
|
-
*/
|
|
11879
|
-
disconnectTransport() {
|
|
11880
|
-
this.client.disconnect();
|
|
11881
|
-
}
|
|
11882
|
-
// ==================== GRAPH CONTAINER READINESS ====================
|
|
11883
|
-
/** The last known graph container status, updated by checkReadiness() or on heartbeat */
|
|
11884
|
-
graphContainerStatus = null;
|
|
11885
|
-
/**
|
|
11886
|
-
* Check if the graph container is ready and warm.
|
|
11887
|
-
*
|
|
11888
|
-
* Sends a lightweight heartbeat RPC to the Session DO which internally
|
|
11889
|
-
* pings the FalkorDB container. The response includes `graphContainerStatus`,
|
|
11890
|
-
* which is stored locally and emitted as a `readiness` event.
|
|
11891
|
-
*
|
|
11892
|
-
* Use this method to proactively warm the graph container before any
|
|
11893
|
-
* GraphQL query that requires it, or to poll the container's state in
|
|
11894
|
-
* the background.
|
|
11895
|
-
*
|
|
11896
|
-
* @returns The current graph container status object
|
|
11897
|
-
*
|
|
11898
|
-
* @example
|
|
11899
|
-
* ```typescript
|
|
11900
|
-
* const status = await env.checkReadiness();
|
|
11901
|
-
* console.log(status.status); // 'hot' | 'warming' | 'unknown'
|
|
11902
|
-
*
|
|
11903
|
-
* // Or listen for live updates
|
|
11904
|
-
* env.on('readiness', (status) => {
|
|
11905
|
-
* console.log('Graph is now:', status.status);
|
|
11906
|
-
* });
|
|
11907
|
-
* ```
|
|
11908
|
-
*/
|
|
11909
|
-
async checkReadiness() {
|
|
11910
|
-
const result = await this.client.call("client.heartbeat", {});
|
|
11911
|
-
const containerStatus = result?.graphContainerStatus ?? {
|
|
11912
|
-
lastKeepAliveAt: Date.now(),
|
|
11913
|
-
status: "unknown"
|
|
11914
|
-
};
|
|
11915
|
-
this.graphContainerStatus = containerStatus;
|
|
11916
|
-
this.emit("readiness", containerStatus);
|
|
11917
|
-
return containerStatus;
|
|
11918
|
-
}
|
|
11919
11938
|
// ==================== ID ↔ GRAPH PATH MAPPING ====================
|
|
11920
11939
|
/**
|
|
11921
11940
|
* Convert a class name + real-world ID into a unique graph path.
|
|
@@ -12776,36 +12795,186 @@ var Environment = class extends Session {
|
|
|
12776
12795
|
}
|
|
12777
12796
|
);
|
|
12778
12797
|
}
|
|
12779
|
-
|
|
12798
|
+
};
|
|
12799
|
+
var EnvironmentSession = class extends Session {
|
|
12800
|
+
environment;
|
|
12801
|
+
/** The last known graph container status, updated by checkReadiness() or on heartbeat */
|
|
12802
|
+
graphContainerStatus = null;
|
|
12803
|
+
constructor(client, environment, clientId) {
|
|
12804
|
+
super(client, clientId);
|
|
12805
|
+
this.environment = environment;
|
|
12806
|
+
}
|
|
12807
|
+
get environmentId() {
|
|
12808
|
+
return this.environment.environmentId;
|
|
12809
|
+
}
|
|
12810
|
+
get sandboxId() {
|
|
12811
|
+
return this.environment.sandboxId;
|
|
12812
|
+
}
|
|
12813
|
+
get ontologyId() {
|
|
12814
|
+
return this.environment.ontologyId;
|
|
12815
|
+
}
|
|
12816
|
+
get subjectId() {
|
|
12817
|
+
return this.environment.subjectId;
|
|
12818
|
+
}
|
|
12819
|
+
get envName() {
|
|
12820
|
+
return this.environment.envName;
|
|
12821
|
+
}
|
|
12822
|
+
get versionId() {
|
|
12823
|
+
return this.environment.versionId;
|
|
12824
|
+
}
|
|
12825
|
+
get granularId() {
|
|
12826
|
+
return this.environment.granularId;
|
|
12827
|
+
}
|
|
12828
|
+
get permissionProfileId() {
|
|
12829
|
+
return this.environment.permissionProfileId;
|
|
12830
|
+
}
|
|
12831
|
+
get apiEndpoint() {
|
|
12832
|
+
return this.environment.apiEndpoint;
|
|
12833
|
+
}
|
|
12834
|
+
get data() {
|
|
12835
|
+
return this.environment.data;
|
|
12836
|
+
}
|
|
12837
|
+
get feedback() {
|
|
12838
|
+
return this.environment.feedback;
|
|
12839
|
+
}
|
|
12780
12840
|
/**
|
|
12781
|
-
*
|
|
12841
|
+
* Return a plain JS snapshot of the synced session heap.
|
|
12782
12842
|
*/
|
|
12783
|
-
|
|
12784
|
-
|
|
12843
|
+
getHeap() {
|
|
12844
|
+
const doc = this.document;
|
|
12845
|
+
return normalizeHeapSnapshot(doc?.heap);
|
|
12846
|
+
}
|
|
12847
|
+
async graphql(query, variables) {
|
|
12848
|
+
return this.environment.graphql(query, variables);
|
|
12849
|
+
}
|
|
12850
|
+
async defineRelationship(options) {
|
|
12851
|
+
return this.environment.defineRelationship(options);
|
|
12852
|
+
}
|
|
12853
|
+
async getRelationships(modelPath) {
|
|
12854
|
+
return this.environment.getRelationships(modelPath);
|
|
12855
|
+
}
|
|
12856
|
+
async attach(modelPath, submodelPath, targetPath) {
|
|
12857
|
+
return this.environment.attach(modelPath, submodelPath, targetPath);
|
|
12858
|
+
}
|
|
12859
|
+
async detach(modelPath, submodelPath, targetPath) {
|
|
12860
|
+
return this.environment.detach(modelPath, submodelPath, targetPath);
|
|
12861
|
+
}
|
|
12862
|
+
async listRelated(modelPath, submodelPath) {
|
|
12863
|
+
return this.environment.listRelated(modelPath, submodelPath);
|
|
12864
|
+
}
|
|
12865
|
+
async applyManifest(manifest) {
|
|
12866
|
+
return this.environment.applyManifest(manifest);
|
|
12867
|
+
}
|
|
12868
|
+
async recordObject(options) {
|
|
12869
|
+
return this.environment.recordObject(options);
|
|
12870
|
+
}
|
|
12871
|
+
async recordObjects(records, options) {
|
|
12872
|
+
return this.environment.recordObjects(records, options);
|
|
12873
|
+
}
|
|
12874
|
+
async enqueueRecordImport(records, options = {}) {
|
|
12875
|
+
return this.environment.enqueueRecordImport(records, options);
|
|
12876
|
+
}
|
|
12877
|
+
async listRecordImports(status) {
|
|
12878
|
+
return this.environment.listRecordImports(status);
|
|
12879
|
+
}
|
|
12880
|
+
async getRecordImportSummary() {
|
|
12881
|
+
return this.environment.getRecordImportSummary();
|
|
12882
|
+
}
|
|
12883
|
+
async getAwaitingRecordCount() {
|
|
12884
|
+
return this.environment.getAwaitingRecordCount();
|
|
12885
|
+
}
|
|
12886
|
+
async getRecordImport(importId) {
|
|
12887
|
+
return this.environment.getRecordImport(importId);
|
|
12888
|
+
}
|
|
12889
|
+
async cancelRecordImport(importId) {
|
|
12890
|
+
return this.environment.cancelRecordImport(importId);
|
|
12891
|
+
}
|
|
12892
|
+
async listFeedback() {
|
|
12893
|
+
return this.environment.listFeedback();
|
|
12785
12894
|
}
|
|
12786
12895
|
/**
|
|
12787
|
-
*
|
|
12896
|
+
* Close the session and disconnect from the sandbox.
|
|
12897
|
+
*
|
|
12898
|
+
* Sends `client.goodbye` over WebSocket first, then issues an HTTP fallback
|
|
12899
|
+
* to the runtime goodbye endpoint if no definitive WS-side runtime notify
|
|
12900
|
+
* acknowledgement was observed.
|
|
12788
12901
|
*/
|
|
12789
|
-
async
|
|
12790
|
-
|
|
12902
|
+
async disconnect() {
|
|
12903
|
+
let wsNotifiedRuntime = false;
|
|
12904
|
+
try {
|
|
12905
|
+
const goodbye = await this.rpc(
|
|
12906
|
+
"client.goodbye",
|
|
12907
|
+
{
|
|
12908
|
+
timestamp: Date.now()
|
|
12909
|
+
}
|
|
12910
|
+
);
|
|
12911
|
+
wsNotifiedRuntime = Boolean(goodbye?.ok && goodbye?.via);
|
|
12912
|
+
} catch {
|
|
12913
|
+
wsNotifiedRuntime = false;
|
|
12914
|
+
}
|
|
12915
|
+
if (!wsNotifiedRuntime) {
|
|
12916
|
+
try {
|
|
12917
|
+
await fetch(
|
|
12918
|
+
`${this.environment.runtimeBaseUrl}/orchestrator/runtime/environments/${this.environmentId}/session-goodbye`,
|
|
12919
|
+
{
|
|
12920
|
+
method: "POST",
|
|
12921
|
+
headers: {
|
|
12922
|
+
"Content-Type": "application/json",
|
|
12923
|
+
Authorization: `Bearer ${this.environment.authToken}`,
|
|
12924
|
+
Connection: "close"
|
|
12925
|
+
},
|
|
12926
|
+
body: JSON.stringify({
|
|
12927
|
+
reason: "sdk_disconnect_http_fallback",
|
|
12928
|
+
sessionId: this.client.currentSessionId
|
|
12929
|
+
})
|
|
12930
|
+
}
|
|
12931
|
+
);
|
|
12932
|
+
} catch {
|
|
12933
|
+
}
|
|
12934
|
+
}
|
|
12935
|
+
this.client.disconnect();
|
|
12791
12936
|
}
|
|
12792
12937
|
/**
|
|
12793
|
-
*
|
|
12938
|
+
* Close only the socket transport without sending `client.goodbye`.
|
|
12794
12939
|
*/
|
|
12795
|
-
|
|
12796
|
-
|
|
12940
|
+
disconnectTransport() {
|
|
12941
|
+
this.client.disconnect();
|
|
12797
12942
|
}
|
|
12798
12943
|
/**
|
|
12799
|
-
*
|
|
12944
|
+
* Backwards-compatible alias for `disconnect()`.
|
|
12800
12945
|
*/
|
|
12801
|
-
async
|
|
12802
|
-
|
|
12946
|
+
async close() {
|
|
12947
|
+
await this.disconnect();
|
|
12803
12948
|
}
|
|
12804
12949
|
/**
|
|
12805
|
-
*
|
|
12950
|
+
* Check if the graph container is ready and warm.
|
|
12806
12951
|
*/
|
|
12807
|
-
async
|
|
12808
|
-
|
|
12952
|
+
async checkReadiness() {
|
|
12953
|
+
const result = await this.client.call("client.heartbeat", {});
|
|
12954
|
+
const containerStatus = result?.graphContainerStatus ?? {
|
|
12955
|
+
lastKeepAliveAt: Date.now(),
|
|
12956
|
+
status: "unknown"
|
|
12957
|
+
};
|
|
12958
|
+
this.graphContainerStatus = containerStatus;
|
|
12959
|
+
this.emit("readiness", containerStatus);
|
|
12960
|
+
return containerStatus;
|
|
12961
|
+
}
|
|
12962
|
+
};
|
|
12963
|
+
var OntologyHandle = class {
|
|
12964
|
+
granular;
|
|
12965
|
+
ontologyNameOrId;
|
|
12966
|
+
constructor(granular, ontologyNameOrId) {
|
|
12967
|
+
this.granular = granular;
|
|
12968
|
+
this.ontologyNameOrId = ontologyNameOrId;
|
|
12969
|
+
}
|
|
12970
|
+
get effects() {
|
|
12971
|
+
return {
|
|
12972
|
+
register: async (effect) => this.granular.registerEffect(this.ontologyNameOrId, effect),
|
|
12973
|
+
registerMany: async (effects) => this.granular.registerEffects(this.ontologyNameOrId, effects),
|
|
12974
|
+
unregister: async (name) => this.granular.unregisterEffect(this.ontologyNameOrId, name),
|
|
12975
|
+
clear: async () => this.granular.unregisterAllEffects(this.ontologyNameOrId),
|
|
12976
|
+
disconnect: async () => this.granular.disconnectEffects(this.ontologyNameOrId)
|
|
12977
|
+
};
|
|
12809
12978
|
}
|
|
12810
12979
|
};
|
|
12811
12980
|
var Granular = class _Granular {
|
|
@@ -12842,6 +13011,12 @@ var Granular = class _Granular {
|
|
|
12842
13011
|
this.onReconnectError = options.onReconnectError;
|
|
12843
13012
|
this.httpUrl = this.apiUrl.replace(/^wss:\/\//, "https://").replace(/^ws:\/\//, "http://").replace(/\/ws$/, "");
|
|
12844
13013
|
}
|
|
13014
|
+
/**
|
|
13015
|
+
* Return an ontology-scoped handle for effects and other ontology-level APIs.
|
|
13016
|
+
*/
|
|
13017
|
+
ontology(ontologyNameOrId) {
|
|
13018
|
+
return new OntologyHandle(this, ontologyNameOrId);
|
|
13019
|
+
}
|
|
12845
13020
|
/**
|
|
12846
13021
|
* Records/upserts a user and prepares them for sandbox connections
|
|
12847
13022
|
*
|
|
@@ -12878,7 +13053,23 @@ var Granular = class _Granular {
|
|
|
12878
13053
|
permissions: options.permissions || []
|
|
12879
13054
|
});
|
|
12880
13055
|
}
|
|
13056
|
+
/**
|
|
13057
|
+
* Alias for `recordUser()` with user-facing naming that matches upsert semantics.
|
|
13058
|
+
*/
|
|
13059
|
+
async upsertUser(options) {
|
|
13060
|
+
return this.recordUser(options);
|
|
13061
|
+
}
|
|
12881
13062
|
async resolveConnectUser(options) {
|
|
13063
|
+
const providedIdentityCount = [
|
|
13064
|
+
Boolean(options.user),
|
|
13065
|
+
Boolean(options.userId),
|
|
13066
|
+
Boolean(options.granularId)
|
|
13067
|
+
].filter(Boolean).length;
|
|
13068
|
+
if (providedIdentityCount !== 1) {
|
|
13069
|
+
throw new Error(
|
|
13070
|
+
"openEnvironment() requires exactly one of userId, granularId, or a user object returned by recordUser()."
|
|
13071
|
+
);
|
|
13072
|
+
}
|
|
12882
13073
|
if (options.user) {
|
|
12883
13074
|
const user = normalizeUser(options.user);
|
|
12884
13075
|
return {
|
|
@@ -12915,56 +13106,85 @@ var Granular = class _Granular {
|
|
|
12915
13106
|
};
|
|
12916
13107
|
}
|
|
12917
13108
|
throw new Error(
|
|
12918
|
-
"
|
|
13109
|
+
"openEnvironment() requires either userId, granularId, or a user object returned by recordUser()."
|
|
12919
13110
|
);
|
|
12920
13111
|
}
|
|
12921
13112
|
/**
|
|
12922
|
-
*
|
|
12923
|
-
*
|
|
12924
|
-
* Effects are registered at the sandbox level via `granular.registerEffect()`
|
|
12925
|
-
* or `granular.registerEffects()`. Sessions pick up live availability from
|
|
12926
|
-
* the sandbox registry automatically.
|
|
12927
|
-
*
|
|
12928
|
-
* @param options - Connection options
|
|
12929
|
-
* @returns An active environment session
|
|
13113
|
+
* Open or resolve an ontology environment for one user without opening a session.
|
|
12930
13114
|
*
|
|
12931
13115
|
* @example
|
|
12932
13116
|
* ```typescript
|
|
12933
|
-
* const environment = await granular.
|
|
13117
|
+
* const environment = await granular.openEnvironment({
|
|
12934
13118
|
* ontology: 'my-ontology',
|
|
12935
|
-
*
|
|
13119
|
+
* tag: 'dev',
|
|
12936
13120
|
* userId: 'user_123',
|
|
12937
13121
|
* permissions: ['agent'],
|
|
12938
13122
|
* });
|
|
12939
13123
|
*
|
|
12940
|
-
* await
|
|
12941
|
-
*
|
|
12942
|
-
*
|
|
12943
|
-
*
|
|
12944
|
-
* handler: async () => 'Hello!',
|
|
13124
|
+
* await environment.data.record({
|
|
13125
|
+
* className: 'customer',
|
|
13126
|
+
* id: 'acme',
|
|
13127
|
+
* fields: { name: 'Acme' },
|
|
12945
13128
|
* });
|
|
12946
13129
|
*
|
|
12947
|
-
*
|
|
12948
|
-
* const job = await
|
|
12949
|
-
*
|
|
12950
|
-
* return await tools.greet({});
|
|
12951
|
-
* `);
|
|
12952
|
-
*
|
|
12953
|
-
* console.log(await job.result); // 'Hello!'
|
|
13130
|
+
* const session = await environment.sessions.create();
|
|
13131
|
+
* const job = await session.submitJob(`return "hello";`);
|
|
13132
|
+
* console.log(await job.result);
|
|
12954
13133
|
* ```
|
|
12955
13134
|
*/
|
|
13135
|
+
async openEnvironment(options) {
|
|
13136
|
+
const envData = await this.resolveOpenEnvironmentData(
|
|
13137
|
+
options,
|
|
13138
|
+
"openEnvironment"
|
|
13139
|
+
);
|
|
13140
|
+
return this.bindEnvironmentHandle(envData);
|
|
13141
|
+
}
|
|
13142
|
+
/**
|
|
13143
|
+
* Deprecated compatibility alias for `openEnvironment()`.
|
|
13144
|
+
*
|
|
13145
|
+
* `connect()` no longer opens a runtime session automatically.
|
|
13146
|
+
*/
|
|
12956
13147
|
async connect(options) {
|
|
12957
|
-
|
|
13148
|
+
return this.openEnvironment({
|
|
13149
|
+
...options,
|
|
13150
|
+
tag: this.resolveRequestedTag(options, "connect"),
|
|
13151
|
+
permissions: options.permissions || options.user?.permissions || []
|
|
13152
|
+
});
|
|
13153
|
+
}
|
|
13154
|
+
resolveRequestedTag(options, methodName) {
|
|
13155
|
+
const tag = options.tag?.trim() || options.tagName?.trim() || options.environment?.trim();
|
|
13156
|
+
if (!tag) {
|
|
13157
|
+
throw new Error(`${methodName}() requires \`tag\`.`);
|
|
13158
|
+
}
|
|
13159
|
+
return tag;
|
|
13160
|
+
}
|
|
13161
|
+
buildManagedEnvironmentName(tag, versionId) {
|
|
13162
|
+
return `__sdk__${tag}__${versionId}`;
|
|
13163
|
+
}
|
|
13164
|
+
matchesTagTrackedEnvironment(environment, tagName, tagId) {
|
|
13165
|
+
const environmentTagName = environment.tag?.name || environment.buildPolicy.tagName || null;
|
|
13166
|
+
return environment.tagId === tagId || environmentTagName === tagName || environment.environment === tagName || environment.envName === tagName || environment.environment === this.buildManagedEnvironmentName(tagName, environment.versionId) || environment.envName === this.buildManagedEnvironmentName(tagName, environment.versionId);
|
|
13167
|
+
}
|
|
13168
|
+
sortEnvironmentsByRecency(environments) {
|
|
13169
|
+
return [...environments].sort(
|
|
13170
|
+
(left, right) => right.updatedAt - left.updatedAt
|
|
13171
|
+
);
|
|
13172
|
+
}
|
|
13173
|
+
async resolveOpenEnvironmentData(options, methodName) {
|
|
12958
13174
|
const ontology = options.ontology;
|
|
12959
13175
|
if (!ontology) {
|
|
12960
|
-
throw new Error(
|
|
13176
|
+
throw new Error(`${methodName}() requires \`ontology\`.`);
|
|
12961
13177
|
}
|
|
12962
|
-
const
|
|
12963
|
-
if (!
|
|
12964
|
-
throw new Error(
|
|
13178
|
+
const tagName = options.tag?.trim();
|
|
13179
|
+
if (!tagName) {
|
|
13180
|
+
throw new Error(`${methodName}() requires \`tag\`.`);
|
|
12965
13181
|
}
|
|
12966
|
-
const tagName = options.tagName?.trim() || void 0;
|
|
12967
13182
|
const user = await this.resolveConnectUser(options);
|
|
13183
|
+
if (!Array.isArray(user.permissions) || user.permissions.length === 0) {
|
|
13184
|
+
throw new Error(
|
|
13185
|
+
`${methodName}() requires at least one permission so the SDK can ensure assignments for new users.`
|
|
13186
|
+
);
|
|
13187
|
+
}
|
|
12968
13188
|
const sandbox = await this.findOrCreateSandbox(ontology);
|
|
12969
13189
|
for (const profileName of user.permissions) {
|
|
12970
13190
|
const profileId = await this.ensurePermissionProfile(
|
|
@@ -12977,22 +13197,49 @@ var Granular = class _Granular {
|
|
|
12977
13197
|
profileId
|
|
12978
13198
|
);
|
|
12979
13199
|
}
|
|
12980
|
-
const
|
|
13200
|
+
const tags = await this.request(
|
|
13201
|
+
`/control/sandboxes/${sandbox.sandboxId}/tags`
|
|
13202
|
+
);
|
|
13203
|
+
const tag = (tags.items || []).find(
|
|
13204
|
+
(candidate) => Boolean(candidate?.name === tagName)
|
|
13205
|
+
);
|
|
13206
|
+
if (!tag) {
|
|
13207
|
+
throw new Error(
|
|
13208
|
+
`Tag "${tagName}" was not found for ontology ${sandbox.sandboxId}.`
|
|
13209
|
+
);
|
|
13210
|
+
}
|
|
13211
|
+
const targetVersionId = tag.targetVersionId || tag.targetBuildId;
|
|
13212
|
+
if (!targetVersionId) {
|
|
13213
|
+
throw new Error(
|
|
13214
|
+
`Tag "${tagName}" does not currently point to a build/version.`
|
|
13215
|
+
);
|
|
13216
|
+
}
|
|
13217
|
+
const allEnvironments = await this.environments.list(sandbox.sandboxId);
|
|
13218
|
+
const userEnvironments = allEnvironments.filter(
|
|
13219
|
+
(environment) => environment.subjectId === user.granularId
|
|
13220
|
+
);
|
|
13221
|
+
const currentMatches = this.sortEnvironmentsByRecency(
|
|
13222
|
+
userEnvironments.filter(
|
|
13223
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId) && environment.versionId === targetVersionId
|
|
13224
|
+
)
|
|
13225
|
+
);
|
|
13226
|
+
if (currentMatches.length > 0) {
|
|
13227
|
+
return currentMatches[0];
|
|
13228
|
+
}
|
|
13229
|
+
const outdatedMatches = this.sortEnvironmentsByRecency(
|
|
13230
|
+
userEnvironments.filter(
|
|
13231
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId)
|
|
13232
|
+
)
|
|
13233
|
+
);
|
|
13234
|
+
if (outdatedMatches.length > 0 && options.createFreshIfOutdated !== true) {
|
|
13235
|
+
return outdatedMatches[0];
|
|
13236
|
+
}
|
|
13237
|
+
return this.environments.create(sandbox.sandboxId, {
|
|
12981
13238
|
subjectId: user.granularId,
|
|
12982
|
-
environment:
|
|
12983
|
-
|
|
13239
|
+
environment: this.buildManagedEnvironmentName(tagName, targetVersionId),
|
|
13240
|
+
tagId: tag.tagId,
|
|
12984
13241
|
permissionProfileId: null
|
|
12985
13242
|
});
|
|
12986
|
-
await this.activateEnvironment(envData.environmentId);
|
|
12987
|
-
const session = await this.request("/ws/sessions", {
|
|
12988
|
-
method: "POST",
|
|
12989
|
-
body: JSON.stringify({
|
|
12990
|
-
environmentId: envData.environmentId,
|
|
12991
|
-
clientId,
|
|
12992
|
-
initialHeap: options.initialHeap
|
|
12993
|
-
})
|
|
12994
|
-
});
|
|
12995
|
-
return this.bindWebSocketEnvironment(envData, clientId, session);
|
|
12996
13243
|
}
|
|
12997
13244
|
/**
|
|
12998
13245
|
* List active (open) sessions for an environment — each session is one agent conversation thread.
|
|
@@ -13055,6 +13302,7 @@ var Granular = class _Granular {
|
|
|
13055
13302
|
const clientId = options.clientId || `client_${Date.now()}`;
|
|
13056
13303
|
await this.activateEnvironment(options.environmentId);
|
|
13057
13304
|
const envData = await this.environments.get(options.environmentId);
|
|
13305
|
+
const environment = this.bindEnvironmentHandle(envData);
|
|
13058
13306
|
const session = await this.request("/ws/sessions", {
|
|
13059
13307
|
method: "POST",
|
|
13060
13308
|
body: JSON.stringify({
|
|
@@ -13063,7 +13311,7 @@ var Granular = class _Granular {
|
|
|
13063
13311
|
initialHeap: options.initialHeap
|
|
13064
13312
|
})
|
|
13065
13313
|
});
|
|
13066
|
-
return this.
|
|
13314
|
+
return this.bindWebSocketEnvironmentSession(environment, clientId, session);
|
|
13067
13315
|
}
|
|
13068
13316
|
/**
|
|
13069
13317
|
* Connect to an existing open session (same conversation thread) using a freshly minted WebSocket token.
|
|
@@ -13075,7 +13323,8 @@ var Granular = class _Granular {
|
|
|
13075
13323
|
body: JSON.stringify({})
|
|
13076
13324
|
});
|
|
13077
13325
|
const envData = await this.environments.get(minted.environmentId);
|
|
13078
|
-
|
|
13326
|
+
const environment = this.bindEnvironmentHandle(envData);
|
|
13327
|
+
return this.bindWebSocketEnvironmentSession(environment, clientId, minted);
|
|
13079
13328
|
}
|
|
13080
13329
|
/**
|
|
13081
13330
|
* Mark a session closed in the control plane. If `environment` is the connected handle for that
|
|
@@ -13107,7 +13356,11 @@ var Granular = class _Granular {
|
|
|
13107
13356
|
});
|
|
13108
13357
|
return this.connectSession({ sessionId, clientId: options?.clientId });
|
|
13109
13358
|
}
|
|
13110
|
-
|
|
13359
|
+
bindEnvironmentHandle(envData) {
|
|
13360
|
+
const graphqlEndpoint = `${this.httpUrl}/orchestrator/graphql`;
|
|
13361
|
+
return new Environment(this, envData, this.apiKey, graphqlEndpoint);
|
|
13362
|
+
}
|
|
13363
|
+
async bindWebSocketEnvironmentSession(environment, clientId, session) {
|
|
13111
13364
|
const client = new WSClient({
|
|
13112
13365
|
url: session.wsUrl,
|
|
13113
13366
|
sessionId: session.sessionId,
|
|
@@ -13118,16 +13371,13 @@ var Granular = class _Granular {
|
|
|
13118
13371
|
onReconnectError: this.onReconnectError
|
|
13119
13372
|
});
|
|
13120
13373
|
await client.connect();
|
|
13121
|
-
const
|
|
13122
|
-
const environment = new Environment(
|
|
13374
|
+
const environmentSession = new EnvironmentSession(
|
|
13123
13375
|
client,
|
|
13124
|
-
|
|
13125
|
-
clientId
|
|
13126
|
-
this.apiKey,
|
|
13127
|
-
graphqlEndpoint
|
|
13376
|
+
environment,
|
|
13377
|
+
clientId
|
|
13128
13378
|
);
|
|
13129
|
-
await
|
|
13130
|
-
return
|
|
13379
|
+
await environmentSession.hello();
|
|
13380
|
+
return environmentSession;
|
|
13131
13381
|
}
|
|
13132
13382
|
async activateEnvironment(environmentId) {
|
|
13133
13383
|
await this.request(`/orchestrator/runtime/environments/${environmentId}/activate`, {
|
|
@@ -15066,19 +15316,27 @@ function matchesPattern(text, matcher) {
|
|
|
15066
15316
|
function assertMatches(label, text, includes = [], excludes = []) {
|
|
15067
15317
|
for (const matcher of includes) {
|
|
15068
15318
|
if (!matchesPattern(text, matcher)) {
|
|
15069
|
-
throw new Error(
|
|
15070
|
-
${
|
|
15319
|
+
throw new Error(
|
|
15320
|
+
`${label} did not match ${matcherToString(matcher)}:
|
|
15321
|
+
${text}`
|
|
15322
|
+
);
|
|
15071
15323
|
}
|
|
15072
15324
|
}
|
|
15073
15325
|
for (const matcher of excludes) {
|
|
15074
15326
|
if (matchesPattern(text, matcher)) {
|
|
15075
|
-
throw new Error(
|
|
15076
|
-
${
|
|
15327
|
+
throw new Error(
|
|
15328
|
+
`${label} matched forbidden ${matcherToString(matcher)}:
|
|
15329
|
+
${text}`
|
|
15330
|
+
);
|
|
15077
15331
|
}
|
|
15078
15332
|
}
|
|
15079
15333
|
}
|
|
15080
15334
|
function buildArtifactDir(baseDir, suiteName = "granular-agent-evals") {
|
|
15081
|
-
return path.join(
|
|
15335
|
+
return path.join(
|
|
15336
|
+
baseDir || path.join(process.cwd(), "test-artifacts"),
|
|
15337
|
+
suiteName,
|
|
15338
|
+
timestampId()
|
|
15339
|
+
);
|
|
15082
15340
|
}
|
|
15083
15341
|
function createTimestampedArtifactDirectory(options) {
|
|
15084
15342
|
return buildArtifactDir(options?.baseDir, options?.suiteName);
|
|
@@ -15092,17 +15350,16 @@ function buildScenarioSteps(scenario) {
|
|
|
15092
15350
|
return scenario.steps;
|
|
15093
15351
|
}
|
|
15094
15352
|
if (!scenario.request) {
|
|
15095
|
-
throw new Error(
|
|
15353
|
+
throw new Error(
|
|
15354
|
+
`Scenario ${scenario.id} must provide either request or steps`
|
|
15355
|
+
);
|
|
15096
15356
|
}
|
|
15097
15357
|
const compatibilityStep = {
|
|
15098
15358
|
id: scenario.id,
|
|
15099
15359
|
request: scenario.request,
|
|
15100
15360
|
human: scenario.human,
|
|
15101
15361
|
expect: scenario.expect,
|
|
15102
|
-
inspect: [
|
|
15103
|
-
...asArray2(scenario.inspect),
|
|
15104
|
-
...asArray2(scenario.verify)
|
|
15105
|
-
],
|
|
15362
|
+
inspect: [...asArray2(scenario.inspect), ...asArray2(scenario.verify)],
|
|
15106
15363
|
check: scenario.check,
|
|
15107
15364
|
maxIterations: scenario.maxIterations,
|
|
15108
15365
|
setup: {
|
|
@@ -15120,22 +15377,27 @@ function buildAssistantHistoryContent(entry) {
|
|
|
15120
15377
|
${entry.content}`);
|
|
15121
15378
|
if (entry.jobStatus) parts.push(`[Job status]
|
|
15122
15379
|
${entry.jobStatus}`);
|
|
15123
|
-
if (entry.jobResultPreview)
|
|
15380
|
+
if (entry.jobResultPreview)
|
|
15381
|
+
parts.push(`[Job result]
|
|
15124
15382
|
${entry.jobResultPreview}`);
|
|
15125
15383
|
if (entry.error) parts.push(`[Job error]
|
|
15126
15384
|
${entry.error}`);
|
|
15127
15385
|
return parts.join("\n\n") || entry.content;
|
|
15128
15386
|
}
|
|
15129
15387
|
function buildHistory(entries) {
|
|
15130
|
-
return entries.reduce(
|
|
15131
|
-
|
|
15132
|
-
if (entry.
|
|
15388
|
+
return entries.reduce(
|
|
15389
|
+
(history, entry) => {
|
|
15390
|
+
if (entry.role === "user") {
|
|
15391
|
+
if (entry.content.trim())
|
|
15392
|
+
history.push({ role: "user", content: entry.content });
|
|
15393
|
+
return history;
|
|
15394
|
+
}
|
|
15395
|
+
const content = buildAssistantHistoryContent(entry).trim();
|
|
15396
|
+
if (content) history.push({ role: "assistant", content });
|
|
15133
15397
|
return history;
|
|
15134
|
-
}
|
|
15135
|
-
|
|
15136
|
-
|
|
15137
|
-
return history;
|
|
15138
|
-
}, []);
|
|
15398
|
+
},
|
|
15399
|
+
[]
|
|
15400
|
+
);
|
|
15139
15401
|
}
|
|
15140
15402
|
function getOpenPromptsFromDoc(liveDoc) {
|
|
15141
15403
|
const jobsById = asRecord4(asRecord4(liveDoc?.jobs)?.byId) || {};
|
|
@@ -15144,7 +15406,8 @@ function getOpenPromptsFromDoc(liveDoc) {
|
|
|
15144
15406
|
const promptRecords = asRecord4(asRecord4(job)?.prompts) || {};
|
|
15145
15407
|
for (const raw of Object.values(promptRecords)) {
|
|
15146
15408
|
const record = asRecord4(raw);
|
|
15147
|
-
if (!record || record.status !== "open" || typeof record.promptId !== "string")
|
|
15409
|
+
if (!record || record.status !== "open" || typeof record.promptId !== "string")
|
|
15410
|
+
continue;
|
|
15148
15411
|
const prompt = normalizePrompt({
|
|
15149
15412
|
promptId: record.promptId,
|
|
15150
15413
|
kind: record.kind,
|
|
@@ -15188,7 +15451,9 @@ ${prompt.message || ""}`;
|
|
|
15188
15451
|
return resolvePromptAnswer(prompt, rawAnswer);
|
|
15189
15452
|
}
|
|
15190
15453
|
if (fallback) return fallback({ prompt, history });
|
|
15191
|
-
throw new Error(
|
|
15454
|
+
throw new Error(
|
|
15455
|
+
`No scripted prompt responder matched prompt ${prompt.id}: ${promptText}`
|
|
15456
|
+
);
|
|
15192
15457
|
};
|
|
15193
15458
|
}
|
|
15194
15459
|
function extractJsonObject(text) {
|
|
@@ -15212,13 +15477,19 @@ function modelOutputInstruction() {
|
|
|
15212
15477
|
].join("\n");
|
|
15213
15478
|
}
|
|
15214
15479
|
function createOpenAIChatTurnGenerator(options) {
|
|
15215
|
-
const baseUrl = (options.baseUrl || "https://api.openai.com/v1").replace(
|
|
15480
|
+
const baseUrl = (options.baseUrl || "https://api.openai.com/v1").replace(
|
|
15481
|
+
/\/$/,
|
|
15482
|
+
""
|
|
15483
|
+
);
|
|
15216
15484
|
const model = options.model || "gpt-5-mini";
|
|
15217
15485
|
return async (input) => {
|
|
15218
15486
|
const messages = [
|
|
15219
|
-
{
|
|
15487
|
+
{
|
|
15488
|
+
role: "system",
|
|
15489
|
+
content: `${input.systemPrompt}
|
|
15220
15490
|
|
|
15221
|
-
${modelOutputInstruction()}`
|
|
15491
|
+
${modelOutputInstruction()}`
|
|
15492
|
+
},
|
|
15222
15493
|
...input.history,
|
|
15223
15494
|
{ role: "user", content: input.request }
|
|
15224
15495
|
];
|
|
@@ -15247,10 +15518,14 @@ ${modelOutputInstruction()}` },
|
|
|
15247
15518
|
await sleep2(500 * attempt);
|
|
15248
15519
|
continue;
|
|
15249
15520
|
}
|
|
15250
|
-
throw new Error(
|
|
15521
|
+
throw new Error(
|
|
15522
|
+
`OpenAI chat generation failed: ${response.status} ${errorText}`
|
|
15523
|
+
);
|
|
15251
15524
|
}
|
|
15252
15525
|
const raw = await response.json();
|
|
15253
|
-
const content = asRecord4(
|
|
15526
|
+
const content = asRecord4(
|
|
15527
|
+
asRecord4(raw.choices?.[0])?.message
|
|
15528
|
+
)?.content;
|
|
15254
15529
|
const text = typeof content === "string" ? content : Array.isArray(content) ? content.map((part) => asRecord4(part)?.text || "").join("") : "";
|
|
15255
15530
|
const parsed = extractJsonObject(text);
|
|
15256
15531
|
if (!parsed) {
|
|
@@ -15268,7 +15543,9 @@ ${text}`);
|
|
|
15268
15543
|
};
|
|
15269
15544
|
} catch (error) {
|
|
15270
15545
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
15271
|
-
if (attempt < 3 && /socket connection was closed unexpectedly|ECONNRESET|network|timed out/i.test(
|
|
15546
|
+
if (attempt < 3 && /socket connection was closed unexpectedly|ECONNRESET|network|timed out/i.test(
|
|
15547
|
+
lastError.message
|
|
15548
|
+
)) {
|
|
15272
15549
|
await sleep2(500 * attempt);
|
|
15273
15550
|
continue;
|
|
15274
15551
|
}
|
|
@@ -15290,7 +15567,10 @@ async function withTimeout(promise, ms, label) {
|
|
|
15290
15567
|
return await Promise.race([
|
|
15291
15568
|
promise,
|
|
15292
15569
|
new Promise((_, reject) => {
|
|
15293
|
-
timeoutId = setTimeout(
|
|
15570
|
+
timeoutId = setTimeout(
|
|
15571
|
+
() => reject(new Error(`${label} timed out after ${ms}ms`)),
|
|
15572
|
+
ms
|
|
15573
|
+
);
|
|
15294
15574
|
})
|
|
15295
15575
|
]);
|
|
15296
15576
|
} finally {
|
|
@@ -15300,7 +15580,9 @@ async function withTimeout(promise, ms, label) {
|
|
|
15300
15580
|
function getActionSummary(liveDoc, jobId) {
|
|
15301
15581
|
const jobsById = asRecord4(asRecord4(liveDoc?.jobs)?.byId) || {};
|
|
15302
15582
|
const job = asRecord4(jobsById[jobId]);
|
|
15303
|
-
return Array.isArray(job?.actionSummary) ? job.actionSummary.filter(
|
|
15583
|
+
return Array.isArray(job?.actionSummary) ? job.actionSummary.filter(
|
|
15584
|
+
(line) => typeof line === "string"
|
|
15585
|
+
) : [];
|
|
15304
15586
|
}
|
|
15305
15587
|
function normalizeHeapSnapshot2(heap) {
|
|
15306
15588
|
return {
|
|
@@ -15330,12 +15612,20 @@ async function waitForJobOutcome(input) {
|
|
|
15330
15612
|
const startedAt = Date.now();
|
|
15331
15613
|
while (Date.now() - startedAt < input.timeoutMs) {
|
|
15332
15614
|
const liveDoc = cloneJson(input.environment.document);
|
|
15333
|
-
const prompts = filterPromptsByBoundary(
|
|
15615
|
+
const prompts = filterPromptsByBoundary(
|
|
15616
|
+
liveDoc,
|
|
15617
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
15618
|
+
input.boundaryTimestamp
|
|
15619
|
+
);
|
|
15334
15620
|
if (prompts.length > 0) {
|
|
15335
15621
|
return { kind: "prompt", prompts, liveDoc, stdout, stderr };
|
|
15336
15622
|
}
|
|
15337
15623
|
try {
|
|
15338
|
-
const result = await withTimeout(
|
|
15624
|
+
const result = await withTimeout(
|
|
15625
|
+
input.job.result,
|
|
15626
|
+
input.pollIntervalMs,
|
|
15627
|
+
`job ${input.job.id} tick`
|
|
15628
|
+
);
|
|
15339
15629
|
return { kind: "completed", result, liveDoc, stdout, stderr };
|
|
15340
15630
|
} catch (error) {
|
|
15341
15631
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -15401,7 +15691,9 @@ function buildResultReport(result) {
|
|
|
15401
15691
|
...result.actionSummary.length ? result.actionSummary.map((line) => `- ${line}`) : ["- None"],
|
|
15402
15692
|
"",
|
|
15403
15693
|
"## Prompt Interactions",
|
|
15404
|
-
...result.promptInteractions.length ? result.promptInteractions.map(
|
|
15694
|
+
...result.promptInteractions.length ? result.promptInteractions.map(
|
|
15695
|
+
(interaction) => `- [${interaction.type}] ${interaction.message || interaction.title} -> ${JSON.stringify(interaction.answer)}`
|
|
15696
|
+
) : ["- None"],
|
|
15405
15697
|
"",
|
|
15406
15698
|
...stepSection,
|
|
15407
15699
|
"## Raw Files",
|
|
@@ -15416,7 +15708,9 @@ function buildSuiteIndex(results) {
|
|
|
15416
15708
|
const lines = [
|
|
15417
15709
|
"# Agent Eval Report Index",
|
|
15418
15710
|
"",
|
|
15419
|
-
...results.map(
|
|
15711
|
+
...results.map(
|
|
15712
|
+
(result) => `- [${result.scenario.id}](./${result.scenario.id}/REPORT.md) - ${result.status}`
|
|
15713
|
+
)
|
|
15420
15714
|
];
|
|
15421
15715
|
return `${lines.join("\n")}
|
|
15422
15716
|
`;
|
|
@@ -15430,7 +15724,7 @@ async function applySetup(setup, context) {
|
|
|
15430
15724
|
await context.environment.recordObjects(setup.records);
|
|
15431
15725
|
}
|
|
15432
15726
|
if (setup.effects?.length) {
|
|
15433
|
-
await context.environment.
|
|
15727
|
+
await context.granular.ontology(context.environment.sandboxId).effects.registerMany(setup.effects);
|
|
15434
15728
|
}
|
|
15435
15729
|
if (setup.run) {
|
|
15436
15730
|
await setup.run(context);
|
|
@@ -15452,6 +15746,7 @@ async function runAgentEvalSuite(options) {
|
|
|
15452
15746
|
);
|
|
15453
15747
|
try {
|
|
15454
15748
|
await applySetup(scenario.setup, {
|
|
15749
|
+
granular: options.harness.granular,
|
|
15455
15750
|
conversation,
|
|
15456
15751
|
environment: conversation.environment,
|
|
15457
15752
|
turnDir: conversation.artifactDir
|
|
@@ -15464,14 +15759,19 @@ async function runAgentEvalSuite(options) {
|
|
|
15464
15759
|
conversation,
|
|
15465
15760
|
request: step.request,
|
|
15466
15761
|
prepare: async (ctx) => {
|
|
15467
|
-
await applySetup(step.setup,
|
|
15762
|
+
await applySetup(step.setup, {
|
|
15763
|
+
granular: options.harness.granular,
|
|
15764
|
+
...ctx
|
|
15765
|
+
});
|
|
15468
15766
|
},
|
|
15469
15767
|
human: step.human,
|
|
15470
15768
|
maxIterations: step.maxIterations,
|
|
15471
15769
|
autoAnswerPrompts: step.autoAnswerPrompts
|
|
15472
15770
|
});
|
|
15473
15771
|
if ("prompts" in completed) {
|
|
15474
|
-
throw new Error(
|
|
15772
|
+
throw new Error(
|
|
15773
|
+
`Scenario ${scenario.id} step ${index + 1} paused for human input instead of completing automatically`
|
|
15774
|
+
);
|
|
15475
15775
|
}
|
|
15476
15776
|
const inspectionResults = [];
|
|
15477
15777
|
const stepChecks = asArray2(step.check);
|
|
@@ -15487,12 +15787,23 @@ async function runAgentEvalSuite(options) {
|
|
|
15487
15787
|
actionSummary: completed.actionSummary,
|
|
15488
15788
|
promptInteractions: completed.promptInteractions,
|
|
15489
15789
|
result: completed.result,
|
|
15490
|
-
heap: normalizeHeapSnapshot2(
|
|
15491
|
-
|
|
15790
|
+
heap: normalizeHeapSnapshot2(
|
|
15791
|
+
asRecord4(
|
|
15792
|
+
cloneJson(conversation.environment.document)?.heap
|
|
15793
|
+
)
|
|
15794
|
+
),
|
|
15795
|
+
openPrompts: getOpenPromptsFromDoc(
|
|
15796
|
+
cloneJson(conversation.environment.document)
|
|
15797
|
+
),
|
|
15492
15798
|
liveDoc: cloneJson(conversation.environment.document),
|
|
15493
15799
|
inspect: async (code) => {
|
|
15494
|
-
const
|
|
15495
|
-
|
|
15800
|
+
const session = conversation.environment;
|
|
15801
|
+
const job = await session.submitJob(code);
|
|
15802
|
+
return withTimeout(
|
|
15803
|
+
job.result,
|
|
15804
|
+
9e4,
|
|
15805
|
+
`inspection job ${job.id}`
|
|
15806
|
+
);
|
|
15496
15807
|
},
|
|
15497
15808
|
assertMatches
|
|
15498
15809
|
};
|
|
@@ -15550,7 +15861,9 @@ async function runAgentEvalSuite(options) {
|
|
|
15550
15861
|
}
|
|
15551
15862
|
const lastStep = stepResults[stepResults.length - 1];
|
|
15552
15863
|
if (!lastStep) {
|
|
15553
|
-
throw new Error(
|
|
15864
|
+
throw new Error(
|
|
15865
|
+
`Scenario ${scenario.id} produced no completed steps`
|
|
15866
|
+
);
|
|
15554
15867
|
}
|
|
15555
15868
|
const result = {
|
|
15556
15869
|
scenario,
|
|
@@ -15564,9 +15877,18 @@ async function runAgentEvalSuite(options) {
|
|
|
15564
15877
|
steps: stepResults,
|
|
15565
15878
|
turnDir: conversation.artifactDir
|
|
15566
15879
|
};
|
|
15567
|
-
await writeJson(
|
|
15568
|
-
|
|
15569
|
-
|
|
15880
|
+
await writeJson(
|
|
15881
|
+
path.join(conversation.artifactDir, "result.json"),
|
|
15882
|
+
result
|
|
15883
|
+
);
|
|
15884
|
+
await writeJson(
|
|
15885
|
+
path.join(conversation.artifactDir, "report.json"),
|
|
15886
|
+
result
|
|
15887
|
+
);
|
|
15888
|
+
await writeFile(
|
|
15889
|
+
path.join(conversation.artifactDir, "REPORT.md"),
|
|
15890
|
+
buildResultReport(result)
|
|
15891
|
+
);
|
|
15570
15892
|
finalResult = result;
|
|
15571
15893
|
} catch (error) {
|
|
15572
15894
|
const failureMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -15588,7 +15910,10 @@ async function runAgentEvalSuite(options) {
|
|
|
15588
15910
|
await ensureDir(failed.turnDir);
|
|
15589
15911
|
await writeJson(path.join(failed.turnDir, "result.json"), failed);
|
|
15590
15912
|
await writeJson(path.join(failed.turnDir, "report.json"), failed);
|
|
15591
|
-
await writeFile(
|
|
15913
|
+
await writeFile(
|
|
15914
|
+
path.join(failed.turnDir, "REPORT.md"),
|
|
15915
|
+
buildResultReport(failed)
|
|
15916
|
+
);
|
|
15592
15917
|
finalResult = failed;
|
|
15593
15918
|
} finally {
|
|
15594
15919
|
await options.harness.closeConversation(conversation);
|
|
@@ -15609,12 +15934,21 @@ async function runAgentEvalSuite(options) {
|
|
|
15609
15934
|
}
|
|
15610
15935
|
results.push(finalResult);
|
|
15611
15936
|
}
|
|
15612
|
-
await writeJson(
|
|
15613
|
-
|
|
15937
|
+
await writeJson(
|
|
15938
|
+
path.join(options.harness.artifactDir, "summary.json"),
|
|
15939
|
+
results
|
|
15940
|
+
);
|
|
15941
|
+
await writeFile(
|
|
15942
|
+
path.join(options.harness.artifactDir, "REPORT_INDEX.md"),
|
|
15943
|
+
buildSuiteIndex(results)
|
|
15944
|
+
);
|
|
15614
15945
|
return { artifactDir: options.harness.artifactDir, results };
|
|
15615
15946
|
}
|
|
15616
15947
|
function createAgentEvalHarness(options) {
|
|
15617
|
-
const artifactDir = buildArtifactDir(
|
|
15948
|
+
const artifactDir = buildArtifactDir(
|
|
15949
|
+
options.artifactBaseDir,
|
|
15950
|
+
options.suiteName
|
|
15951
|
+
);
|
|
15618
15952
|
const controllerBudgets = {
|
|
15619
15953
|
...DEFAULT_CONTROLLER_BUDGETS,
|
|
15620
15954
|
...options.controllerBudgets || {}
|
|
@@ -15626,7 +15960,9 @@ function createAgentEvalHarness(options) {
|
|
|
15626
15960
|
await ensureDir(artifactDir);
|
|
15627
15961
|
const clientId = `${slugify(label)}-${Date.now()}`;
|
|
15628
15962
|
if (!options.openEnvironment && !options.environmentId) {
|
|
15629
|
-
throw new Error(
|
|
15963
|
+
throw new Error(
|
|
15964
|
+
"createAgentEvalHarness requires either environmentId or openEnvironment"
|
|
15965
|
+
);
|
|
15630
15966
|
}
|
|
15631
15967
|
const environment = options.openEnvironment ? await options.openEnvironment({ label, clientId }) : await options.granular.createSession({
|
|
15632
15968
|
environmentId: options.environmentId,
|
|
@@ -15650,12 +15986,15 @@ function createAgentEvalHarness(options) {
|
|
|
15650
15986
|
}
|
|
15651
15987
|
async function closeConversation(conversation) {
|
|
15652
15988
|
try {
|
|
15653
|
-
await options.granular.closeSession(
|
|
15989
|
+
await options.granular.closeSession(
|
|
15990
|
+
conversation.environment.sessionId,
|
|
15991
|
+
conversation.environment
|
|
15992
|
+
);
|
|
15654
15993
|
} catch {
|
|
15655
15994
|
}
|
|
15656
15995
|
}
|
|
15657
|
-
async function runCheckJob(code,
|
|
15658
|
-
const job = await
|
|
15996
|
+
async function runCheckJob(code, session) {
|
|
15997
|
+
const job = await session.submitJob(code);
|
|
15659
15998
|
return withTimeout(job.result, jobTimeoutMs, `check job ${job.id}`);
|
|
15660
15999
|
}
|
|
15661
16000
|
function buildCheckContext(conversation, completed, turnDir) {
|
|
@@ -15700,8 +16039,12 @@ function createAgentEvalHarness(options) {
|
|
|
15700
16039
|
async function resumePendingTurn(pending, responder) {
|
|
15701
16040
|
const prompt = pending.prompts[0];
|
|
15702
16041
|
if (!prompt) throw new Error("Pending turn has no prompts to answer");
|
|
15703
|
-
const answer = await responder({
|
|
15704
|
-
|
|
16042
|
+
const answer = await responder({
|
|
16043
|
+
prompt,
|
|
16044
|
+
history: pending.promptInteractions
|
|
16045
|
+
});
|
|
16046
|
+
const session = pending.conversation.environment;
|
|
16047
|
+
await session.answerPrompt(prompt.id, answer);
|
|
15705
16048
|
pending.promptInteractions.push({
|
|
15706
16049
|
promptId: prompt.id,
|
|
15707
16050
|
type: prompt.type,
|
|
@@ -15725,7 +16068,9 @@ function createAgentEvalHarness(options) {
|
|
|
15725
16068
|
};
|
|
15726
16069
|
}
|
|
15727
16070
|
await sleep2(350);
|
|
15728
|
-
const liveDoc = cloneJson(
|
|
16071
|
+
const liveDoc = cloneJson(
|
|
16072
|
+
pending.conversation.environment.document
|
|
16073
|
+
);
|
|
15729
16074
|
const presentation = resolveJobPresentation({
|
|
15730
16075
|
jobId: pending.job.id,
|
|
15731
16076
|
result: resumed.result,
|
|
@@ -15770,25 +16115,40 @@ function createAgentEvalHarness(options) {
|
|
|
15770
16115
|
await conversation.environment.recordObjects(input.prepareRecords);
|
|
15771
16116
|
}
|
|
15772
16117
|
if (input.prepareTools?.length) {
|
|
15773
|
-
await conversation.environment.
|
|
16118
|
+
await options.granular.ontology(conversation.environment.sandboxId).effects.registerMany(input.prepareTools);
|
|
15774
16119
|
}
|
|
15775
16120
|
if (input.prepare) {
|
|
15776
|
-
await input.prepare({
|
|
16121
|
+
await input.prepare({
|
|
16122
|
+
conversation,
|
|
16123
|
+
environment: conversation.environment,
|
|
16124
|
+
turnDir
|
|
16125
|
+
});
|
|
15777
16126
|
}
|
|
15778
16127
|
const boundaryTimestamp = Date.now();
|
|
15779
16128
|
conversation.history.push({ role: "user", content: input.request });
|
|
15780
|
-
await writeJson(path.join(turnDir, "request.json"), {
|
|
16129
|
+
await writeJson(path.join(turnDir, "request.json"), {
|
|
16130
|
+
request: input.request,
|
|
16131
|
+
boundaryTimestamp
|
|
16132
|
+
});
|
|
15781
16133
|
let iteration = 0;
|
|
15782
16134
|
let noProgressCount = 0;
|
|
15783
16135
|
let previousSnapshot = null;
|
|
15784
16136
|
let latestCheckpoint = null;
|
|
15785
16137
|
const maxIterations = input.maxIterations || controllerBudgets.maxIterations;
|
|
15786
16138
|
const autoAnswerPrompts = input.autoAnswerPrompts ?? true;
|
|
15787
|
-
const baselineClosureId = getCurrentClosureId(
|
|
16139
|
+
const baselineClosureId = getCurrentClosureId(
|
|
16140
|
+
cloneJson(conversation.environment.document)
|
|
16141
|
+
);
|
|
15788
16142
|
while (iteration < maxIterations) {
|
|
15789
16143
|
const liveDoc = cloneJson(conversation.environment.document);
|
|
15790
|
-
const pendingPrompts = filterPromptsByBoundary(
|
|
15791
|
-
|
|
16144
|
+
const pendingPrompts = filterPromptsByBoundary(
|
|
16145
|
+
liveDoc,
|
|
16146
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
16147
|
+
boundaryTimestamp
|
|
16148
|
+
);
|
|
16149
|
+
const workflowFocus = projectWorkflowFocus(liveDoc, pendingPrompts, {
|
|
16150
|
+
boundaryTimestamp
|
|
16151
|
+
});
|
|
15792
16152
|
const systemPrompt = buildGranularAgentSystemPrompt({
|
|
15793
16153
|
domainDocumentation: await conversation.environment.getDomainDocumentation(),
|
|
15794
16154
|
sessionContext: {
|
|
@@ -15799,8 +16159,12 @@ function createAgentEvalHarness(options) {
|
|
|
15799
16159
|
heapSummary: projectHeapSummary(liveDoc, {
|
|
15800
16160
|
focus: workflowFocus
|
|
15801
16161
|
}),
|
|
15802
|
-
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
15803
|
-
|
|
16162
|
+
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
16163
|
+
boundaryTimestamp
|
|
16164
|
+
}),
|
|
16165
|
+
workflowSummary: projectWorkflowSummary(liveDoc, pendingPrompts, {
|
|
16166
|
+
boundaryTimestamp
|
|
16167
|
+
}),
|
|
15804
16168
|
tools: conversation.environment.getEffects().map((tool) => ({
|
|
15805
16169
|
name: tool.name,
|
|
15806
16170
|
description: tool.description,
|
|
@@ -15810,14 +16174,23 @@ function createAgentEvalHarness(options) {
|
|
|
15810
16174
|
})),
|
|
15811
16175
|
checkpoint: latestCheckpoint
|
|
15812
16176
|
});
|
|
15813
|
-
const request = iteration === 0 ? input.request : buildContinuationInstruction(
|
|
15814
|
-
|
|
15815
|
-
|
|
15816
|
-
|
|
15817
|
-
|
|
15818
|
-
|
|
15819
|
-
|
|
15820
|
-
|
|
16177
|
+
const request = iteration === 0 ? input.request : buildContinuationInstruction(
|
|
16178
|
+
buildContinuationPreview(latestCheckpoint, noProgressCount)
|
|
16179
|
+
);
|
|
16180
|
+
const generation = await withTimeout(
|
|
16181
|
+
generateTurnWithRepair(options.generator, {
|
|
16182
|
+
systemPrompt,
|
|
16183
|
+
history: buildHistory(conversation.history),
|
|
16184
|
+
request,
|
|
16185
|
+
attempt: 1
|
|
16186
|
+
}),
|
|
16187
|
+
chatTimeoutMs,
|
|
16188
|
+
`chat generation for ${conversation.label} iteration ${iteration + 1}`
|
|
16189
|
+
);
|
|
16190
|
+
await writeJson(
|
|
16191
|
+
path.join(turnDir, `iteration-${iteration + 1}-generation.json`),
|
|
16192
|
+
generation
|
|
16193
|
+
);
|
|
15821
16194
|
if (!generation.code) {
|
|
15822
16195
|
const responseText2 = generation.reply?.trim() || "Done.";
|
|
15823
16196
|
conversation.history.push({ role: "assistant", content: responseText2 });
|
|
@@ -15833,12 +16206,18 @@ function createAgentEvalHarness(options) {
|
|
|
15833
16206
|
result: generation.reply?.trim() || responseText2
|
|
15834
16207
|
};
|
|
15835
16208
|
if (input.verification) {
|
|
15836
|
-
completed.verification = await runInspection(
|
|
16209
|
+
completed.verification = await runInspection(
|
|
16210
|
+
conversation,
|
|
16211
|
+
input.verification,
|
|
16212
|
+
completed,
|
|
16213
|
+
turnDir
|
|
16214
|
+
);
|
|
15837
16215
|
}
|
|
15838
16216
|
await writeJson(path.join(turnDir, "result.json"), completed);
|
|
15839
16217
|
return completed;
|
|
15840
16218
|
}
|
|
15841
|
-
const
|
|
16219
|
+
const session = conversation.environment;
|
|
16220
|
+
const job = await session.submitJob(generation.code);
|
|
15842
16221
|
const outcome = await waitForJobOutcome({
|
|
15843
16222
|
environment: conversation.environment,
|
|
15844
16223
|
job,
|
|
@@ -15863,7 +16242,9 @@ function createAgentEvalHarness(options) {
|
|
|
15863
16242
|
};
|
|
15864
16243
|
}
|
|
15865
16244
|
if (!input.human) {
|
|
15866
|
-
throw new Error(
|
|
16245
|
+
throw new Error(
|
|
16246
|
+
"This turn reached a human prompt but no responder was provided"
|
|
16247
|
+
);
|
|
15867
16248
|
}
|
|
15868
16249
|
let pending = {
|
|
15869
16250
|
conversation,
|
|
@@ -15885,16 +16266,25 @@ function createAgentEvalHarness(options) {
|
|
|
15885
16266
|
continue;
|
|
15886
16267
|
}
|
|
15887
16268
|
if (input.verification) {
|
|
15888
|
-
resumed.verification = await runInspection(
|
|
16269
|
+
resumed.verification = await runInspection(
|
|
16270
|
+
conversation,
|
|
16271
|
+
input.verification,
|
|
16272
|
+
resumed,
|
|
16273
|
+
turnDir
|
|
16274
|
+
);
|
|
15889
16275
|
}
|
|
15890
16276
|
return resumed;
|
|
15891
16277
|
}
|
|
15892
16278
|
}
|
|
15893
16279
|
if (outcome.kind !== "completed") {
|
|
15894
|
-
throw new Error(
|
|
16280
|
+
throw new Error(
|
|
16281
|
+
"Unexpected non-completed outcome after prompt handling"
|
|
16282
|
+
);
|
|
15895
16283
|
}
|
|
15896
16284
|
await sleep2(350);
|
|
15897
|
-
const settledLiveDoc = cloneJson(
|
|
16285
|
+
const settledLiveDoc = cloneJson(
|
|
16286
|
+
conversation.environment.document
|
|
16287
|
+
);
|
|
15898
16288
|
const sessionHeap = normalizeHeapSnapshot2(asRecord4(settledLiveDoc?.heap));
|
|
15899
16289
|
const presentation = resolveJobPresentation({
|
|
15900
16290
|
jobId: job.id,
|
|
@@ -15915,7 +16305,11 @@ function createAgentEvalHarness(options) {
|
|
|
15915
16305
|
baselineClosureId,
|
|
15916
16306
|
currentClosureId: getCurrentClosureId(settledLiveDoc),
|
|
15917
16307
|
liveDoc: settledLiveDoc,
|
|
15918
|
-
pendingPrompts: filterPromptsByBoundary(
|
|
16308
|
+
pendingPrompts: filterPromptsByBoundary(
|
|
16309
|
+
settledLiveDoc,
|
|
16310
|
+
getOpenPromptsFromDoc(settledLiveDoc),
|
|
16311
|
+
boundaryTimestamp
|
|
16312
|
+
),
|
|
15919
16313
|
projectionOptions: { boundaryTimestamp },
|
|
15920
16314
|
latestResponseText: responseText,
|
|
15921
16315
|
previousSnapshot,
|
|
@@ -15940,12 +16334,15 @@ function createAgentEvalHarness(options) {
|
|
|
15940
16334
|
jobStatus: "succeeded",
|
|
15941
16335
|
jobResultPreview: JSON.stringify(outcome.result, null, 2)
|
|
15942
16336
|
});
|
|
15943
|
-
await writeJson(
|
|
15944
|
-
|
|
15945
|
-
|
|
15946
|
-
|
|
15947
|
-
|
|
15948
|
-
|
|
16337
|
+
await writeJson(
|
|
16338
|
+
path.join(turnDir, `iteration-${iteration + 1}-result.json`),
|
|
16339
|
+
{
|
|
16340
|
+
responseText,
|
|
16341
|
+
continuation,
|
|
16342
|
+
actionSummary: latestCheckpoint.latestActionSummary,
|
|
16343
|
+
result: outcome.result
|
|
16344
|
+
}
|
|
16345
|
+
);
|
|
15949
16346
|
if (!continuation.shouldContinue) {
|
|
15950
16347
|
const completed = {
|
|
15951
16348
|
conversation,
|
|
@@ -15960,17 +16357,25 @@ function createAgentEvalHarness(options) {
|
|
|
15960
16357
|
result: outcome.result
|
|
15961
16358
|
};
|
|
15962
16359
|
if (input.verification) {
|
|
15963
|
-
completed.verification = await runInspection(
|
|
16360
|
+
completed.verification = await runInspection(
|
|
16361
|
+
conversation,
|
|
16362
|
+
input.verification,
|
|
16363
|
+
completed,
|
|
16364
|
+
turnDir
|
|
16365
|
+
);
|
|
15964
16366
|
}
|
|
15965
16367
|
await writeJson(path.join(turnDir, "result.json"), completed);
|
|
15966
16368
|
return completed;
|
|
15967
16369
|
}
|
|
15968
16370
|
iteration += 1;
|
|
15969
16371
|
}
|
|
15970
|
-
throw new Error(
|
|
16372
|
+
throw new Error(
|
|
16373
|
+
`Turn exceeded iteration budget (${maxIterations}) for ${conversation.label}`
|
|
16374
|
+
);
|
|
15971
16375
|
}
|
|
15972
16376
|
return {
|
|
15973
16377
|
artifactDir,
|
|
16378
|
+
granular: options.granular,
|
|
15974
16379
|
openConversation,
|
|
15975
16380
|
closeConversation,
|
|
15976
16381
|
runTurn,
|
|
@@ -16006,12 +16411,11 @@ function createAgentTester(options) {
|
|
|
16006
16411
|
}
|
|
16007
16412
|
if ("connect" in options.target && !connectSeeded) {
|
|
16008
16413
|
connectSeeded = true;
|
|
16009
|
-
const environment = await granular.
|
|
16010
|
-
...options.target.connect
|
|
16011
|
-
clientId
|
|
16414
|
+
const environment = await granular.openEnvironment({
|
|
16415
|
+
...options.target.connect
|
|
16012
16416
|
});
|
|
16013
16417
|
resolvedEnvironmentId = environment.environmentId;
|
|
16014
|
-
return environment;
|
|
16418
|
+
return environment.sessions.create({ clientId });
|
|
16015
16419
|
}
|
|
16016
16420
|
if ("createEnvironment" in options.target && !resolvedEnvironmentId) {
|
|
16017
16421
|
const envData = await granular.environments.create(
|