@inflexa-ai/harness 0.36.2 → 0.36.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/execution/step-summary-metrics.d.ts +1 -1
- package/dist/execution/step-summary-metrics.js +1 -1
- package/dist/lib/zod-issue-shape.d.ts +21 -0
- package/dist/lib/zod-issue-shape.js +122 -0
- package/dist/loop/genai-spans.d.ts +5 -0
- package/dist/loop/genai-spans.js +10 -0
- package/dist/loop/metrics.d.ts +1 -1
- package/dist/loop/metrics.js +1 -1
- package/dist/loop/run-agent.js +6 -3
- package/dist/tools/lib/alphafold-client.d.ts +5 -3
- package/dist/tools/lib/alphafold-client.js +12 -7
- package/dist/tools/lib/api-utils.d.ts +3 -0
- package/dist/tools/lib/api-utils.js +9 -1
- package/package.json +1 -1
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shape of a Zod validation failure, with no value of the input in it.
|
|
3
|
+
*
|
|
4
|
+
* The span and the log of a rejected tool call may not carry the arguments of
|
|
5
|
+
* the model. A Zod `message` can quote the input, and so can the `keys` of an
|
|
6
|
+
* `unrecognized_keys` issue and the string segments of a path under a record.
|
|
7
|
+
* Thus each entry is built only from parts that the schema defines:
|
|
8
|
+
*
|
|
9
|
+
* - the path, where a string segment is kept only when the schema declares
|
|
10
|
+
* that key, and each other string segment becomes `*`. An array index is
|
|
11
|
+
* kept as a number.
|
|
12
|
+
* - the issue `code`.
|
|
13
|
+
* - schema-side detail: the `expected` type, the `origin` and bound of a size
|
|
14
|
+
* check, the string `format`, the divisor, the allowed literals, and the
|
|
15
|
+
* discriminator of a union. For `unrecognized_keys`, only the count of keys.
|
|
16
|
+
*
|
|
17
|
+
* Example: `runId: invalid_type (expected string)`.
|
|
18
|
+
*/
|
|
19
|
+
import type { z } from "zod";
|
|
20
|
+
/** Describe the issues of `error` against `schema`, one entry for each issue. */
|
|
21
|
+
export declare function describeZodIssueShapes(error: z.ZodError, schema: z.ZodType): string[];
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shape of a Zod validation failure, with no value of the input in it.
|
|
3
|
+
*
|
|
4
|
+
* The span and the log of a rejected tool call may not carry the arguments of
|
|
5
|
+
* the model. A Zod `message` can quote the input, and so can the `keys` of an
|
|
6
|
+
* `unrecognized_keys` issue and the string segments of a path under a record.
|
|
7
|
+
* Thus each entry is built only from parts that the schema defines:
|
|
8
|
+
*
|
|
9
|
+
* - the path, where a string segment is kept only when the schema declares
|
|
10
|
+
* that key, and each other string segment becomes `*`. An array index is
|
|
11
|
+
* kept as a number.
|
|
12
|
+
* - the issue `code`.
|
|
13
|
+
* - schema-side detail: the `expected` type, the `origin` and bound of a size
|
|
14
|
+
* check, the string `format`, the divisor, the allowed literals, and the
|
|
15
|
+
* discriminator of a union. For `unrecognized_keys`, only the count of keys.
|
|
16
|
+
*
|
|
17
|
+
* Example: `runId: invalid_type (expected string)`.
|
|
18
|
+
*/
|
|
19
|
+
/** The most issues that one failure describes. */
|
|
20
|
+
const MAX_ISSUES = 10;
|
|
21
|
+
/** The cap of one entry, in characters. */
|
|
22
|
+
const MAX_ENTRY_CHARS = 200;
|
|
23
|
+
/** Describe the issues of `error` against `schema`, one entry for each issue. */
|
|
24
|
+
export function describeZodIssueShapes(error, schema) {
|
|
25
|
+
const entries = error.issues.slice(0, MAX_ISSUES).map((issue) => {
|
|
26
|
+
const detail = issueDetail(issue);
|
|
27
|
+
const entry = `${schemaPath(schema, issue.path)}: ${issue.code}${detail === undefined ? "" : ` (${detail})`}`;
|
|
28
|
+
return entry.length > MAX_ENTRY_CHARS ? `${entry.slice(0, MAX_ENTRY_CHARS - 1)}…` : entry;
|
|
29
|
+
});
|
|
30
|
+
const rest = error.issues.length - MAX_ISSUES;
|
|
31
|
+
if (rest > 0)
|
|
32
|
+
entries.push(`+${rest} more issues`);
|
|
33
|
+
return entries;
|
|
34
|
+
}
|
|
35
|
+
function issueDetail(issue) {
|
|
36
|
+
switch (issue.code) {
|
|
37
|
+
case "invalid_type":
|
|
38
|
+
return `expected ${issue.expected}`;
|
|
39
|
+
case "too_small":
|
|
40
|
+
return `${issue.origin} ${issue.exact === true ? "exactly" : "min"} ${issue.minimum}`;
|
|
41
|
+
case "too_big":
|
|
42
|
+
return `${issue.origin} ${issue.exact === true ? "exactly" : "max"} ${issue.maximum}`;
|
|
43
|
+
case "invalid_format":
|
|
44
|
+
return `format ${issue.format}`;
|
|
45
|
+
case "not_multiple_of":
|
|
46
|
+
return `divisor ${issue.divisor}`;
|
|
47
|
+
case "unrecognized_keys":
|
|
48
|
+
return `${issue.keys.length} keys`;
|
|
49
|
+
case "invalid_value":
|
|
50
|
+
return `expected ${issue.values.map(String).join("|")}`;
|
|
51
|
+
case "invalid_union":
|
|
52
|
+
return issue.discriminator === undefined ? undefined : `discriminator ${issue.discriminator}`;
|
|
53
|
+
default:
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** The path of an issue, with each string segment that the schema does not declare replaced by `*`. */
|
|
58
|
+
function schemaPath(schema, path) {
|
|
59
|
+
if (path.length === 0)
|
|
60
|
+
return "(root)";
|
|
61
|
+
let current = schema;
|
|
62
|
+
const segments = path.map((segment) => {
|
|
63
|
+
if (typeof segment === "number") {
|
|
64
|
+
current = current === undefined ? undefined : elementOf(current, segment);
|
|
65
|
+
return String(segment);
|
|
66
|
+
}
|
|
67
|
+
const field = current === undefined || typeof segment !== "string" ? undefined : fieldOf(current, segment);
|
|
68
|
+
current = field;
|
|
69
|
+
return field === undefined ? "*" : String(segment);
|
|
70
|
+
});
|
|
71
|
+
return segments.join(".");
|
|
72
|
+
}
|
|
73
|
+
/** The schema of the declared field `key`, looking through wrappers and unions. */
|
|
74
|
+
function fieldOf(schema, key) {
|
|
75
|
+
const def = unwrap(schema)._zod.def;
|
|
76
|
+
if (def.type === "object")
|
|
77
|
+
return Object.hasOwn(def.shape, key) ? def.shape[key] : undefined;
|
|
78
|
+
if (def.type === "union") {
|
|
79
|
+
for (const option of def.options) {
|
|
80
|
+
const field = fieldOf(option, key);
|
|
81
|
+
if (field !== undefined)
|
|
82
|
+
return field;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (def.type === "intersection")
|
|
86
|
+
return fieldOf(def.left, key) ?? fieldOf(def.right, key);
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
function elementOf(schema, index) {
|
|
90
|
+
const def = unwrap(schema)._zod.def;
|
|
91
|
+
if (def.type === "array")
|
|
92
|
+
return def.element;
|
|
93
|
+
if (def.type === "tuple")
|
|
94
|
+
return def.items[index] ?? def.rest ?? undefined;
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
function unwrap(schema) {
|
|
98
|
+
let current = schema;
|
|
99
|
+
for (let depth = 0; depth < 16; depth++) {
|
|
100
|
+
const def = current._zod.def;
|
|
101
|
+
switch (def.type) {
|
|
102
|
+
case "optional":
|
|
103
|
+
case "nullable":
|
|
104
|
+
case "default":
|
|
105
|
+
case "prefault":
|
|
106
|
+
case "nonoptional":
|
|
107
|
+
case "catch":
|
|
108
|
+
case "readonly":
|
|
109
|
+
current = def.innerType;
|
|
110
|
+
break;
|
|
111
|
+
case "pipe":
|
|
112
|
+
current = def.in;
|
|
113
|
+
break;
|
|
114
|
+
case "lazy":
|
|
115
|
+
current = def.getter();
|
|
116
|
+
break;
|
|
117
|
+
default:
|
|
118
|
+
return current;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return current;
|
|
122
|
+
}
|
|
@@ -31,4 +31,9 @@ export declare function recordToolException(err: Error): void;
|
|
|
31
31
|
* span. `message` is text that our tool wrote, never text from the model.
|
|
32
32
|
*/
|
|
33
33
|
export declare function labelToolFailure(type: string, message?: string): void;
|
|
34
|
+
/**
|
|
35
|
+
* Label a tool call whose input failed validation on the active `execute_tool`
|
|
36
|
+
* span. `issues` is the shape of the failure, with no value of the input.
|
|
37
|
+
*/
|
|
38
|
+
export declare function labelToolValidationFailure(issues: readonly string[]): void;
|
|
34
39
|
export {};
|
package/dist/loop/genai-spans.js
CHANGED
|
@@ -55,6 +55,16 @@ export function labelToolFailure(type, message) {
|
|
|
55
55
|
if (message !== undefined)
|
|
56
56
|
span.setAttribute("inflexa.tool.error", message.slice(0, TOOL_ERROR_MAX_CHARS));
|
|
57
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Label a tool call whose input failed validation on the active `execute_tool`
|
|
60
|
+
* span. `issues` is the shape of the failure, with no value of the input.
|
|
61
|
+
*/
|
|
62
|
+
export function labelToolValidationFailure(issues) {
|
|
63
|
+
const span = trace.getActiveSpan();
|
|
64
|
+
if (span === undefined)
|
|
65
|
+
return;
|
|
66
|
+
span.setAttributes({ "error.type": "validation", "inflexa.tool.validation.issues": [...issues] });
|
|
67
|
+
}
|
|
58
68
|
function errorTypeOf(err) {
|
|
59
69
|
const code = "code" in err ? err.code : undefined;
|
|
60
70
|
if ((typeof code === "string" && code !== "") || typeof code === "number")
|
package/dist/loop/metrics.d.ts
CHANGED
package/dist/loop/metrics.js
CHANGED
package/dist/loop/run-agent.js
CHANGED
|
@@ -5,6 +5,7 @@ import { stripNulCharacters } from "../input-sanitization.js";
|
|
|
5
5
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
6
6
|
import { ATTR_INFLEXA_TOOL_USE_ID, stableSpan } from "../lib/otel-spans.js";
|
|
7
7
|
import { ResultError } from "../lib/result.js";
|
|
8
|
+
import { describeZodIssueShapes } from "../lib/zod-issue-shape.js";
|
|
8
9
|
import { hintForZodIssue, repairToolInput } from "../lib/zod-issues.js";
|
|
9
10
|
import { markInterruptedMessage, syntheticUserMessage } from "../memory/ai-sdk-message-storage.js";
|
|
10
11
|
import { stripUnansweredToolCalls } from "../memory/tool-call-integrity.js";
|
|
@@ -14,7 +15,7 @@ import { DEFAULT_REASONING } from "../providers/reasoning.js";
|
|
|
14
15
|
import { resultStep } from "./run-step.js";
|
|
15
16
|
import { AskRejectedError, UnavailableAsk } from "../tools/approval/contract.js";
|
|
16
17
|
import { isToolError, readToolResultImages } from "../tools/define-tool.js";
|
|
17
|
-
import { labelToolFailure, recordToolException, traceAgentRun, traceToolCall } from "./genai-spans.js";
|
|
18
|
+
import { labelToolFailure, labelToolValidationFailure, recordToolException, traceAgentRun, traceToolCall } from "./genai-spans.js";
|
|
18
19
|
import { addChatUsage, hasReportedUsage, recordAgentRun } from "./metrics.js";
|
|
19
20
|
import { computeDetail, computeResultDetail } from "./tool-detail.js";
|
|
20
21
|
import { toolOutcomeForOutputType } from "./tool-outcome.js";
|
|
@@ -617,8 +618,10 @@ async function dispatchTool(tu, toolsById, ctx, isFatalLoopError, encoding) {
|
|
|
617
618
|
const repaired = repairedInput === undefined ? undefined : tool.inputSchema.safeParse(repairedInput);
|
|
618
619
|
if (repaired?.success === true)
|
|
619
620
|
return execute(tu, tool, repaired.data, ctx, isFatalLoopError, encoding);
|
|
620
|
-
// The Zod text can quote the input, thus the span
|
|
621
|
-
|
|
621
|
+
// The Zod text can quote the input, thus the span and the log get only the shape of the failure.
|
|
622
|
+
const issues = describeZodIssueShapes(parsed.error, tool.inputSchema);
|
|
623
|
+
labelToolValidationFailure(issues);
|
|
624
|
+
encoding.log.warn("tool call rejected", { tool: tool.id, toolCallId: tu.toolCallId, issues });
|
|
622
625
|
return { result: errorResult(tu, `input validation failed: ${formatZodIssues(parsed.error, tu.input)}`) };
|
|
623
626
|
}
|
|
624
627
|
async function execute(tu, tool, input, ctx, isFatalLoopError, encoding) {
|
|
@@ -14,8 +14,9 @@
|
|
|
14
14
|
* identifier that does not parse as a UniProt accession or an AlphaFold DB id
|
|
15
15
|
* gives 400 with an `Invalid identifier format` body. A well-formed accession
|
|
16
16
|
* that AlphaFold holds no model for gives 404 with an empty object. Both are
|
|
17
|
-
* the same outcome for a caller,
|
|
18
|
-
*
|
|
17
|
+
* the same outcome for a caller, thus one branch answers both. Only these two
|
|
18
|
+
* codes mean absence. A different 4xx, such as a 403 refusal, throws, because
|
|
19
|
+
* `found: false` tells the agent to report the absence and not to retry.
|
|
19
20
|
*/
|
|
20
21
|
import { z } from "zod";
|
|
21
22
|
/**
|
|
@@ -46,7 +47,8 @@ export type AlphaFoldPrediction = z.infer<typeof AlphaFoldPredictionSchema>;
|
|
|
46
47
|
* Fetch the AlphaFold structure prediction for one UniProt accession.
|
|
47
48
|
*
|
|
48
49
|
* Gives `null` when AlphaFold holds no model for the accession, and `null` for
|
|
49
|
-
* an identifier it cannot parse.
|
|
50
|
+
* an identifier it cannot parse. Throws on each other failure. When the array
|
|
51
|
+
* holds more than one entry —
|
|
50
52
|
* the isoforms of one canonical accession — the entry whose own
|
|
51
53
|
* `uniprotAccession` matches the queried accession wins. A query that names no
|
|
52
54
|
* exact isoform falls back to the first entry, which AlphaFold orders
|
|
@@ -14,11 +14,12 @@
|
|
|
14
14
|
* identifier that does not parse as a UniProt accession or an AlphaFold DB id
|
|
15
15
|
* gives 400 with an `Invalid identifier format` body. A well-formed accession
|
|
16
16
|
* that AlphaFold holds no model for gives 404 with an empty object. Both are
|
|
17
|
-
* the same outcome for a caller,
|
|
18
|
-
*
|
|
17
|
+
* the same outcome for a caller, thus one branch answers both. Only these two
|
|
18
|
+
* codes mean absence. A different 4xx, such as a 403 refusal, throws, because
|
|
19
|
+
* `found: false` tells the agent to report the absence and not to retry.
|
|
19
20
|
*/
|
|
20
21
|
import { z } from "zod";
|
|
21
|
-
import { apiFetchValidated, describeApiError
|
|
22
|
+
import { apiFetchValidated, describeApiError } from "./api-utils.js";
|
|
22
23
|
const ALPHAFOLD_BASE = "https://alphafold.ebi.ac.uk/api/prediction";
|
|
23
24
|
/**
|
|
24
25
|
* One entry of the AlphaFold DB prediction array. A query by a canonical
|
|
@@ -50,7 +51,8 @@ export const AlphaFoldPredictionSchema = z.object({
|
|
|
50
51
|
* Fetch the AlphaFold structure prediction for one UniProt accession.
|
|
51
52
|
*
|
|
52
53
|
* Gives `null` when AlphaFold holds no model for the accession, and `null` for
|
|
53
|
-
* an identifier it cannot parse.
|
|
54
|
+
* an identifier it cannot parse. Throws on each other failure. When the array
|
|
55
|
+
* holds more than one entry —
|
|
54
56
|
* the isoforms of one canonical accession — the entry whose own
|
|
55
57
|
* `uniprotAccession` matches the queried accession wins. A query that names no
|
|
56
58
|
* exact isoform falls back to the first entry, which AlphaFold orders
|
|
@@ -60,9 +62,9 @@ export async function fetchAlphaFoldPrediction(uniprotAccession) {
|
|
|
60
62
|
const accession = uniprotAccession.trim();
|
|
61
63
|
const res = await apiFetchValidated(`${ALPHAFOLD_BASE}/${encodeURIComponent(accession)}`, z.array(AlphaFoldPredictionSchema));
|
|
62
64
|
if (res.isErr()) {
|
|
63
|
-
if (
|
|
64
|
-
|
|
65
|
-
|
|
65
|
+
if (isAbsence(res.error))
|
|
66
|
+
return null;
|
|
67
|
+
throw new Error(describeApiError(res.error));
|
|
66
68
|
}
|
|
67
69
|
const entries = res.value;
|
|
68
70
|
if (entries.length === 0)
|
|
@@ -70,3 +72,6 @@ export async function fetchAlphaFoldPrediction(uniprotAccession) {
|
|
|
70
72
|
const exact = entries.find((entry) => entry.uniprotAccession.toUpperCase() === accession.toUpperCase());
|
|
71
73
|
return exact ?? entries[0];
|
|
72
74
|
}
|
|
75
|
+
function isAbsence(e) {
|
|
76
|
+
return e.type === "http_status" && (e.status === 400 || e.status === 404);
|
|
77
|
+
}
|
|
@@ -48,6 +48,7 @@ export type ApiError = {
|
|
|
48
48
|
readonly type: "invalid_response";
|
|
49
49
|
readonly issues: string;
|
|
50
50
|
};
|
|
51
|
+
export declare const HARNESS_USER_AGENT = "inflexa-harness (+https://inflexa.ai)";
|
|
51
52
|
/**
|
|
52
53
|
* Fetch a URL with retry on 429/502/503/504, exponential backoff, and timeout.
|
|
53
54
|
* `err` carries the structured failure; a non-ok HTTP status is reported as
|
|
@@ -61,6 +62,8 @@ export type ApiError = {
|
|
|
61
62
|
*
|
|
62
63
|
* Between two attempts the wait obeys `Retry-After` when the response carries it,
|
|
63
64
|
* and every wait stays at or below `maxRetryDelayMs`.
|
|
65
|
+
*
|
|
66
|
+
* Each request sends `HARNESS_USER_AGENT`, unless `headers` sets a User-Agent.
|
|
64
67
|
*/
|
|
65
68
|
export declare function apiFetch<T = unknown>(url: string, options?: ApiFetchOptions): ResultAsync<T, ApiError>;
|
|
66
69
|
/**
|
|
@@ -15,6 +15,9 @@ import { sleep } from "../../lib/async-utils.js";
|
|
|
15
15
|
// provider answers with one of them while the origin restarts, and it sends an
|
|
16
16
|
// HTML body. That answer is transient, thus a retry gets the real payload.
|
|
17
17
|
const RETRYABLE_STATUSES = new Set([429, 502, 503, 504]);
|
|
18
|
+
// AlphaFold DB answers 403 to the default User-Agent of the runtime (`node`,
|
|
19
|
+
// `Bun/<version>`, or none), and a descriptive value passes.
|
|
20
|
+
export const HARNESS_USER_AGENT = "inflexa-harness (+https://inflexa.ai)";
|
|
18
21
|
/**
|
|
19
22
|
* Fetch a URL with retry on 429/502/503/504, exponential backoff, and timeout.
|
|
20
23
|
* `err` carries the structured failure; a non-ok HTTP status is reported as
|
|
@@ -28,6 +31,8 @@ const RETRYABLE_STATUSES = new Set([429, 502, 503, 504]);
|
|
|
28
31
|
*
|
|
29
32
|
* Between two attempts the wait obeys `Retry-After` when the response carries it,
|
|
30
33
|
* and every wait stays at or below `maxRetryDelayMs`.
|
|
34
|
+
*
|
|
35
|
+
* Each request sends `HARNESS_USER_AGENT`, unless `headers` sets a User-Agent.
|
|
31
36
|
*/
|
|
32
37
|
export function apiFetch(url, options = {}) {
|
|
33
38
|
return new ResultAsync(runFetch(url, options));
|
|
@@ -92,12 +97,15 @@ function retryAfterMilliseconds(res) {
|
|
|
92
97
|
}
|
|
93
98
|
async function runFetch(url, options) {
|
|
94
99
|
const { method = "GET", headers = {}, body, maxRetries = 3, retryDelayMs = 1000, maxRetryDelayMs = 30_000, sleep: wait = sleep, timeoutMs = 90_000, parseAs = "json", } = options;
|
|
100
|
+
const requestHeaders = new Headers(headers);
|
|
101
|
+
if (!requestHeaders.has("user-agent"))
|
|
102
|
+
requestHeaders.set("User-Agent", HARNESS_USER_AGENT);
|
|
95
103
|
let lastError = "";
|
|
96
104
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
97
105
|
try {
|
|
98
106
|
const res = await fetch(url, {
|
|
99
107
|
method,
|
|
100
|
-
headers,
|
|
108
|
+
headers: requestHeaders,
|
|
101
109
|
body,
|
|
102
110
|
signal: AbortSignal.timeout(timeoutMs),
|
|
103
111
|
});
|
package/package.json
CHANGED