@alexkroman1/aai-cli 6.11.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/{_agent-DpH2pBJd.mjs → _agent-BTwZJu4P.mjs} +2 -2
- package/dist/{_bundler-C2BDwU7r.mjs → _bundler-CDuYl5Gb.mjs} +2 -2
- package/dist/_dev-env.d.ts +22 -2
- package/dist/{_dev-server-LLLWnFBs.mjs → _dev-server-Me06wgeF.mjs} +38 -47
- package/dist/_dev-server.d.ts +0 -17
- package/dist/{_init-C57bWAQ8.mjs → _init-BhEfQ6Yi.mjs} +3 -2
- package/dist/{_preflight-BtfaYtbE.mjs → _preflight-Co1G9Jww.mjs} +1 -1
- package/dist/_preflight.d.ts +1 -1
- package/dist/_resource-commands.d.ts +23 -0
- package/dist/{_slug-api-BxrsJXVI.mjs → _slug-api-z-USxPtL.mjs} +16 -12
- package/dist/_slug-api.d.ts +5 -2
- package/dist/{_studio-U3jhKlA3.mjs → _studio-DR5yzZ1T.mjs} +2 -1
- package/dist/{_templates-Dxb_P2Wz.mjs → _templates-4WcKOjS5.mjs} +5 -6
- package/dist/_templates.d.ts +3 -4
- package/dist/_workflow-scan.d.ts +108 -0
- package/dist/{build-Dfza2pRI.mjs → build-BBxf8vom.mjs} +6 -3
- package/dist/cli.mjs +346 -349
- package/dist/{client-bundler-BEIqgOtd.mjs → client-bundler-CWnG42cU.mjs} +2 -1
- package/dist/client-bundler.mjs +1 -1
- package/dist/{delete-vbpjGzqs.mjs → delete-Dl_LSfvI.mjs} +3 -3
- package/dist/{deploy-B8lubiRT.mjs → deploy-DRC8jWr8.mjs} +5 -5
- package/dist/{dev-Uq5ujP8a.mjs → dev-BvcctCn2.mjs} +1 -1
- package/dist/eval-DfQ8cm5p.mjs +56 -0
- package/dist/eval.d.ts +43 -0
- package/dist/{init-l6xfU4xX.mjs → init-CUo87gcI.mjs} +3 -3
- package/dist/{login-C71-qz8F.mjs → login-BBuM1sxH.mjs} +3 -4
- package/dist/{logs-BWVGSjLR.mjs → logs-CQ2T46dr.mjs} +2 -2
- package/dist/project-config.mjs +1 -1
- package/dist/scaffold/CLAUDE.md +442 -80
- package/dist/scaffold/package.json +6 -3
- package/dist/scaffold/server.mjs +11 -4
- package/dist/scaffold/tsconfig.json +0 -1
- package/dist/{secret-BuMuFR4B.mjs → secret-Ct34kdgU.mjs} +2 -2
- package/dist/{storage-DzRZ-eCw.mjs → storage-i_79ZwAR.mjs} +2 -2
- package/dist/{studio-BCNUpDgP.mjs → studio-CzTzPqe1.mjs} +4 -4
- package/dist/templates/briefing-desk/agent.test.ts +350 -0
- package/dist/templates/briefing-desk/agent.ts +30 -0
- package/dist/templates/briefing-desk/shared.ts +200 -0
- package/dist/templates/briefing-desk/system-prompt.md +39 -0
- package/dist/templates/briefing-desk/tools/briefing_so_far.ts +35 -0
- package/dist/templates/briefing-desk/tools/research_topic.ts +88 -0
- package/dist/templates/briefing-desk/tools/verify_claim.ts +77 -0
- package/dist/templates/call-audit/agent.eval.test.ts +474 -0
- package/dist/templates/call-audit/agent.test.ts +57 -204
- package/dist/templates/call-audit/agent.ts +32 -19
- package/dist/templates/call-audit/client.tsx +17 -54
- package/dist/templates/call-audit/workflows/audit.ts +11 -18
- package/dist/templates/call-audit/workflows/ingest.ts +106 -114
- package/dist/templates/call-audit/workflows/media.ts +2 -12
- package/dist/templates/call-audit/workflows/summarize.ts +47 -52
- package/dist/templates/call-audit/workflows/sync-api.ts +7 -7
- package/dist/templates/code-interpreter/agent.eval.test.ts +165 -0
- package/dist/templates/dispatch-center/agent.eval.test.ts +349 -0
- package/dist/templates/dispatch-center/agent.test.ts +149 -25
- package/dist/templates/dispatch-center/client.tsx +239 -129
- package/dist/templates/dispatch-center/shared.ts +99 -1
- package/dist/templates/dispatch-center/system-prompt.md +3 -1
- package/dist/templates/dispatch-center/tools/incident_add_note.ts +16 -13
- package/dist/templates/dispatch-center/tools/incident_create.ts +19 -1
- package/dist/templates/dispatch-center/tools/incident_escalate.ts +68 -59
- package/dist/templates/dispatch-center/tools/incident_triage.ts +57 -43
- package/dist/templates/dispatch-center/tools/incident_update_status.ts +49 -40
- package/dist/templates/dispatch-center/tools/ops_run_scenario.ts +8 -2
- package/dist/templates/dispatch-center/tools/resources_dispatch.ts +69 -52
- package/dist/templates/dispatch-center/tools/resources_update_status.ts +45 -41
- package/dist/templates/embedded-assets/agent.eval.test.ts +138 -0
- package/dist/templates/embedded-assets/agent.test.ts +17 -14
- package/dist/templates/health-assistant/agent.eval.test.ts +167 -0
- package/dist/templates/health-assistant/agent.test.ts +22 -4
- package/dist/templates/infocom-adventure/agent.eval.test.ts +158 -0
- package/dist/templates/infocom-adventure/agent.test.ts +53 -24
- package/dist/templates/infocom-adventure/agent.ts +21 -0
- package/dist/templates/infocom-adventure/client.tsx +246 -164
- package/dist/templates/infocom-adventure/shared.ts +20 -9
- package/dist/templates/infocom-adventure/system-prompt.md +1 -1
- package/dist/templates/infocom-adventure/tools/game_state_move.ts +3 -2
- package/dist/templates/link-digest/agent.eval.test.ts +195 -0
- package/dist/templates/link-digest/agent.test.ts +24 -19
- package/dist/templates/link-digest/client.tsx +47 -61
- package/dist/templates/link-digest/workflows/digest.ts +19 -31
- package/dist/templates/math-buddy/agent.eval.test.ts +146 -0
- package/dist/templates/night-owl/agent.eval.test.ts +272 -0
- package/dist/templates/night-owl/agent.test.ts +70 -19
- package/dist/templates/night-owl/agent.ts +5 -0
- package/dist/templates/night-owl/client.tsx +56 -56
- package/dist/templates/night-owl/shared.ts +24 -0
- package/dist/templates/night-owl/tools/recommend.ts +18 -6
- package/dist/templates/personal-finance/agent.eval.test.ts +168 -0
- package/dist/templates/personal-finance/system-prompt.md +4 -0
- package/dist/templates/pipeline-simple/agent.eval.test.ts +78 -0
- package/dist/templates/pipeline-simple/agent.ts +2 -2
- package/dist/templates/pizza-ordering/agent.eval.test.ts +171 -0
- package/dist/templates/pizza-ordering/agent.test.ts +42 -27
- package/dist/templates/pizza-ordering/client.tsx +9 -26
- package/dist/templates/pizza-ordering/shared.ts +20 -1
- package/dist/templates/pizza-ordering/system-prompt.md +6 -0
- package/dist/templates/plan-and-execute/agent.eval.test.ts +209 -0
- package/dist/templates/plan-and-execute/agent.test.ts +99 -58
- package/dist/templates/plan-and-execute/agent.ts +1 -1
- package/dist/templates/plan-and-execute/client.tsx +12 -15
- package/dist/templates/plan-and-execute/shared.ts +71 -2
- package/dist/templates/plan-and-execute/tools/plan_status.ts +18 -6
- package/dist/templates/plan-and-execute/tools/revise_plan.ts +18 -10
- package/dist/templates/plan-and-execute/tools/start_plan.ts +13 -2
- package/dist/templates/plan-and-execute/tools/work_next_step.ts +70 -22
- package/dist/templates/podcast-digest/agent.eval.test.ts +411 -0
- package/dist/templates/podcast-digest/agent.test.ts +717 -0
- package/dist/templates/podcast-digest/agent.ts +139 -0
- package/dist/templates/podcast-digest/client.tsx +154 -0
- package/dist/templates/podcast-digest/workflows/digest.ts +444 -0
- package/dist/templates/podcast-digest/workflows/feeds.ts +507 -0
- package/dist/templates/podcast-digest/workflows/slack.ts +88 -0
- package/dist/templates/recap-workflow/agent.eval.test.ts +366 -0
- package/dist/templates/recap-workflow/agent.test.ts +97 -94
- package/dist/templates/recap-workflow/tools/recap_progress.ts +9 -12
- package/dist/templates/recap-workflow/tools/recap_status.ts +8 -9
- package/dist/templates/recap-workflow/tools/request_recap.ts +2 -1
- package/dist/templates/recap-workflow/workflows/recap.ts +36 -40
- package/dist/templates/redline/agent.eval.test.ts +209 -0
- package/dist/templates/redline/agent.test.ts +17 -15
- package/dist/templates/redline/client.tsx +12 -12
- package/dist/templates/redline/workflows/redline.ts +19 -31
- package/dist/templates/research-workflow/agent.eval.test.ts +329 -0
- package/dist/templates/research-workflow/agent.test.ts +60 -59
- package/dist/templates/research-workflow/tools/research_progress.ts +9 -12
- package/dist/templates/research-workflow/tools/research_status.ts +9 -11
- package/dist/templates/research-workflow/workflows/research.ts +44 -61
- package/dist/templates/retail/address.ts +87 -1
- package/dist/templates/retail/agent.eval.test.ts +314 -0
- package/dist/templates/retail/agent.test.ts +195 -112
- package/dist/templates/retail/agent.ts +35 -1
- package/dist/templates/retail/cancel.ts +116 -0
- package/dist/templates/retail/client.tsx +265 -117
- package/dist/templates/retail/payment.ts +107 -0
- package/dist/templates/retail/pending.test.ts +196 -0
- package/dist/templates/retail/pending.ts +137 -0
- package/dist/templates/retail/registry.test.ts +150 -7
- package/dist/templates/retail/returns.ts +102 -0
- package/dist/templates/retail/shared.test.ts +42 -0
- package/dist/templates/retail/shared.ts +35 -0
- package/dist/templates/retail/store.test.ts +84 -15
- package/dist/templates/retail/store.ts +231 -48
- package/dist/templates/retail/swap.test.ts +4 -4
- package/dist/templates/retail/swap.ts +248 -15
- package/dist/templates/retail/system-prompt.md +69 -28
- package/dist/templates/retail/tools/cancel_change.ts +33 -0
- package/dist/templates/retail/tools/cancel_pending_order.ts +16 -60
- package/dist/templates/retail/tools/confirm_change.ts +42 -0
- package/dist/templates/retail/tools/exchange_delivered_order_items.ts +22 -67
- package/dist/templates/retail/tools/find_user_id_by_email.ts +8 -5
- package/dist/templates/retail/tools/find_user_id_by_name_zip.ts +4 -5
- package/dist/templates/retail/tools/get_item_details.ts +3 -4
- package/dist/templates/retail/tools/get_order_details.ts +3 -4
- package/dist/templates/retail/tools/get_product_details.ts +3 -4
- package/dist/templates/retail/tools/get_user_details.ts +2 -2
- package/dist/templates/retail/tools/list_all_product_types.ts +5 -5
- package/dist/templates/retail/tools/modify_pending_order_address.ts +13 -25
- package/dist/templates/retail/tools/modify_pending_order_items.ts +23 -62
- package/dist/templates/retail/tools/modify_pending_order_payment.ts +16 -78
- package/dist/templates/retail/tools/modify_user_address.ts +13 -16
- package/dist/templates/retail/tools/return_delivered_order_items.ts +17 -72
- package/dist/templates/retail/tools/transfer_to_human_agents.ts +12 -6
- package/dist/templates/simple/agent.eval.test.ts +54 -0
- package/dist/templates/solo-rpg/agent.eval.test.ts +320 -0
- package/dist/templates/solo-rpg/agent.test.ts +227 -48
- package/dist/templates/solo-rpg/client.tsx +38 -37
- package/dist/templates/solo-rpg/shared.ts +145 -19
- package/dist/templates/solo-rpg/system-prompt.md +3 -2
- package/dist/templates/solo-rpg/tools/action_roll.ts +82 -63
- package/dist/templates/solo-rpg/tools/burn_momentum.ts +85 -50
- package/dist/templates/solo-rpg/tools/check_state.ts +24 -7
- package/dist/templates/solo-rpg/tools/load_game.ts +13 -1
- package/dist/templates/solo-rpg/tools/save_game.ts +16 -7
- package/dist/templates/solo-rpg/tools/setup_character.ts +22 -2
- package/dist/templates/solo-rpg/tools/update_state.ts +113 -100
- package/dist/templates/spoken-summary/agent.eval.test.ts +342 -0
- package/dist/templates/spoken-summary/agent.test.ts +68 -97
- package/dist/templates/spoken-summary/agent.ts +30 -17
- package/dist/templates/spoken-summary/client.tsx +10 -50
- package/dist/templates/spoken-summary/workflows/summarize.ts +17 -22
- package/dist/templates/spoken-summary/workflows/transcribe.ts +19 -26
- package/dist/templates/support-line/agent.eval.test.ts +273 -0
- package/dist/templates/support-line/agent.test.ts +11 -16
- package/dist/templates/support-line/agent.ts +1 -1
- package/dist/templates/support-line/client.tsx +9 -9
- package/dist/templates/support-line/nodes.ts +100 -0
- package/dist/templates/support-line/procedure.ts +407 -0
- package/dist/templates/support-line/system-prompt.md +2 -1
- package/dist/templates/support-line/tools/answer_question.ts +66 -4
- package/dist/templates/transcription-workflow/agent.eval.test.ts +234 -0
- package/dist/templates/transcription-workflow/agent.test.ts +94 -194
- package/dist/templates/transcription-workflow/agent.ts +1 -1
- package/dist/templates/transcription-workflow/client.tsx +17 -42
- package/dist/templates/transcription-workflow/workflows/batch.ts +19 -22
- package/dist/templates/transcription-workflow/workflows/normalize.ts +90 -186
- package/dist/templates/transcription-workflow/workflows/stitch.ts +0 -11
- package/dist/templates/transcription-workflow/workflows/stream.ts +4 -5
- package/dist/templates/transcription-workflow/workflows/sync-api.ts +6 -6
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +32 -26
- package/dist/templates/transcription-workflow/workflows/wav.ts +0 -31
- package/dist/templates/travel-concierge/agent.eval.test.ts +323 -0
- package/dist/templates/travel-concierge/agent.test.ts +174 -43
- package/dist/templates/travel-concierge/agent.ts +18 -9
- package/dist/templates/travel-concierge/client.tsx +11 -23
- package/dist/templates/travel-concierge/routing.ts +34 -15
- package/dist/templates/travel-concierge/shared.ts +180 -12
- package/dist/templates/travel-concierge/system-prompt.md +7 -1
- package/dist/templates/travel-concierge/tools/book_car_rental.ts +8 -5
- package/dist/templates/travel-concierge/tools/book_excursion.ts +8 -5
- package/dist/templates/travel-concierge/tools/book_hotel.ts +8 -5
- package/dist/templates/travel-concierge/tools/cancel_ticket.ts +8 -5
- package/dist/templates/travel-concierge/tools/search_car_rentals.ts +9 -5
- package/dist/templates/travel-concierge/tools/search_excursions.ts +9 -5
- package/dist/templates/travel-concierge/tools/search_flights.ts +9 -6
- package/dist/templates/travel-concierge/tools/search_hotels.ts +9 -5
- package/dist/templates/travel-concierge/tools/update_ticket.ts +8 -5
- package/dist/templates/web-researcher/agent.eval.test.ts +121 -0
- package/dist/templates/web-researcher/system-prompt.md +15 -4
- package/dist/{test-9kPKJI-w.mjs → test-BvyT6sAc.mjs} +24 -10
- package/dist/test.d.ts +29 -5
- package/dist/{worker-bundler-CGD4r8Kc.mjs → worker-bundler-CCVEDjm1.mjs} +241 -3
- package/dist/worker-bundler.mjs +1 -1
- package/dist/{workflow-CFpxOFfQ.mjs → workflow-BflATMmU.mjs} +47 -12
- package/dist/workflow-bundler.d.ts +6 -1
- package/dist/workflow.d.ts +1 -1
- package/package.json +5 -4
- package/dist/eject-31gjtaHF.mjs +0 -116
- package/dist/eject.d.ts +0 -60
- package/dist/templates/call-audit/workflows/temp-media.ts +0 -138
- package/dist/templates/infocom-adventure/tools/game_state_history.ts +0 -14
- package/dist/templates/support-line/graph.ts +0 -224
- package/dist/{_config-D_s09e7g.mjs → _config-CmJOFsAP.mjs} +1 -1
- /package/dist/templates/plan-and-execute/{graph.ts → procedure.ts} +0 -0
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
/// <reference types="vite/client" />
|
|
2
2
|
|
|
3
|
-
import type { GenerateFn, ToolContext } from "@alexkroman1/aai";
|
|
4
3
|
import {
|
|
5
4
|
createToolContext,
|
|
6
|
-
runTool,
|
|
7
5
|
stubGenerate,
|
|
6
|
+
toolRunner,
|
|
8
7
|
withDiscoveredTools,
|
|
9
8
|
} from "@alexkroman1/aai/testing";
|
|
10
9
|
import { describe, expect, test } from "vitest";
|
|
@@ -23,7 +22,7 @@ const agentDef = withDiscoveredTools(
|
|
|
23
22
|
import.meta.glob("./tools/*.ts", { eager: true }),
|
|
24
23
|
);
|
|
25
24
|
|
|
26
|
-
import { MAX_ATTEMPTS, runCorrectiveRag } from "./
|
|
25
|
+
import { MAX_ATTEMPTS, runCorrectiveRag } from "./procedure.ts";
|
|
27
26
|
import {
|
|
28
27
|
ANSWER_SYSTEM,
|
|
29
28
|
ANSWERS_SYSTEM,
|
|
@@ -95,15 +94,11 @@ function scriptedModel(script: Script = {}) {
|
|
|
95
94
|
return { generate, calls };
|
|
96
95
|
}
|
|
97
96
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
* and its "no such tool" message are `runTool`'s (`@alexkroman1/aai/testing`);
|
|
104
|
-
* what is local is only which agent they run against. */
|
|
105
|
-
const run = (name: string, args: Record<string, unknown>, ctx: ToolContext) =>
|
|
106
|
-
runTool(agentDef, name, args, ctx);
|
|
97
|
+
/** A tool by the name the model calls it by, bound to this agent. The lookup,
|
|
98
|
+
* its "no such tool" message and the args-or-context shape are all
|
|
99
|
+
* `toolRunner`'s (`@alexkroman1/aai/testing`); what is local is only which
|
|
100
|
+
* agent it runs against. */
|
|
101
|
+
const run = toolRunner(agentDef);
|
|
107
102
|
|
|
108
103
|
/** Node names without the per-call suffix, for sequence assertions. */
|
|
109
104
|
function nodes(calls: string[]): string[] {
|
|
@@ -254,7 +249,7 @@ describe("answer_question", () => {
|
|
|
254
249
|
relevant: (id) => id === "D8",
|
|
255
250
|
answers: ["Area outages are on the status page, and rebooting will not help."],
|
|
256
251
|
});
|
|
257
|
-
const ctx =
|
|
252
|
+
const ctx = createToolContext({ generate });
|
|
258
253
|
const result = (await run("answer_question", { question: "is there an outage" }, ctx)) as {
|
|
259
254
|
answer: string;
|
|
260
255
|
sources: string[];
|
|
@@ -276,7 +271,7 @@ describe("answer_question", () => {
|
|
|
276
271
|
|
|
277
272
|
test("with nothing grounded it returns no answer and points at the ticket", async () => {
|
|
278
273
|
const { generate } = scriptedModel({ relevant: () => false });
|
|
279
|
-
const ctx =
|
|
274
|
+
const ctx = createToolContext({ generate });
|
|
280
275
|
const result = (await run("answer_question", { question: "do you sell phones" }, ctx)) as {
|
|
281
276
|
answer: null;
|
|
282
277
|
guidance: string;
|
|
@@ -302,8 +297,8 @@ describe("answer_question", () => {
|
|
|
302
297
|
// session ids would prove nothing extra, and `sessionSlot` could stop
|
|
303
298
|
// keying by session with this still passing.
|
|
304
299
|
const { generate } = scriptedModel({ relevant: (id) => id === "D2", answers: ["Reboot it."] });
|
|
305
|
-
const first =
|
|
306
|
-
const second =
|
|
300
|
+
const first = createToolContext({ generate });
|
|
301
|
+
const second = createToolContext({ generate });
|
|
307
302
|
|
|
308
303
|
await run("answer_question", { question: "how do I reboot" }, first);
|
|
309
304
|
expect(supportSlot.get(second).trace).toBeNull();
|
|
@@ -3,7 +3,7 @@ import { PRODUCT, supportProjection } from "./shared.ts";
|
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* A support line that grades its own retrieval before it speaks — the
|
|
6
|
-
* self-RAG / CRAG graph, ported to voice. `
|
|
6
|
+
* self-RAG / CRAG graph, ported to voice. `procedure.ts` holds the loop, `prompts.ts`
|
|
7
7
|
* the attribution and the graders, `shared.ts` the knowledge base and why its
|
|
8
8
|
* retriever is lexical.
|
|
9
9
|
*
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import "@alexkroman1/aai-ui/styles.css";
|
|
2
|
-
import { AutoScroll, client, useAgentState
|
|
2
|
+
import { AutoScroll, client, useAgentState } from "@alexkroman1/aai-ui";
|
|
3
3
|
import { PRODUCT, supportProjection } from "./shared.ts";
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -11,12 +11,11 @@ import { PRODUCT, supportProjection } from "./shared.ts";
|
|
|
11
11
|
* support line nobody can tell apart from one that just guesses well.
|
|
12
12
|
*/
|
|
13
13
|
function TraceSidebar() {
|
|
14
|
-
const theme = useTheme();
|
|
15
14
|
const support = useAgentState(supportProjection);
|
|
16
15
|
const trace = support.trace;
|
|
17
16
|
|
|
18
17
|
return (
|
|
19
|
-
<div className="flex h-full min-h-0 flex-col gap-4 p-4
|
|
18
|
+
<div className="flex h-full min-h-0 flex-col gap-4 p-4 text-aai-text">
|
|
20
19
|
<div>
|
|
21
20
|
<h3 className="text-sm font-bold uppercase tracking-wide opacity-60">{support.product}</h3>
|
|
22
21
|
<p className="text-xs opacity-50">
|
|
@@ -36,7 +35,7 @@ function TraceSidebar() {
|
|
|
36
35
|
scrollClassName="min-h-0 overflow-y-auto"
|
|
37
36
|
contentClassName="flex flex-col gap-4 pr-1"
|
|
38
37
|
>
|
|
39
|
-
<div className="rounded-lg p-3
|
|
38
|
+
<div className="rounded-lg p-3 bg-aai-surface">
|
|
40
39
|
<p className="text-[11px] font-bold uppercase tracking-wide opacity-60">Question</p>
|
|
41
40
|
<p className="text-sm">{trace.question}</p>
|
|
42
41
|
{trace.rewrites > 0 && (
|
|
@@ -54,10 +53,11 @@ function TraceSidebar() {
|
|
|
54
53
|
{trace.docs.map((doc) => (
|
|
55
54
|
<div
|
|
56
55
|
key={doc.id}
|
|
57
|
-
className=
|
|
58
|
-
|
|
56
|
+
className={`flex items-start gap-2 rounded-lg p-2 bg-aai-surface ${
|
|
57
|
+
doc.relevant ? "" : "opacity-50"
|
|
58
|
+
}`}
|
|
59
59
|
>
|
|
60
|
-
<span
|
|
60
|
+
<span className={doc.relevant ? "text-aai-primary" : "text-aai-text"}>
|
|
61
61
|
{doc.relevant ? "✓" : "×"}
|
|
62
62
|
</span>
|
|
63
63
|
<div className="min-w-0">
|
|
@@ -69,7 +69,7 @@ function TraceSidebar() {
|
|
|
69
69
|
</div>
|
|
70
70
|
|
|
71
71
|
<div className="flex flex-col gap-1">
|
|
72
|
-
<p className="text-[11px] font-bold uppercase tracking-wide opacity-60">
|
|
72
|
+
<p className="text-[11px] font-bold uppercase tracking-wide opacity-60">Procedure</p>
|
|
73
73
|
{trace.steps.map((entry, index) => (
|
|
74
74
|
// Steps are append-only within a run, so the index is stable.
|
|
75
75
|
<p key={`${index}-${entry.node}`} className="text-xs">
|
|
@@ -79,7 +79,7 @@ function TraceSidebar() {
|
|
|
79
79
|
))}
|
|
80
80
|
</div>
|
|
81
81
|
|
|
82
|
-
<div className="rounded-lg p-3
|
|
82
|
+
<div className="rounded-lg p-3 bg-aai-surface">
|
|
83
83
|
<p className="text-[11px] font-bold uppercase tracking-wide opacity-60">Verdict</p>
|
|
84
84
|
{trace.answer ? (
|
|
85
85
|
<p className="text-sm">{trace.answer}</p>
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The CRAG nodes' BODIES — one model call each, and nothing about routing.
|
|
3
|
+
*
|
|
4
|
+
* Split from `procedure.ts` when that file became an actual machine: the machine
|
|
5
|
+
* there declares which node runs next, and these are what each node does. The
|
|
6
|
+
* seam is worth having beyond the line count — everything here is a plain async
|
|
7
|
+
* function of its arguments, so a spec can drive one directly, and the machine
|
|
8
|
+
* holds no prompt text at all.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { GenerateFn } from "@alexkroman1/aai";
|
|
12
|
+
import {
|
|
13
|
+
ANSWER_SYSTEM,
|
|
14
|
+
ANSWERS_SYSTEM,
|
|
15
|
+
binaryScore,
|
|
16
|
+
DOC_GRADER_SYSTEM,
|
|
17
|
+
GROUNDED_SYSTEM,
|
|
18
|
+
REWRITE_SYSTEM,
|
|
19
|
+
} from "./prompts.ts";
|
|
20
|
+
import type { Doc, GradedDoc } from "./shared.ts";
|
|
21
|
+
import { formatDoc } from "./shared.ts";
|
|
22
|
+
|
|
23
|
+
/** A closed verdict from one of the graders. */
|
|
24
|
+
export interface Verdict {
|
|
25
|
+
pass: boolean;
|
|
26
|
+
reason: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** `grade_documents` — one closed question per document, all at once. */
|
|
30
|
+
export async function gradeDocuments(
|
|
31
|
+
generate: GenerateFn,
|
|
32
|
+
question: string,
|
|
33
|
+
docs: readonly Doc[],
|
|
34
|
+
): Promise<GradedDoc[]> {
|
|
35
|
+
return await Promise.all(
|
|
36
|
+
docs.map(async (doc) => {
|
|
37
|
+
const { object } = await generate({
|
|
38
|
+
system: DOC_GRADER_SYSTEM,
|
|
39
|
+
prompt: `Caller's question: ${question}\n\nDocument:\n${formatDoc(doc)}`,
|
|
40
|
+
schema: binaryScore,
|
|
41
|
+
});
|
|
42
|
+
return {
|
|
43
|
+
id: doc.id,
|
|
44
|
+
title: doc.title,
|
|
45
|
+
relevant: object.score === "yes",
|
|
46
|
+
reason: object.reason,
|
|
47
|
+
};
|
|
48
|
+
}),
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** `transform_query` — their question re-writer, aimed at a keyword index. */
|
|
53
|
+
export async function transformQuery(generate: GenerateFn, question: string): Promise<string> {
|
|
54
|
+
const { text } = await generate({
|
|
55
|
+
system: REWRITE_SYSTEM,
|
|
56
|
+
prompt: `Caller's question: ${question}`,
|
|
57
|
+
});
|
|
58
|
+
const rewritten = text.trim().replace(/^["']|["']$/g, "");
|
|
59
|
+
// A rewriter that returns nothing usable must not empty the query — that
|
|
60
|
+
// would retrieve zero documents and read as "we have nothing on this".
|
|
61
|
+
return rewritten.length > 0 ? rewritten : question;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** `generate` — the answer, from the graded-relevant documents only. */
|
|
65
|
+
export async function generateAnswer(
|
|
66
|
+
generate: GenerateFn,
|
|
67
|
+
question: string,
|
|
68
|
+
docs: readonly Doc[],
|
|
69
|
+
): Promise<string> {
|
|
70
|
+
const { text } = await generate({
|
|
71
|
+
system: ANSWER_SYSTEM,
|
|
72
|
+
prompt: `Documents:\n\n${docs.map(formatDoc).join("\n\n")}\n\nCaller's question: ${question}`,
|
|
73
|
+
});
|
|
74
|
+
return text.trim();
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** The two generation graders — same closed question, different subject. */
|
|
78
|
+
async function grade(generate: GenerateFn, system: string, prompt: string): Promise<Verdict> {
|
|
79
|
+
const { object } = await generate({ system, prompt, schema: binaryScore });
|
|
80
|
+
return { pass: object.score === "yes", reason: object.reason };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** `grade_generation_v_documents` — is the answer IN the documents? */
|
|
84
|
+
export async function gradeGrounded(
|
|
85
|
+
generate: GenerateFn,
|
|
86
|
+
docs: readonly Doc[],
|
|
87
|
+
answer: string,
|
|
88
|
+
): Promise<Verdict> {
|
|
89
|
+
const facts = docs.map(formatDoc).join("\n\n");
|
|
90
|
+
return await grade(generate, GROUNDED_SYSTEM, `Facts:\n\n${facts}\n\nAnswer: ${answer}`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** `grade_generation_v_question` — grounded is not the same as useful. */
|
|
94
|
+
export async function gradeUseful(
|
|
95
|
+
generate: GenerateFn,
|
|
96
|
+
question: string,
|
|
97
|
+
answer: string,
|
|
98
|
+
): Promise<Verdict> {
|
|
99
|
+
return await grade(generate, ANSWERS_SYSTEM, `Question: ${question}\n\nAnswer: ${answer}`);
|
|
100
|
+
}
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The corrective-RAG loop, as a `procedure()`.
|
|
3
|
+
*
|
|
4
|
+
* This is the port of the self-RAG / CRAG graph (see `prompts.ts` for the
|
|
5
|
+
* attribution), and it is now the same SHAPE as the thing it ports: their
|
|
6
|
+
* control flow is edges between nodes, and so is this.
|
|
7
|
+
*
|
|
8
|
+
* ```text
|
|
9
|
+
* retrieve → grade_documents → decide_to_generate
|
|
10
|
+
* ├─ no relevant docs → transform_query → retrieve
|
|
11
|
+
* └─ generate → grade_generation_v_documents
|
|
12
|
+
* ├─ not grounded → generate (once)
|
|
13
|
+
* └─ grade_generation_v_question
|
|
14
|
+
* ├─ not useful → transform_query
|
|
15
|
+
* └─ done
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* **It used to be a `while` loop with the node names preserved in a trace**, so
|
|
19
|
+
* that a run stayed readable as the graph it came from. That worked and cost two
|
|
20
|
+
* things worth getting back. The node names were a STRING passed to a `step()`
|
|
21
|
+
* helper beside the code that did the work, so nothing stopped the two drifting;
|
|
22
|
+
* here a node IS a state and the trace is written by that state's `entry`, so a
|
|
23
|
+
* renamed node cannot keep an old label. And the routing was spread across a
|
|
24
|
+
* `continue`, a nested `for`, and four early `return`s — the exact structure a
|
|
25
|
+
* statechart exists to make declarative. `xstate` is a dependency of the SDK, so
|
|
26
|
+
* this costs the template nothing to reach for.
|
|
27
|
+
*
|
|
28
|
+
* **The budget is the mechanism, not the prompt.** A procedure with a
|
|
29
|
+
* `transform_query` edge can cycle, and their notebooks bound it with a recursion
|
|
30
|
+
* limit that raises on hit. A caller is holding the line, so the bound here is
|
|
31
|
+
* two attempts and one regeneration — {@link MAX_ATTEMPTS} — expressed as GUARDS
|
|
32
|
+
* (`canRetry`, `canRegenerate`) rather than as loop conditions. Running out is an
|
|
33
|
+
* ANSWER (`exhausted`, the caller is offered a ticket) rather than an error: an
|
|
34
|
+
* agent that cannot say "I don't have that documented" is the failure the whole
|
|
35
|
+
* grading apparatus exists to prevent, so it has to be a reachable state — and
|
|
36
|
+
* here it is literally one.
|
|
37
|
+
*
|
|
38
|
+
* **Documents are graded CONCURRENTLY.** Their loop is sequential because a
|
|
39
|
+
* notebook does not care; four serial grader calls is four round trips of dead
|
|
40
|
+
* air on a phone. The grades are independent, so the fan-out is free — it lives
|
|
41
|
+
* inside the `gradeDocuments` node (`nodes.ts`), not in the routing.
|
|
42
|
+
*
|
|
43
|
+
* **`procedure()` is what runs it, and no actor appears in this template.** The two
|
|
44
|
+
* SDK machine primitives are for different jobs and the split is worth knowing:
|
|
45
|
+
* a `dialog()` is where a CONVERSATION is (persisted in a slot, moved one event at
|
|
46
|
+
* a time by the caller's turns), and a `procedure()` is one unit of WORK inside a
|
|
47
|
+
* single tool call (never stored, driving itself through invoked actors). This is
|
|
48
|
+
* the second, so its context may hold a `GenerateFn` that no stored slot could.
|
|
49
|
+
*
|
|
50
|
+
* `procedure().run` is also what makes the loop interruptible: it takes
|
|
51
|
+
* `ctx.signal`, so a caller who barges in on the second of nine model calls is
|
|
52
|
+
* not charged for the remaining seven.
|
|
53
|
+
*/
|
|
54
|
+
|
|
55
|
+
import { type GenerateFn, omitUndefined, procedure } from "@alexkroman1/aai";
|
|
56
|
+
import { assign, fromPromise, setup } from "xstate";
|
|
57
|
+
import {
|
|
58
|
+
generateAnswer,
|
|
59
|
+
gradeDocuments,
|
|
60
|
+
gradeGrounded,
|
|
61
|
+
gradeUseful,
|
|
62
|
+
transformQuery,
|
|
63
|
+
type Verdict,
|
|
64
|
+
} from "./nodes.ts";
|
|
65
|
+
import type { AnswerTrace, Doc, GradedDoc, TraceStep } from "./shared.ts";
|
|
66
|
+
import { retrieve } from "./shared.ts";
|
|
67
|
+
|
|
68
|
+
/** Retrieve-and-answer attempts, i.e. one query rewrite. */
|
|
69
|
+
export const MAX_ATTEMPTS = 2;
|
|
70
|
+
/** Regenerations after a "not grounded" verdict, within one attempt. */
|
|
71
|
+
export const MAX_REGENERATIONS = 1;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* What the machine carries: the trace it is building, plus the two counters the
|
|
75
|
+
* guards read and the documents the current attempt is answering from.
|
|
76
|
+
*
|
|
77
|
+
* `generate` rides in here because this machine is never stored — see the module
|
|
78
|
+
* doc. A {@link dialog}'s context has to survive `structuredClone`; this one only
|
|
79
|
+
* has to survive one tool call.
|
|
80
|
+
*/
|
|
81
|
+
interface Ctx extends AnswerTrace {
|
|
82
|
+
generate: GenerateFn;
|
|
83
|
+
/** 1-based, bounded by {@link MAX_ATTEMPTS}. */
|
|
84
|
+
attempt: number;
|
|
85
|
+
/** Regenerations spent WITHIN this attempt; reset by `transform_query`. */
|
|
86
|
+
regenerations: number;
|
|
87
|
+
/** What this attempt's query retrieved, before grading. */
|
|
88
|
+
retrieved: Doc[];
|
|
89
|
+
/** The documents this attempt retrieved and the grader passed. */
|
|
90
|
+
relevant: Doc[];
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* One trace entry, as a `{ type: "note", params }` action.
|
|
95
|
+
*
|
|
96
|
+
* A parameterized action declared INSIDE `setup` rather than a helper that
|
|
97
|
+
* returns a bare `assign`: an action built outside carries no knowledge of this
|
|
98
|
+
* machine's actors, so `exactOptionalPropertyTypes` refuses it wherever the
|
|
99
|
+
* machine expects one of its own. `params` may be a function, which is what lets
|
|
100
|
+
* a detail string read the context it is describing.
|
|
101
|
+
*/
|
|
102
|
+
function noteAt(node: string, detail: string): { type: "note"; params: NoteParams } {
|
|
103
|
+
return { type: "note", params: { node, detail } };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** The same, for a detail computed from the context the entry describes. */
|
|
107
|
+
function noteFrom(node: string, detail: (context: Ctx) => string) {
|
|
108
|
+
return {
|
|
109
|
+
type: "note" as const,
|
|
110
|
+
params: ({ context }: { context: Ctx }): NoteParams => ({ node, detail: detail(context) }),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
interface NoteParams {
|
|
115
|
+
node: string;
|
|
116
|
+
detail: string;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const machine = setup({
|
|
120
|
+
types: {} as {
|
|
121
|
+
context: Ctx;
|
|
122
|
+
input: { generate: GenerateFn; question: string };
|
|
123
|
+
output: AnswerTrace;
|
|
124
|
+
},
|
|
125
|
+
actors: {
|
|
126
|
+
gradeDocuments: fromPromise(
|
|
127
|
+
async ({ input }: { input: { ctx: Ctx; docs: Doc[] } }): Promise<GradedDoc[]> =>
|
|
128
|
+
await gradeDocuments(input.ctx.generate, input.ctx.question, input.docs),
|
|
129
|
+
),
|
|
130
|
+
generateAnswer: fromPromise(
|
|
131
|
+
async ({ input }: { input: { ctx: Ctx } }): Promise<string> =>
|
|
132
|
+
await generateAnswer(input.ctx.generate, input.ctx.question, input.ctx.relevant),
|
|
133
|
+
),
|
|
134
|
+
gradeGrounded: fromPromise(
|
|
135
|
+
async ({ input }: { input: { ctx: Ctx } }): Promise<Verdict> =>
|
|
136
|
+
await gradeGrounded(input.ctx.generate, input.ctx.relevant, input.ctx.answer ?? ""),
|
|
137
|
+
),
|
|
138
|
+
gradeUseful: fromPromise(
|
|
139
|
+
async ({ input }: { input: { ctx: Ctx } }): Promise<Verdict> =>
|
|
140
|
+
await gradeUseful(input.ctx.generate, input.ctx.question, input.ctx.answer ?? ""),
|
|
141
|
+
),
|
|
142
|
+
transformQuery: fromPromise(
|
|
143
|
+
async ({ input }: { input: { ctx: Ctx } }): Promise<string> =>
|
|
144
|
+
await transformQuery(input.ctx.generate, input.ctx.question),
|
|
145
|
+
),
|
|
146
|
+
},
|
|
147
|
+
actions: {
|
|
148
|
+
note: assign({
|
|
149
|
+
steps: ({ context }, params: NoteParams): TraceStep[] => [
|
|
150
|
+
...context.steps,
|
|
151
|
+
{ node: params.node, detail: params.detail },
|
|
152
|
+
],
|
|
153
|
+
}),
|
|
154
|
+
},
|
|
155
|
+
guards: {
|
|
156
|
+
/** Their `decide_to_generate`: is there anything to answer FROM? */
|
|
157
|
+
hasRelevant: ({ context }) => context.relevant.length > 0,
|
|
158
|
+
canRetry: ({ context }) => context.attempt < MAX_ATTEMPTS,
|
|
159
|
+
canRegenerate: ({ context }) => context.regenerations < MAX_REGENERATIONS,
|
|
160
|
+
},
|
|
161
|
+
// A grader's verdict is read INLINE (`event.output.pass`) rather than through
|
|
162
|
+
// a named guard: a guard in `setup` is typed against the machine's whole event
|
|
163
|
+
// union, so it cannot see that this particular transition is a done-event.
|
|
164
|
+
}).createMachine({
|
|
165
|
+
id: "correctiveRag",
|
|
166
|
+
initial: "retrieve",
|
|
167
|
+
context: ({ input }) => ({
|
|
168
|
+
generate: input.generate,
|
|
169
|
+
question: input.question,
|
|
170
|
+
query: input.question,
|
|
171
|
+
rewrites: 0,
|
|
172
|
+
steps: [],
|
|
173
|
+
docs: [],
|
|
174
|
+
answer: null,
|
|
175
|
+
grounded: null,
|
|
176
|
+
useful: null,
|
|
177
|
+
exhausted: false,
|
|
178
|
+
attempt: 1,
|
|
179
|
+
regenerations: 0,
|
|
180
|
+
retrieved: [],
|
|
181
|
+
relevant: [],
|
|
182
|
+
}),
|
|
183
|
+
states: {
|
|
184
|
+
/**
|
|
185
|
+
* Retrieval is LEXICAL and synchronous (`shared.ts`), so it is an entry
|
|
186
|
+
* action rather than an invoked actor — there is nothing to await.
|
|
187
|
+
*/
|
|
188
|
+
retrieve: {
|
|
189
|
+
entry: [
|
|
190
|
+
// Assigned ONCE and read from context by everything downstream: the
|
|
191
|
+
// grader's input, the relevance filter and the trace all have to be
|
|
192
|
+
// talking about the same retrieval, and calling `retrieve` again per
|
|
193
|
+
// reader is how they come to disagree.
|
|
194
|
+
assign({
|
|
195
|
+
retrieved: ({ context }) => retrieve(context.query).map((one) => one.doc),
|
|
196
|
+
relevant: () => [],
|
|
197
|
+
docs: () => [],
|
|
198
|
+
}),
|
|
199
|
+
// Actions run in order and an `assign` is visible to the next one, so
|
|
200
|
+
// this reads the retrieval above rather than redoing it.
|
|
201
|
+
noteFrom("retrieve", (ctx) => `${ctx.retrieved.length} document(s) for "${ctx.query}"`),
|
|
202
|
+
],
|
|
203
|
+
always: "gradeDocuments",
|
|
204
|
+
},
|
|
205
|
+
|
|
206
|
+
gradeDocuments: {
|
|
207
|
+
invoke: {
|
|
208
|
+
src: "gradeDocuments",
|
|
209
|
+
input: ({ context }) => ({ ctx: context, docs: context.retrieved }),
|
|
210
|
+
onDone: {
|
|
211
|
+
target: "decideToGenerate",
|
|
212
|
+
actions: [
|
|
213
|
+
assign({
|
|
214
|
+
docs: ({ event }) => event.output,
|
|
215
|
+
relevant: ({ context, event }) =>
|
|
216
|
+
context.retrieved.filter(
|
|
217
|
+
(doc) => event.output.find((graded) => graded.id === doc.id)?.relevant,
|
|
218
|
+
),
|
|
219
|
+
}),
|
|
220
|
+
noteFrom(
|
|
221
|
+
"grade_documents",
|
|
222
|
+
(ctx) => `${ctx.relevant.length} of ${ctx.docs.length} relevant`,
|
|
223
|
+
),
|
|
224
|
+
],
|
|
225
|
+
},
|
|
226
|
+
},
|
|
227
|
+
},
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Nothing relevant means the RETRIEVAL was wrong, not that the answer is
|
|
231
|
+
* unknown — so rewrite and try again before giving up. Pure routing: three
|
|
232
|
+
* guarded edges and no work of its own.
|
|
233
|
+
*/
|
|
234
|
+
decideToGenerate: {
|
|
235
|
+
always: [
|
|
236
|
+
{ guard: "hasRelevant", target: "generate" },
|
|
237
|
+
{ guard: "canRetry", target: "transformQuery" },
|
|
238
|
+
{
|
|
239
|
+
target: "exhausted",
|
|
240
|
+
actions: [
|
|
241
|
+
assign({ exhausted: () => true }),
|
|
242
|
+
noteAt("decide_to_generate", "no relevant documents, and no attempts left"),
|
|
243
|
+
],
|
|
244
|
+
},
|
|
245
|
+
],
|
|
246
|
+
},
|
|
247
|
+
|
|
248
|
+
generate: {
|
|
249
|
+
invoke: {
|
|
250
|
+
src: "generateAnswer",
|
|
251
|
+
input: ({ context }) => ({ ctx: context }),
|
|
252
|
+
onDone: {
|
|
253
|
+
target: "gradeGrounded",
|
|
254
|
+
actions: [
|
|
255
|
+
assign({ answer: ({ event }) => event.output }),
|
|
256
|
+
noteFrom("generate", (ctx) =>
|
|
257
|
+
ctx.regenerations > 0
|
|
258
|
+
? "regenerated"
|
|
259
|
+
: `answered from ${ctx.relevant.map((doc) => doc.id).join(", ")}`,
|
|
260
|
+
),
|
|
261
|
+
],
|
|
262
|
+
},
|
|
263
|
+
},
|
|
264
|
+
},
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Is it grounded? A regeneration is worth one shot — the same documents,
|
|
268
|
+
* asked again. Two answers the grader rejected is a signal about the
|
|
269
|
+
* DOCUMENTS rather than about the phrasing, so the second refusal is final
|
|
270
|
+
* and the answer is withheld: an ungrounded answer is never spoken.
|
|
271
|
+
*/
|
|
272
|
+
gradeGrounded: {
|
|
273
|
+
invoke: {
|
|
274
|
+
src: "gradeGrounded",
|
|
275
|
+
input: ({ context }) => ({ ctx: context }),
|
|
276
|
+
onDone: [
|
|
277
|
+
{
|
|
278
|
+
guard: ({ event }) => event.output.pass,
|
|
279
|
+
target: "gradeUseful",
|
|
280
|
+
actions: [
|
|
281
|
+
assign({ grounded: () => true }),
|
|
282
|
+
noteAt("grade_generation_v_documents", "grounded"),
|
|
283
|
+
],
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
guard: "canRegenerate",
|
|
287
|
+
target: "generate",
|
|
288
|
+
actions: [
|
|
289
|
+
noteAt("grade_generation_v_documents", "not grounded"),
|
|
290
|
+
assign({ regenerations: ({ context }) => context.regenerations + 1 }),
|
|
291
|
+
],
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
target: "ungrounded",
|
|
295
|
+
actions: [
|
|
296
|
+
assign({ grounded: () => false, answer: () => null, exhausted: () => true }),
|
|
297
|
+
noteAt("grade_generation_v_documents", "still not grounded"),
|
|
298
|
+
],
|
|
299
|
+
},
|
|
300
|
+
],
|
|
301
|
+
},
|
|
302
|
+
},
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Grounded is not the same as useful. A beside-the-point answer is rewritten
|
|
306
|
+
* once and, out of attempts, RETURNED with its verdict — it is still true,
|
|
307
|
+
* and the tool tells the model to offer a ticket alongside it.
|
|
308
|
+
*/
|
|
309
|
+
gradeUseful: {
|
|
310
|
+
invoke: {
|
|
311
|
+
src: "gradeUseful",
|
|
312
|
+
input: ({ context }) => ({ ctx: context }),
|
|
313
|
+
onDone: [
|
|
314
|
+
{
|
|
315
|
+
guard: ({ event }) => event.output.pass,
|
|
316
|
+
target: "done",
|
|
317
|
+
actions: [
|
|
318
|
+
assign({ useful: () => true }),
|
|
319
|
+
noteAt("grade_generation_v_question", "useful"),
|
|
320
|
+
],
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
guard: "canRetry",
|
|
324
|
+
target: "transformQuery",
|
|
325
|
+
actions: [
|
|
326
|
+
assign({ useful: () => false }),
|
|
327
|
+
noteAt("grade_generation_v_question", "not useful"),
|
|
328
|
+
],
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
target: "exhausted",
|
|
332
|
+
actions: [
|
|
333
|
+
assign({ useful: () => false, exhausted: () => true }),
|
|
334
|
+
noteAt("grade_generation_v_question", "not useful, and no attempts left"),
|
|
335
|
+
],
|
|
336
|
+
},
|
|
337
|
+
],
|
|
338
|
+
},
|
|
339
|
+
},
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* The corrective edge. It resets the attempt's verdicts as well as its
|
|
343
|
+
* counters: a rewrite reopens the question, so an answer graded against the
|
|
344
|
+
* OLD query must not survive into the new attempt's trace.
|
|
345
|
+
*/
|
|
346
|
+
transformQuery: {
|
|
347
|
+
invoke: {
|
|
348
|
+
src: "transformQuery",
|
|
349
|
+
input: ({ context }) => ({ ctx: context }),
|
|
350
|
+
onDone: {
|
|
351
|
+
target: "retrieve",
|
|
352
|
+
actions: [
|
|
353
|
+
assign({
|
|
354
|
+
query: ({ event }) => event.output,
|
|
355
|
+
rewrites: ({ context }) => context.rewrites + 1,
|
|
356
|
+
attempt: ({ context }) => context.attempt + 1,
|
|
357
|
+
regenerations: () => 0,
|
|
358
|
+
answer: () => null,
|
|
359
|
+
grounded: () => null,
|
|
360
|
+
useful: () => null,
|
|
361
|
+
}),
|
|
362
|
+
noteFrom("transform_query", (ctx) => `retrying as "${ctx.query}"`),
|
|
363
|
+
],
|
|
364
|
+
},
|
|
365
|
+
},
|
|
366
|
+
},
|
|
367
|
+
|
|
368
|
+
done: { type: "final" },
|
|
369
|
+
exhausted: { type: "final" },
|
|
370
|
+
ungrounded: { type: "final" },
|
|
371
|
+
},
|
|
372
|
+
output: ({ context }) => ({
|
|
373
|
+
question: context.question,
|
|
374
|
+
query: context.query,
|
|
375
|
+
rewrites: context.rewrites,
|
|
376
|
+
steps: context.steps,
|
|
377
|
+
docs: context.docs,
|
|
378
|
+
answer: context.answer,
|
|
379
|
+
grounded: context.grounded,
|
|
380
|
+
useful: context.useful,
|
|
381
|
+
exhausted: context.exhausted,
|
|
382
|
+
}),
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
const rag = procedure(machine);
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* Run the procedure for one caller question.
|
|
389
|
+
*
|
|
390
|
+
* Never throws for a bad ANSWER — every way of failing to answer is a final
|
|
391
|
+
* STATE, and the trace says which one. Two things DO throw, and both are the
|
|
392
|
+
* tool's to report: a broken model call (an invoked actor's rejection has no
|
|
393
|
+
* `onError` here, so it stops the machine) and an aborted run.
|
|
394
|
+
*
|
|
395
|
+
* `signal` is optional only so the procedure stays drivable from a spec that has no
|
|
396
|
+
* context; a tool body should always pass `ctx.signal`.
|
|
397
|
+
*/
|
|
398
|
+
export async function runCorrectiveRag(
|
|
399
|
+
generate: GenerateFn,
|
|
400
|
+
question: string,
|
|
401
|
+
signal?: AbortSignal,
|
|
402
|
+
): Promise<AnswerTrace> {
|
|
403
|
+
// `omitUndefined` rather than a conditional spread: `ProcedureRunOptions.signal`
|
|
404
|
+
// is optional, and under `exactOptionalPropertyTypes` a present-and-undefined
|
|
405
|
+
// key is not the same as an absent one.
|
|
406
|
+
return await rag.run({ generate, question }, omitUndefined({ signal }));
|
|
407
|
+
}
|
|
@@ -7,7 +7,8 @@ The one rule that matters:
|
|
|
7
7
|
|
|
8
8
|
Running a call:
|
|
9
9
|
|
|
10
|
-
-
|
|
10
|
+
- **Every turn in which the caller asks anything about Meridian Fibre begins with a call to `answer_question`.** The call goes out first, in that turn, before you decide what to say. That is the whole procedure and there is no version of it where the lookup happens later.
|
|
11
|
+
- "Let me check that for you" is not a check. It is what you may say while the call you have already made is running, in the same turn — five words, no more. **A turn that ends on that sentence with no `answer_question` call in it has hung up on the caller**: they are holding a line that never comes back, which is worse than the silence the sentence exists to fill. If you have said it, the tool call is already out or the turn is not finished.
|
|
11
12
|
- Give the answer it returns in your own words, keeping every number exactly as it came back.
|
|
12
13
|
- When it returns no answer, say plainly that you do not have that documented. Do not guess, and do not soften it into a maybe. Offer to log a ticket.
|
|
13
14
|
- When it returns an answer with a caveat, give the answer and then offer the ticket.
|