@terreno/ai 57.11.0 → 57.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aiApp.d.ts +5 -0
- package/dist/aiApp.js +4 -2
- package/dist/harness/commit.d.ts +17 -1
- package/dist/harness/commit.js +86 -2
- package/dist/harness/routes/approvals.js +2 -2
- package/dist/harness/routes/conversations.js +1 -1
- package/dist/harness/routes/tasks.js +85 -3
- package/dist/harness/runtime.js +7 -1
- package/dist/harness/subagent.d.ts +17 -2
- package/dist/harness/subagent.js +26 -2
- package/dist/index.d.ts +5 -1
- package/dist/index.js +11 -2
- package/dist/models/project.d.ts +13 -0
- package/dist/models/project.js +32 -2
- package/dist/observability/local/models/obsTrace.js +16 -2
- package/dist/observability/local/traceStore.d.ts +5 -0
- package/dist/observability/local/traceStore.js +5 -0
- package/dist/observability/routes/traces.js +2 -0
- package/dist/routes/gptActions.js +15 -2
- package/dist/routes/projects.js +3 -3
- package/dist/service/agentBlocks.d.ts +20 -0
- package/dist/service/agentBlocks.js +92 -0
- package/dist/service/chatTurn.d.ts +3 -1
- package/dist/service/chatTurn.js +34 -9
- package/dist/service/prompts.d.ts +12 -2
- package/dist/service/prompts.js +20 -4
- package/dist/service/scaleStepper.d.ts +8 -0
- package/dist/service/scaleStepper.js +29 -0
- package/dist/service/toggleChecklist.d.ts +9 -0
- package/dist/service/toggleChecklist.js +37 -0
- package/dist/types/harness.d.ts +28 -2
- package/dist/types/index.d.ts +19 -4
- package/dist/types/observability.d.ts +3 -0
- package/package.json +56 -8
- package/src/aiApp.test.ts +15 -0
- package/src/aiApp.ts +9 -1
- package/src/harness/commit.ts +103 -1
- package/src/harness/routes/approvals.ts +2 -2
- package/src/harness/routes/conversations.ts +1 -1
- package/src/harness/routes/tasks.test.ts +34 -0
- package/src/harness/routes/tasks.ts +98 -3
- package/src/harness/runtime.ts +8 -1
- package/src/harness/subagent.test.ts +124 -3
- package/src/harness/subagent.ts +43 -1
- package/src/index.test.ts +4 -0
- package/src/index.ts +5 -1
- package/src/isolated/lazyProjectModel.isolated.ts +25 -0
- package/src/models/project.ts +31 -1
- package/src/observability/local/models/obsTrace.ts +22 -5
- package/src/observability/local/traceStore.test.ts +7 -0
- package/src/observability/local/traceStore.ts +12 -0
- package/src/observability/routes/traces.test.ts +15 -0
- package/src/observability/routes/traces.ts +2 -0
- package/src/routes/gpt.test.ts +90 -1
- package/src/routes/gptActions.test.ts +282 -1
- package/src/routes/gptActions.ts +16 -2
- package/src/routes/projects.ts +4 -4
- package/src/service/agentBlocks.test.ts +95 -0
- package/src/service/agentBlocks.ts +114 -0
- package/src/service/chatTurn.ts +44 -15
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +34 -7
- package/src/service/scaleStepper.test.ts +105 -0
- package/src/service/scaleStepper.ts +29 -0
- package/src/service/toggleChecklist.test.ts +126 -0
- package/src/service/toggleChecklist.ts +44 -0
- package/src/types/harness.ts +31 -2
- package/src/types/index.ts +22 -9
- package/src/types/observability.ts +3 -0
|
@@ -15,7 +15,12 @@ import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
|
15
15
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
16
16
|
import {generateToStream, withGenerateStreaming} from "../tests/generateStream";
|
|
17
17
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
18
|
-
import type {
|
|
18
|
+
import type {
|
|
19
|
+
HarnessAgentDefinition,
|
|
20
|
+
HarnessPromptInstructions,
|
|
21
|
+
HarnessPromptRef,
|
|
22
|
+
HarnessTestHooks,
|
|
23
|
+
} from "../types/harness";
|
|
19
24
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
20
25
|
import {
|
|
21
26
|
AGENT_TURN_TASK_NAME,
|
|
@@ -211,12 +216,14 @@ const parentTask = ({
|
|
|
211
216
|
instructions,
|
|
212
217
|
name = "test.parent",
|
|
213
218
|
output,
|
|
219
|
+
prompts,
|
|
214
220
|
}: {
|
|
215
221
|
agent: HarnessAgentDefinition;
|
|
216
222
|
inputs?: unknown[];
|
|
217
|
-
instructions?: string;
|
|
223
|
+
instructions?: HarnessPromptInstructions | string;
|
|
218
224
|
name?: string;
|
|
219
225
|
output?: z.ZodType;
|
|
226
|
+
prompts?: HarnessPromptRef[];
|
|
220
227
|
}) =>
|
|
221
228
|
defineTask<Record<string, never>, Record<string, never>, {results: unknown[]}>({
|
|
222
229
|
initial: () => ({phase: "summarize"}),
|
|
@@ -227,7 +234,7 @@ const parentTask = ({
|
|
|
227
234
|
run: async (_task, rt) => {
|
|
228
235
|
const results: unknown[] = [];
|
|
229
236
|
for (const input of inputs) {
|
|
230
|
-
results.push(await rt.runAgent(agent, {input, instructions, output}));
|
|
237
|
+
results.push(await rt.runAgent(agent, {input, instructions, output, prompts}));
|
|
231
238
|
}
|
|
232
239
|
await rt.commit({terminal: {result: {results}, status: "completed"}});
|
|
233
240
|
},
|
|
@@ -304,6 +311,120 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
304
311
|
);
|
|
305
312
|
});
|
|
306
313
|
|
|
314
|
+
it("records the task's trace scope, tags and prompt versions, and the subagent's prompts", async () => {
|
|
315
|
+
const agent = summarizerWith();
|
|
316
|
+
const parent = parentTask({
|
|
317
|
+
agent,
|
|
318
|
+
inputs: [{chart: "a"}, {chart: "b"}],
|
|
319
|
+
instructions: {
|
|
320
|
+
body: "Registry says be brief.",
|
|
321
|
+
label: "production",
|
|
322
|
+
name: "summary",
|
|
323
|
+
version: 3,
|
|
324
|
+
},
|
|
325
|
+
prompts: [{name: "style", version: 1}],
|
|
326
|
+
});
|
|
327
|
+
const {calls, model} = scriptedModel([{text: "A."}, {text: "B."}]);
|
|
328
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
329
|
+
|
|
330
|
+
const created = await harness.createTask(
|
|
331
|
+
parent,
|
|
332
|
+
{},
|
|
333
|
+
{
|
|
334
|
+
prompts: [{name: "plan", version: 2}],
|
|
335
|
+
trace: {scope: "workspace-1", tags: ["nightly", "nightly", "beta"]},
|
|
336
|
+
}
|
|
337
|
+
);
|
|
338
|
+
const done = await harness.waitForTask(created._id, {timeout: {seconds: 10}});
|
|
339
|
+
|
|
340
|
+
expect(done.status).toBe("completed");
|
|
341
|
+
const trace = await TraceModel.findExactlyOne({_id: done.traceId});
|
|
342
|
+
expect(trace.scope).toBe("workspace-1");
|
|
343
|
+
expect(plain(trace.tags)).toEqual(["nightly", "beta"]);
|
|
344
|
+
// Both runAgent calls used the same versions; each is recorded once.
|
|
345
|
+
expect(plain(trace.prompts)).toEqual([
|
|
346
|
+
{name: "plan", version: 2},
|
|
347
|
+
{label: "production", name: "summary", version: 3},
|
|
348
|
+
{name: "style", version: 1},
|
|
349
|
+
]);
|
|
350
|
+
const conversations = await ConversationModel.find({});
|
|
351
|
+
expect(conversations.map(({agent: snapshot}) => snapshot.instructions)).toEqual([
|
|
352
|
+
"Registry says be brief.",
|
|
353
|
+
"Registry says be brief.",
|
|
354
|
+
]);
|
|
355
|
+
expect(plain(calls[0]?.prompt[0])).toEqual({
|
|
356
|
+
content: "Registry says be brief.",
|
|
357
|
+
role: "system",
|
|
358
|
+
});
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
it("rejects a blank trace scope and a prompt ref without a version", async () => {
|
|
362
|
+
const agent = summarizerWith();
|
|
363
|
+
const parent = parentTask({agent});
|
|
364
|
+
const {model} = scriptedModel([]);
|
|
365
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
366
|
+
|
|
367
|
+
await expect(harness.createTask(parent, {}, {trace: {scope: " "}})).rejects.toThrow(
|
|
368
|
+
harnessErrorMatching("invalidRequest", "trace.scope must be a non-empty string")
|
|
369
|
+
);
|
|
370
|
+
await expect(
|
|
371
|
+
harness.createTask(parent, {}, {prompts: [{name: "plan"} as HarnessPromptRef]})
|
|
372
|
+
).rejects.toThrow(
|
|
373
|
+
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" needs a positive integer version')
|
|
374
|
+
);
|
|
375
|
+
await expect(
|
|
376
|
+
harness.createTask(parent, {}, {prompts: [{label: 3, name: "plan", version: 1} as never]})
|
|
377
|
+
).rejects.toThrow(
|
|
378
|
+
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" label must be a string')
|
|
379
|
+
);
|
|
380
|
+
await expect(harness.createTask(parent, {}, {trace: {tags: ["ok", " "]}})).rejects.toThrow(
|
|
381
|
+
harnessErrorMatching("invalidRequest", "trace.tags must be an array of non-empty strings")
|
|
382
|
+
);
|
|
383
|
+
expect(await TaskModel.countDocuments({})).toBe(0);
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
it("records a repeated createTask prompt ref once", async () => {
|
|
387
|
+
const agent = summarizerWith();
|
|
388
|
+
const parent = parentTask({agent, inputs: []});
|
|
389
|
+
const {model} = scriptedModel([]);
|
|
390
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
391
|
+
|
|
392
|
+
const plan = {name: "plan", version: 2};
|
|
393
|
+
const created = await harness.createTask(parent, {}, {prompts: [plan, {...plan}]});
|
|
394
|
+
|
|
395
|
+
const trace = await TraceModel.findExactlyOne({_id: created.traceId});
|
|
396
|
+
expect(plain(trace.prompts)).toEqual([plan]);
|
|
397
|
+
});
|
|
398
|
+
|
|
399
|
+
it("fails the caller at once for an instructions prompt without a body or a bad prompt ref", async () => {
|
|
400
|
+
const agent = summarizerWith();
|
|
401
|
+
const noBody = parentTask({
|
|
402
|
+
agent,
|
|
403
|
+
instructions: {name: "summary", version: 1} as HarnessPromptInstructions,
|
|
404
|
+
name: "test.noBody",
|
|
405
|
+
});
|
|
406
|
+
const badRef = parentTask({
|
|
407
|
+
agent,
|
|
408
|
+
name: "test.badRef",
|
|
409
|
+
prompts: [{name: "style", version: 0}],
|
|
410
|
+
});
|
|
411
|
+
const {calls, model} = scriptedModel([]);
|
|
412
|
+
const {harness} = await openProcess({model, registry: [agent, noBody, badRef]});
|
|
413
|
+
|
|
414
|
+
const first = await harness.createTask(noBody, {});
|
|
415
|
+
const second = await harness.createTask(badRef, {});
|
|
416
|
+
const failedNoBody = await harness.waitForTask(first._id, {timeout: {seconds: 10}});
|
|
417
|
+
const failedBadRef = await harness.waitForTask(second._id, {timeout: {seconds: 10}});
|
|
418
|
+
|
|
419
|
+
expect(failedNoBody.status).toBe("failed");
|
|
420
|
+
expect(failedNoBody.outcome?.error).toContain("instructions prompt needs a string body");
|
|
421
|
+
expect(failedBadRef.status).toBe("failed");
|
|
422
|
+
expect(failedBadRef.outcome?.error).toContain(
|
|
423
|
+
'Prompt ref "style" needs a positive integer version'
|
|
424
|
+
);
|
|
425
|
+
expect(calls).toHaveLength(0);
|
|
426
|
+
});
|
|
427
|
+
|
|
307
428
|
it("nests the subagent's AGENT span under the caller and its LLM and TOOL spans under it", async () => {
|
|
308
429
|
const lookup = defineTool({
|
|
309
430
|
description: "Look up a chart",
|
package/src/harness/subagent.ts
CHANGED
|
@@ -7,14 +7,23 @@ import type {
|
|
|
7
7
|
HarnessChildOutcome,
|
|
8
8
|
HarnessExtensionDefinition,
|
|
9
9
|
HarnessLeaseSettings,
|
|
10
|
+
HarnessPromptInstructions,
|
|
11
|
+
HarnessPromptRef,
|
|
10
12
|
HarnessTaskDefinition,
|
|
11
13
|
HarnessTaskDocument,
|
|
12
14
|
HarnessTurnResult,
|
|
13
15
|
} from "../types/harness";
|
|
14
16
|
import {HARNESS_CONVERSATION_STATUSES, HARNESS_MESSAGE_ROLES} from "../types/harness";
|
|
15
17
|
import type {AgentTurnInput} from "./agentLoop";
|
|
16
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
addTracePrompts,
|
|
20
|
+
createChildTaskRecords,
|
|
21
|
+
type HarnessModels,
|
|
22
|
+
tracePromptRefs,
|
|
23
|
+
} from "./commit";
|
|
17
24
|
import {conversationAgentSnapshot} from "./conversation";
|
|
25
|
+
import {HarnessDefinitionError} from "./definitionError";
|
|
26
|
+
import {errorMessage} from "./errors";
|
|
18
27
|
import {insertMessages} from "./events";
|
|
19
28
|
|
|
20
29
|
/** A subagent run by `rt.runAgent` did not produce a usable answer. */
|
|
@@ -56,6 +65,35 @@ export const subagentContent = (input: unknown): string | undefined => {
|
|
|
56
65
|
return JSON.stringify(input);
|
|
57
66
|
};
|
|
58
67
|
|
|
68
|
+
/**
|
|
69
|
+
* Split `rt.runAgent`'s instructions into the text the agent runs with and the prompt
|
|
70
|
+
* versions to record on the trace. A registry prompt passed as `instructions` is used by
|
|
71
|
+
* its `body` and recorded alongside `prompts`.
|
|
72
|
+
*/
|
|
73
|
+
export const subagentPrompts = ({
|
|
74
|
+
instructions,
|
|
75
|
+
prompts,
|
|
76
|
+
where,
|
|
77
|
+
}: {
|
|
78
|
+
instructions?: HarnessPromptInstructions | string;
|
|
79
|
+
prompts?: HarnessPromptRef[];
|
|
80
|
+
where: string;
|
|
81
|
+
}): {instructions?: string; prompts: HarnessPromptRef[]} => {
|
|
82
|
+
const prompt =
|
|
83
|
+
instructions !== undefined && typeof instructions !== "string" ? instructions : undefined;
|
|
84
|
+
if (prompt && typeof prompt.body !== "string") {
|
|
85
|
+
throw new HarnessDefinitionError(`${where} instructions prompt needs a string body`);
|
|
86
|
+
}
|
|
87
|
+
try {
|
|
88
|
+
return {
|
|
89
|
+
instructions: prompt ? prompt.body : (instructions as string | undefined),
|
|
90
|
+
prompts: tracePromptRefs(prompt ? [prompt, ...(prompts ?? [])] : prompts),
|
|
91
|
+
};
|
|
92
|
+
} catch (error: unknown) {
|
|
93
|
+
throw new HarnessDefinitionError(`${where}: ${errorMessage(error)}`);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
59
97
|
/**
|
|
60
98
|
* Serialize `schema` as JSON Schema with the AI SDK's own converter, so the turn requests
|
|
61
99
|
* exactly the structure `Output.object` would. Stored as a string: JSON Schema keys such
|
|
@@ -81,6 +119,7 @@ export const startSubagentTurn = async ({
|
|
|
81
119
|
models,
|
|
82
120
|
outputSchema,
|
|
83
121
|
parent,
|
|
122
|
+
prompts = [],
|
|
84
123
|
turn,
|
|
85
124
|
}: {
|
|
86
125
|
agent: HarnessAgentDefinition;
|
|
@@ -93,6 +132,8 @@ export const startSubagentTurn = async ({
|
|
|
93
132
|
models: HarnessModels;
|
|
94
133
|
outputSchema?: string;
|
|
95
134
|
parent: HarnessTaskDocument;
|
|
135
|
+
/** Prompt versions recorded on the parent's trace with the turn. */
|
|
136
|
+
prompts?: HarnessPromptRef[];
|
|
96
137
|
turn: HarnessTaskDefinition;
|
|
97
138
|
}): Promise<{conversationId: string; turnTask: HarnessTaskDocument}> => {
|
|
98
139
|
const key = `agent:${callIndex}`;
|
|
@@ -113,6 +154,7 @@ export const startSubagentTurn = async ({
|
|
|
113
154
|
name: agent.name,
|
|
114
155
|
},
|
|
115
156
|
writes: async ({session, task}) => {
|
|
157
|
+
await addTracePrompts({models, refs: prompts, session, traceId: parent.traceId});
|
|
116
158
|
await models.conversation.create(
|
|
117
159
|
[
|
|
118
160
|
{
|
package/src/index.test.ts
CHANGED
|
@@ -23,6 +23,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
23
23
|
"createPrompt",
|
|
24
24
|
"createTelemetryConfig",
|
|
25
25
|
"createVertexProvider",
|
|
26
|
+
"findAgentBlock",
|
|
26
27
|
"getCached",
|
|
27
28
|
"getMCPTools",
|
|
28
29
|
"getLangfuseClient",
|
|
@@ -44,6 +45,7 @@ describe("@terreno/ai public exports", () => {
|
|
|
44
45
|
"getObservabilityApp",
|
|
45
46
|
"resetObservabilityApp",
|
|
46
47
|
"resolveObservabilityControl",
|
|
48
|
+
"runBufferedChatTurn",
|
|
47
49
|
"validateObservabilityConfig",
|
|
48
50
|
"setCached",
|
|
49
51
|
"shutdownLangfuseClient",
|
|
@@ -78,7 +80,9 @@ describe("@terreno/ai public exports", () => {
|
|
|
78
80
|
"Output",
|
|
79
81
|
"REMIX_PROMPT",
|
|
80
82
|
"TERRENO_ASKS_SYSTEM_PROMPT",
|
|
83
|
+
"scaleStepperHostAction",
|
|
81
84
|
"TITLE_GENERATION_PROMPT",
|
|
85
|
+
"toggleChecklistHostAction",
|
|
82
86
|
"TRANSLATION_PROMPT",
|
|
83
87
|
"TemperaturePresets",
|
|
84
88
|
] as const;
|
package/src/index.ts
CHANGED
|
@@ -40,7 +40,7 @@ export {AIDataset} from "./models/aiDataset";
|
|
|
40
40
|
export {AIRequest} from "./models/aiRequest";
|
|
41
41
|
export {FileAttachment} from "./models/fileAttachment";
|
|
42
42
|
export {GptHistory} from "./models/gptHistory";
|
|
43
|
-
export {Project} from "./models/project";
|
|
43
|
+
export {getProjectModel, Project} from "./models/project";
|
|
44
44
|
export {AI_OBSERVABILITY_GROUP, observabilityAdminScreens} from "./observability/adminScreens";
|
|
45
45
|
export {LocalDatasetStore} from "./observability/local/datasetStore";
|
|
46
46
|
export {LocalEvaluatorStore} from "./observability/local/evaluatorStore";
|
|
@@ -103,10 +103,12 @@ export {addGptRoutes} from "./routes/gpt";
|
|
|
103
103
|
export {addGptHistoryRoutes} from "./routes/gptHistories";
|
|
104
104
|
export {addMcpRoutes} from "./routes/mcp";
|
|
105
105
|
export {addProjectRoutes} from "./routes/projects";
|
|
106
|
+
export {findAgentBlock} from "./service/agentBlocks";
|
|
106
107
|
export type {RegisteredDataset} from "./service/aiDatasets";
|
|
107
108
|
export {configureAiDatasets, registerAiDataset} from "./service/aiDatasets";
|
|
108
109
|
export {AIService, TemperaturePresets} from "./service/aiService";
|
|
109
110
|
export {createAskTools} from "./service/asks";
|
|
111
|
+
export {runBufferedChatTurn} from "./service/chatTurn";
|
|
110
112
|
export {FileStorageService} from "./service/fileStorage";
|
|
111
113
|
export type {ListGeminiApiModelsOptions} from "./service/gemini";
|
|
112
114
|
export {
|
|
@@ -131,6 +133,8 @@ export {
|
|
|
131
133
|
TITLE_GENERATION_PROMPT,
|
|
132
134
|
TRANSLATION_PROMPT,
|
|
133
135
|
} from "./service/prompts";
|
|
136
|
+
export {scaleStepperHostAction} from "./service/scaleStepper";
|
|
137
|
+
export {toggleChecklistHostAction} from "./service/toggleChecklist";
|
|
134
138
|
export type {
|
|
135
139
|
CreateVertexProviderOptions,
|
|
136
140
|
ListEnabledVertexModelsOptions,
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
describe("@terreno/ai root entry", () => {
|
|
5
|
+
it("does not register the Project model until it is used", async () => {
|
|
6
|
+
const ai = await import("../index");
|
|
7
|
+
expect(mongoose.models.Project).toBeUndefined();
|
|
8
|
+
|
|
9
|
+
expect(ai.Project.modelName).toBe("Project");
|
|
10
|
+
expect(mongoose.models.Project).toBe(ai.getProjectModel());
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it("forwards statics, construction and instanceof through the deprecated Project export", async () => {
|
|
14
|
+
const {Project} = await import("../index");
|
|
15
|
+
const userId = new mongoose.Types.ObjectId();
|
|
16
|
+
|
|
17
|
+
const project = new Project({name: "Roadmap", userId});
|
|
18
|
+
expect(project instanceof Project).toBe(true);
|
|
19
|
+
await project.save();
|
|
20
|
+
|
|
21
|
+
const found = await Project.findById(project._id);
|
|
22
|
+
expect(found?.name).toBe("Roadmap");
|
|
23
|
+
expect(await Project.countDocuments({userId})).toBe(1);
|
|
24
|
+
});
|
|
25
|
+
});
|
package/src/models/project.ts
CHANGED
|
@@ -51,4 +51,34 @@ projectSchema.virtual("ownerId").get(function () {
|
|
|
51
51
|
return this.userId;
|
|
52
52
|
});
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
let projectModel: ProjectModel | undefined;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The AI chat `Project` model, registered on first use rather than on import, so an app
|
|
58
|
+
* that imports `@terreno/ai` can own a model named `Project` as long as it does not also
|
|
59
|
+
* use AI projects.
|
|
60
|
+
*/
|
|
61
|
+
export const getProjectModel = (): ProjectModel => {
|
|
62
|
+
projectModel ??= mongoose.model<ProjectDocument, ProjectModel>("Project", projectSchema);
|
|
63
|
+
return projectModel;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Lazy stand-in for the `Project` model: every access registers it via `getProjectModel`.
|
|
68
|
+
* Statics, `new Project(...)` and `instanceof Project` work, but the proxy is not the model
|
|
69
|
+
* itself (`Project !== mongoose.models.Project`, `Object.keys(Project)` is empty, and reading
|
|
70
|
+
* `Project.prototype` throws).
|
|
71
|
+
* @deprecated Use `getProjectModel()`.
|
|
72
|
+
*/
|
|
73
|
+
// A class target keeps the proxy constructible, like a mongoose Model.
|
|
74
|
+
export const Project = new Proxy(class LazyProject {} as unknown as ProjectModel, {
|
|
75
|
+
construct: (_target, args) => Reflect.construct(getProjectModel(), args),
|
|
76
|
+
get: (_target, property) => {
|
|
77
|
+
const model = getProjectModel();
|
|
78
|
+
const value = Reflect.get(model, property, model);
|
|
79
|
+
return typeof value === "function" ? value.bind(model) : value;
|
|
80
|
+
},
|
|
81
|
+
getPrototypeOf: () => Reflect.getPrototypeOf(getProjectModel()),
|
|
82
|
+
has: (_target, property) => Reflect.has(getProjectModel(), property),
|
|
83
|
+
set: (_target, property, value) => Reflect.set(getProjectModel(), property, value),
|
|
84
|
+
});
|
|
@@ -22,13 +22,21 @@ const obsTraceSchema = new mongoose.Schema<ObsTraceDocument, ObsTraceModel>(
|
|
|
22
22
|
default: [],
|
|
23
23
|
description: "Prompt versions used during this trace",
|
|
24
24
|
type: [
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
// No subdocument _id, so `$addToSet` dedupes repeated refs from replayed phases.
|
|
26
|
+
new mongoose.Schema(
|
|
27
|
+
{
|
|
28
|
+
label: {description: "Label resolved at call time", type: String},
|
|
29
|
+
name: {description: "Prompt name", required: true, type: String},
|
|
30
|
+
version: {description: "Prompt version number", required: true, type: Number},
|
|
31
|
+
},
|
|
32
|
+
{_id: false}
|
|
33
|
+
),
|
|
30
34
|
],
|
|
31
35
|
},
|
|
36
|
+
scope: {
|
|
37
|
+
description: "App-defined scope (tenant, workspace) for metering and filtering traces",
|
|
38
|
+
type: String,
|
|
39
|
+
},
|
|
32
40
|
sensitive: {
|
|
33
41
|
default: false,
|
|
34
42
|
description: "When true, admin I/O disclosures start collapsed",
|
|
@@ -42,6 +50,11 @@ const obsTraceSchema = new mongoose.Schema<ObsTraceDocument, ObsTraceModel>(
|
|
|
42
50
|
required: true,
|
|
43
51
|
type: String,
|
|
44
52
|
},
|
|
53
|
+
tags: {
|
|
54
|
+
default: [],
|
|
55
|
+
description: "App-defined labels for grouping traces",
|
|
56
|
+
type: [String],
|
|
57
|
+
},
|
|
45
58
|
usage: {
|
|
46
59
|
description: "Aggregated token and cost usage for the trace",
|
|
47
60
|
type: mongoose.Schema.Types.Mixed,
|
|
@@ -81,6 +94,10 @@ const traceIndexSpecs: Array<Array<[string, -1 | 1]>> = [
|
|
|
81
94
|
["flaggedForDataset", 1],
|
|
82
95
|
["created", -1],
|
|
83
96
|
],
|
|
97
|
+
[
|
|
98
|
+
["scope", 1],
|
|
99
|
+
["created", -1],
|
|
100
|
+
],
|
|
84
101
|
];
|
|
85
102
|
for (const spec of traceIndexSpecs) {
|
|
86
103
|
obsTraceSchema.index(Object.fromEntries(spec));
|
|
@@ -159,6 +159,13 @@ describe("LocalTraceStore", () => {
|
|
|
159
159
|
const bySession = await store.list({sessionId: "sess-b"});
|
|
160
160
|
expect(bySession.data.map((row) => row.id)).toEqual([newer.id]);
|
|
161
161
|
|
|
162
|
+
// Harness tasks set scope and tags on the trace they create.
|
|
163
|
+
await registerObsTrace().updateOne({_id: newer.id}, {scope: "ws-1", tags: ["beta"]});
|
|
164
|
+
const byScope = await store.list({scope: "ws-1"});
|
|
165
|
+
expect(byScope.data.map((row) => ({id: row.id, scope: row.scope, tags: row.tags}))).toEqual([
|
|
166
|
+
{id: newer.id, scope: "ws-1", tags: ["beta"]},
|
|
167
|
+
]);
|
|
168
|
+
|
|
162
169
|
const bySensitive = await store.list({sensitive: true});
|
|
163
170
|
expect(bySensitive.data.map((row) => row.id)).toEqual([newer.id]);
|
|
164
171
|
|
|
@@ -15,6 +15,7 @@ export interface TraceListQuery {
|
|
|
15
15
|
page?: number;
|
|
16
16
|
prompt?: string;
|
|
17
17
|
promptVersion?: number;
|
|
18
|
+
scope?: string;
|
|
18
19
|
sensitive?: boolean;
|
|
19
20
|
sessionId?: string;
|
|
20
21
|
status?: "error" | "ok";
|
|
@@ -49,12 +50,14 @@ export interface TraceDetail {
|
|
|
49
50
|
name: string;
|
|
50
51
|
output?: unknown;
|
|
51
52
|
prompts: TraceRecord["prompts"];
|
|
53
|
+
scope?: string;
|
|
52
54
|
scores: ScoreRecord[];
|
|
53
55
|
sensitive: boolean;
|
|
54
56
|
sessionId?: string;
|
|
55
57
|
spans: TraceSpanNode[];
|
|
56
58
|
startedAt: string;
|
|
57
59
|
status: "error" | "ok";
|
|
60
|
+
tags: string[];
|
|
58
61
|
usage?: TraceRecord["usage"];
|
|
59
62
|
userId?: string;
|
|
60
63
|
}
|
|
@@ -66,12 +69,14 @@ export interface TraceListItem {
|
|
|
66
69
|
id: string;
|
|
67
70
|
name: string;
|
|
68
71
|
prompts: TraceRecord["prompts"];
|
|
72
|
+
scope?: string;
|
|
69
73
|
scoreCount: number;
|
|
70
74
|
sensitive: boolean;
|
|
71
75
|
sessionId?: string;
|
|
72
76
|
spanCount: number;
|
|
73
77
|
startedAt: string;
|
|
74
78
|
status: "error" | "ok";
|
|
79
|
+
tags: string[];
|
|
75
80
|
usage?: TraceRecord["usage"];
|
|
76
81
|
userId?: string;
|
|
77
82
|
}
|
|
@@ -221,6 +226,9 @@ export class LocalTraceStore {
|
|
|
221
226
|
if (query.sessionId) {
|
|
222
227
|
filter.sessionId = query.sessionId;
|
|
223
228
|
}
|
|
229
|
+
if (query.scope) {
|
|
230
|
+
filter.scope = query.scope;
|
|
231
|
+
}
|
|
224
232
|
if (query.promptVersion !== undefined && !query.prompt) {
|
|
225
233
|
throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
|
|
226
234
|
}
|
|
@@ -324,10 +332,12 @@ export class LocalTraceStore {
|
|
|
324
332
|
flaggedForDataset: boolean;
|
|
325
333
|
name: string;
|
|
326
334
|
prompts: TraceRecord["prompts"];
|
|
335
|
+
scope?: string;
|
|
327
336
|
sensitive: boolean;
|
|
328
337
|
sessionId?: string;
|
|
329
338
|
startedAt: Date;
|
|
330
339
|
status: "error" | "ok";
|
|
340
|
+
tags?: string[];
|
|
331
341
|
usage?: TraceRecord["usage"];
|
|
332
342
|
userId?: mongoose.Types.ObjectId;
|
|
333
343
|
},
|
|
@@ -340,12 +350,14 @@ export class LocalTraceStore {
|
|
|
340
350
|
id: String(row._id),
|
|
341
351
|
name: row.name,
|
|
342
352
|
prompts: row.prompts,
|
|
353
|
+
scope: row.scope,
|
|
343
354
|
scoreCount: counts.scoreCount,
|
|
344
355
|
sensitive: row.sensitive,
|
|
345
356
|
sessionId: row.sessionId,
|
|
346
357
|
spanCount: counts.spanCount,
|
|
347
358
|
startedAt: toIso(row.startedAt) ?? "",
|
|
348
359
|
status: row.status,
|
|
360
|
+
tags: row.tags ?? [],
|
|
349
361
|
usage: usageWithoutUndefinedCost(row.usage),
|
|
350
362
|
userId: row.userId ? String(row.userId) : undefined,
|
|
351
363
|
};
|
|
@@ -94,6 +94,21 @@ describe("observability trace routes", () => {
|
|
|
94
94
|
expect(listed.body.total).toBe(1);
|
|
95
95
|
});
|
|
96
96
|
|
|
97
|
+
it("filters the trace list by scope", async () => {
|
|
98
|
+
const startedAt = DateTime.utc().toISO() ?? "";
|
|
99
|
+
const base = {name: "scoped", prompts: [], sensitive: false, spans: [], startedAt};
|
|
100
|
+
const inScope = await store.exportTrace({...base, id: "a", status: "ok"});
|
|
101
|
+
await store.exportTrace({...base, id: "b", status: "ok"});
|
|
102
|
+
await registerObsTrace().updateOne({_id: inScope.id}, {scope: "ws-1"});
|
|
103
|
+
|
|
104
|
+
const agent = await authAsUser(app, "admin");
|
|
105
|
+
const listed = await agent.get("/ai/observability/traces?scope=ws-1");
|
|
106
|
+
expect(listed.status).toBe(200);
|
|
107
|
+
expect(
|
|
108
|
+
listed.body.data.map((row: {id: string; scope?: string}) => [row.id, row.scope])
|
|
109
|
+
).toEqual([[inScope.id, "ws-1"]]);
|
|
110
|
+
});
|
|
111
|
+
|
|
97
112
|
it("forbids non-admins from listing traces", async () => {
|
|
98
113
|
const agent = await authAsUser(app, "notAdmin");
|
|
99
114
|
const res = await agent.get("/ai/observability/traces");
|
|
@@ -71,6 +71,7 @@ export const addObservabilityTraceRoutes = (
|
|
|
71
71
|
.withSummary("List observability traces")
|
|
72
72
|
.withQueryParameter("prompt", {type: "string"}, {required: false})
|
|
73
73
|
.withQueryParameter("promptVersion", {type: "number"}, {required: false})
|
|
74
|
+
.withQueryParameter("scope", {type: "string"}, {required: false})
|
|
74
75
|
.withResponse(200, {data: {type: "array"}})
|
|
75
76
|
.build()
|
|
76
77
|
),
|
|
@@ -84,6 +85,7 @@ export const addObservabilityTraceRoutes = (
|
|
|
84
85
|
page: parsePositiveInt(req.query.page, 1),
|
|
85
86
|
prompt,
|
|
86
87
|
promptVersion: parsePromptVersion(req.query.promptVersion, prompt),
|
|
88
|
+
scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
|
|
87
89
|
sensitive: parseBoolean(req.query.sensitive),
|
|
88
90
|
sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
|
|
89
91
|
status:
|
package/src/routes/gpt.test.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {afterEach, beforeAll, describe, expect, it, spyOn} from "bun:test";
|
|
2
2
|
import {TerrenoApp} from "@terreno/api";
|
|
3
|
-
import {askPromptSection} from "@terreno/blocks";
|
|
3
|
+
import {askPromptSection, blocksPromptSection} from "@terreno/blocks";
|
|
4
4
|
import {jsonSchema, type LanguageModel, type Tool, tool} from "ai";
|
|
5
5
|
import express from "express";
|
|
6
6
|
|
|
@@ -2933,6 +2933,31 @@ blocks:
|
|
|
2933
2933
|
name: export_csv
|
|
2934
2934
|
`;
|
|
2935
2935
|
|
|
2936
|
+
const stepperReply = (callbackName: string): string => `v: 1
|
|
2937
|
+
blocks:
|
|
2938
|
+
- type: stepper
|
|
2939
|
+
id: guests
|
|
2940
|
+
label: Number of people
|
|
2941
|
+
value: 5
|
|
2942
|
+
min: 1
|
|
2943
|
+
max: 20
|
|
2944
|
+
callback:
|
|
2945
|
+
name: ${callbackName}
|
|
2946
|
+
`;
|
|
2947
|
+
|
|
2948
|
+
const STEPPER_HOST_ACTIONS = {
|
|
2949
|
+
approve: {},
|
|
2950
|
+
scaleStepper: {handles: "stepper" as const},
|
|
2951
|
+
};
|
|
2952
|
+
|
|
2953
|
+
const blocksEventOf = (
|
|
2954
|
+
events: Record<string, unknown>[]
|
|
2955
|
+
): {errors: {code: string; path: string}[]; ok: boolean} =>
|
|
2956
|
+
events.find((event) => "blocks" in event)?.blocks as {
|
|
2957
|
+
errors: {code: string; path: string}[];
|
|
2958
|
+
ok: boolean;
|
|
2959
|
+
};
|
|
2960
|
+
|
|
2936
2961
|
describe("/gpt/prompt uiBlocks", () => {
|
|
2937
2962
|
beforeAll(async () => {
|
|
2938
2963
|
await ensureTestUsers();
|
|
@@ -3276,4 +3301,68 @@ blocks:
|
|
|
3276
3301
|
expect(allowedBlocks.ok).toBe(true);
|
|
3277
3302
|
expect(systemPromptOf(modelCall(allowedModel, 0))).toContain("cdn.example.com");
|
|
3278
3303
|
});
|
|
3304
|
+
|
|
3305
|
+
it("offers the stepper by default when a host action handles steppers", async () => {
|
|
3306
|
+
const model = createScriptedModel({steps: [textStep(stepperReply("scaleStepper"))]});
|
|
3307
|
+
const agent = await authAsUser(
|
|
3308
|
+
buildApp({model, uiBlocks: {hostActions: STEPPER_HOST_ACTIONS}}),
|
|
3309
|
+
"notAdmin"
|
|
3310
|
+
);
|
|
3311
|
+
|
|
3312
|
+
const {events} = await streamPrompt(agent, {prompt: USER_PROMPT});
|
|
3313
|
+
|
|
3314
|
+
const system = String(systemPromptOf(modelCall(model, 0)));
|
|
3315
|
+
expect(system).toContain("details, stepper, checklist, gallery, list.");
|
|
3316
|
+
expect(system).toContain("A stepper callback name must be one of: scaleStepper.");
|
|
3317
|
+
expect(blocksEventOf(events)).toEqual({errors: [], ok: true, warnings: []});
|
|
3318
|
+
});
|
|
3319
|
+
|
|
3320
|
+
it("rejects a stepper whose callback is a host action that does not handle steppers", async () => {
|
|
3321
|
+
const model = createScriptedModel({steps: [textStep(stepperReply("approve"))]});
|
|
3322
|
+
const agent = await authAsUser(
|
|
3323
|
+
buildApp({model, uiBlocks: {hostActions: STEPPER_HOST_ACTIONS}}),
|
|
3324
|
+
"notAdmin"
|
|
3325
|
+
);
|
|
3326
|
+
|
|
3327
|
+
const {events} = await streamPrompt(agent, {prompt: USER_PROMPT});
|
|
3328
|
+
|
|
3329
|
+
const blocks = blocksEventOf(events);
|
|
3330
|
+
expect(blocks.ok).toBe(false);
|
|
3331
|
+
expect(blocks.errors.map((error) => ({code: error.code, path: error.path}))).toEqual([
|
|
3332
|
+
{code: "UNKNOWN_HOST_ACTION", path: "blocks[0].callback.name"},
|
|
3333
|
+
]);
|
|
3334
|
+
const history = await loadHistory(await onlyHistoryId());
|
|
3335
|
+
expect(String(rowsOf(history).find((row) => row.type === "assistant")?.text)).toContain(
|
|
3336
|
+
"UNKNOWN_HOST_ACTION"
|
|
3337
|
+
);
|
|
3338
|
+
});
|
|
3339
|
+
|
|
3340
|
+
it("never mentions the stepper without a stepper action", async () => {
|
|
3341
|
+
const model = createScriptedModel({steps: [textStep(VALID_BLOCKS)]});
|
|
3342
|
+
const agent = await authAsUser(
|
|
3343
|
+
buildApp({model, uiBlocks: {hostActions: {approve: {}}}}),
|
|
3344
|
+
"notAdmin"
|
|
3345
|
+
);
|
|
3346
|
+
|
|
3347
|
+
await streamPrompt(agent, {prompt: USER_PROMPT});
|
|
3348
|
+
|
|
3349
|
+
expect(String(systemPromptOf(modelCall(model, 0)))).not.toContain("stepper");
|
|
3350
|
+
});
|
|
3351
|
+
|
|
3352
|
+
it("keeps the pre-rich-blocks prompt with richBlocks false but still accepts a stepper", async () => {
|
|
3353
|
+
const model = createScriptedModel({steps: [textStep(stepperReply("scaleStepper"))]});
|
|
3354
|
+
const agent = await authAsUser(
|
|
3355
|
+
buildApp({model, uiBlocks: {hostActions: STEPPER_HOST_ACTIONS, richBlocks: false}}),
|
|
3356
|
+
"notAdmin"
|
|
3357
|
+
);
|
|
3358
|
+
|
|
3359
|
+
const {events} = await streamPrompt(agent, {prompt: USER_PROMPT});
|
|
3360
|
+
|
|
3361
|
+
const system = String(systemPromptOf(modelCall(model, 0)));
|
|
3362
|
+
expect(system).not.toContain("stepper");
|
|
3363
|
+
expect(system).toContain(
|
|
3364
|
+
blocksPromptSection({hostActions: ["approve", "scaleStepper"], richBlocks: false})
|
|
3365
|
+
);
|
|
3366
|
+
expect(blocksEventOf(events)).toEqual({errors: [], ok: true, warnings: []});
|
|
3367
|
+
});
|
|
3279
3368
|
});
|