@terreno/ai 57.11.0 → 57.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aiApp.d.ts +5 -0
- package/dist/aiApp.js +4 -2
- package/dist/harness/commit.d.ts +17 -1
- package/dist/harness/commit.js +86 -2
- package/dist/harness/routes/approvals.js +2 -2
- package/dist/harness/routes/conversations.js +1 -1
- package/dist/harness/routes/tasks.js +85 -3
- package/dist/harness/runtime.js +7 -1
- package/dist/harness/subagent.d.ts +17 -2
- package/dist/harness/subagent.js +26 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +3 -2
- package/dist/models/project.d.ts +13 -0
- package/dist/models/project.js +32 -2
- package/dist/observability/local/models/obsTrace.js +16 -2
- package/dist/observability/local/traceStore.d.ts +5 -0
- package/dist/observability/local/traceStore.js +5 -0
- package/dist/observability/routes/traces.js +2 -0
- package/dist/routes/projects.js +3 -3
- package/dist/service/chatTurn.js +1 -1
- package/dist/types/harness.d.ts +28 -2
- package/dist/types/observability.d.ts +3 -0
- package/package.json +56 -8
- package/src/aiApp.test.ts +15 -0
- package/src/aiApp.ts +9 -1
- package/src/harness/commit.ts +103 -1
- package/src/harness/routes/approvals.ts +2 -2
- package/src/harness/routes/conversations.ts +1 -1
- package/src/harness/routes/tasks.test.ts +34 -0
- package/src/harness/routes/tasks.ts +98 -3
- package/src/harness/runtime.ts +8 -1
- package/src/harness/subagent.test.ts +124 -3
- package/src/harness/subagent.ts +43 -1
- package/src/index.ts +1 -1
- package/src/isolated/lazyProjectModel.isolated.ts +25 -0
- package/src/models/project.ts +31 -1
- package/src/observability/local/models/obsTrace.ts +22 -5
- package/src/observability/local/traceStore.test.ts +7 -0
- package/src/observability/local/traceStore.ts +12 -0
- package/src/observability/routes/traces.test.ts +15 -0
- package/src/observability/routes/traces.ts +2 -0
- package/src/routes/projects.ts +4 -4
- package/src/service/chatTurn.ts +2 -2
- package/src/types/harness.ts +31 -2
- package/src/types/observability.ts +3 -0
package/src/harness/runtime.ts
CHANGED
|
@@ -63,6 +63,7 @@ import {
|
|
|
63
63
|
serializeOutputSchema,
|
|
64
64
|
startSubagentTurn,
|
|
65
65
|
subagentContent,
|
|
66
|
+
subagentPrompts,
|
|
66
67
|
subagentResult,
|
|
67
68
|
} from "./subagent";
|
|
68
69
|
import {HarnessSuspendSignal} from "./suspend";
|
|
@@ -632,16 +633,22 @@ const runPhase = async ({
|
|
|
632
633
|
);
|
|
633
634
|
}
|
|
634
635
|
}
|
|
636
|
+
const {instructions, prompts} = subagentPrompts({
|
|
637
|
+
instructions: options.instructions,
|
|
638
|
+
prompts: options.prompts,
|
|
639
|
+
where: `${definition.key}: rt.runAgent`,
|
|
640
|
+
});
|
|
635
641
|
const {conversationId, turnTask} = await startSubagentTurn({
|
|
636
642
|
agent,
|
|
637
643
|
callIndex,
|
|
638
644
|
content,
|
|
639
645
|
extensions: engine.extensions,
|
|
640
|
-
instructions
|
|
646
|
+
instructions,
|
|
641
647
|
lease,
|
|
642
648
|
models,
|
|
643
649
|
outputSchema,
|
|
644
650
|
parent: task,
|
|
651
|
+
prompts,
|
|
645
652
|
turn,
|
|
646
653
|
});
|
|
647
654
|
engine.wake();
|
|
@@ -15,7 +15,12 @@ import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
|
15
15
|
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
16
16
|
import {generateToStream, withGenerateStreaming} from "../tests/generateStream";
|
|
17
17
|
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
18
|
-
import type {
|
|
18
|
+
import type {
|
|
19
|
+
HarnessAgentDefinition,
|
|
20
|
+
HarnessPromptInstructions,
|
|
21
|
+
HarnessPromptRef,
|
|
22
|
+
HarnessTestHooks,
|
|
23
|
+
} from "../types/harness";
|
|
19
24
|
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
20
25
|
import {
|
|
21
26
|
AGENT_TURN_TASK_NAME,
|
|
@@ -211,12 +216,14 @@ const parentTask = ({
|
|
|
211
216
|
instructions,
|
|
212
217
|
name = "test.parent",
|
|
213
218
|
output,
|
|
219
|
+
prompts,
|
|
214
220
|
}: {
|
|
215
221
|
agent: HarnessAgentDefinition;
|
|
216
222
|
inputs?: unknown[];
|
|
217
|
-
instructions?: string;
|
|
223
|
+
instructions?: HarnessPromptInstructions | string;
|
|
218
224
|
name?: string;
|
|
219
225
|
output?: z.ZodType;
|
|
226
|
+
prompts?: HarnessPromptRef[];
|
|
220
227
|
}) =>
|
|
221
228
|
defineTask<Record<string, never>, Record<string, never>, {results: unknown[]}>({
|
|
222
229
|
initial: () => ({phase: "summarize"}),
|
|
@@ -227,7 +234,7 @@ const parentTask = ({
|
|
|
227
234
|
run: async (_task, rt) => {
|
|
228
235
|
const results: unknown[] = [];
|
|
229
236
|
for (const input of inputs) {
|
|
230
|
-
results.push(await rt.runAgent(agent, {input, instructions, output}));
|
|
237
|
+
results.push(await rt.runAgent(agent, {input, instructions, output, prompts}));
|
|
231
238
|
}
|
|
232
239
|
await rt.commit({terminal: {result: {results}, status: "completed"}});
|
|
233
240
|
},
|
|
@@ -304,6 +311,120 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
304
311
|
);
|
|
305
312
|
});
|
|
306
313
|
|
|
314
|
+
it("records the task's trace scope, tags and prompt versions, and the subagent's prompts", async () => {
|
|
315
|
+
const agent = summarizerWith();
|
|
316
|
+
const parent = parentTask({
|
|
317
|
+
agent,
|
|
318
|
+
inputs: [{chart: "a"}, {chart: "b"}],
|
|
319
|
+
instructions: {
|
|
320
|
+
body: "Registry says be brief.",
|
|
321
|
+
label: "production",
|
|
322
|
+
name: "summary",
|
|
323
|
+
version: 3,
|
|
324
|
+
},
|
|
325
|
+
prompts: [{name: "style", version: 1}],
|
|
326
|
+
});
|
|
327
|
+
const {calls, model} = scriptedModel([{text: "A."}, {text: "B."}]);
|
|
328
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
329
|
+
|
|
330
|
+
const created = await harness.createTask(
|
|
331
|
+
parent,
|
|
332
|
+
{},
|
|
333
|
+
{
|
|
334
|
+
prompts: [{name: "plan", version: 2}],
|
|
335
|
+
trace: {scope: "workspace-1", tags: ["nightly", "nightly", "beta"]},
|
|
336
|
+
}
|
|
337
|
+
);
|
|
338
|
+
const done = await harness.waitForTask(created._id, {timeout: {seconds: 10}});
|
|
339
|
+
|
|
340
|
+
expect(done.status).toBe("completed");
|
|
341
|
+
const trace = await TraceModel.findExactlyOne({_id: done.traceId});
|
|
342
|
+
expect(trace.scope).toBe("workspace-1");
|
|
343
|
+
expect(plain(trace.tags)).toEqual(["nightly", "beta"]);
|
|
344
|
+
// Both runAgent calls used the same versions; each is recorded once.
|
|
345
|
+
expect(plain(trace.prompts)).toEqual([
|
|
346
|
+
{name: "plan", version: 2},
|
|
347
|
+
{label: "production", name: "summary", version: 3},
|
|
348
|
+
{name: "style", version: 1},
|
|
349
|
+
]);
|
|
350
|
+
const conversations = await ConversationModel.find({});
|
|
351
|
+
expect(conversations.map(({agent: snapshot}) => snapshot.instructions)).toEqual([
|
|
352
|
+
"Registry says be brief.",
|
|
353
|
+
"Registry says be brief.",
|
|
354
|
+
]);
|
|
355
|
+
expect(plain(calls[0]?.prompt[0])).toEqual({
|
|
356
|
+
content: "Registry says be brief.",
|
|
357
|
+
role: "system",
|
|
358
|
+
});
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
it("rejects a blank trace scope and a prompt ref without a version", async () => {
|
|
362
|
+
const agent = summarizerWith();
|
|
363
|
+
const parent = parentTask({agent});
|
|
364
|
+
const {model} = scriptedModel([]);
|
|
365
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
366
|
+
|
|
367
|
+
await expect(harness.createTask(parent, {}, {trace: {scope: " "}})).rejects.toThrow(
|
|
368
|
+
harnessErrorMatching("invalidRequest", "trace.scope must be a non-empty string")
|
|
369
|
+
);
|
|
370
|
+
await expect(
|
|
371
|
+
harness.createTask(parent, {}, {prompts: [{name: "plan"} as HarnessPromptRef]})
|
|
372
|
+
).rejects.toThrow(
|
|
373
|
+
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" needs a positive integer version')
|
|
374
|
+
);
|
|
375
|
+
await expect(
|
|
376
|
+
harness.createTask(parent, {}, {prompts: [{label: 3, name: "plan", version: 1} as never]})
|
|
377
|
+
).rejects.toThrow(
|
|
378
|
+
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" label must be a string')
|
|
379
|
+
);
|
|
380
|
+
await expect(harness.createTask(parent, {}, {trace: {tags: ["ok", " "]}})).rejects.toThrow(
|
|
381
|
+
harnessErrorMatching("invalidRequest", "trace.tags must be an array of non-empty strings")
|
|
382
|
+
);
|
|
383
|
+
expect(await TaskModel.countDocuments({})).toBe(0);
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
it("records a repeated createTask prompt ref once", async () => {
|
|
387
|
+
const agent = summarizerWith();
|
|
388
|
+
const parent = parentTask({agent, inputs: []});
|
|
389
|
+
const {model} = scriptedModel([]);
|
|
390
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
391
|
+
|
|
392
|
+
const plan = {name: "plan", version: 2};
|
|
393
|
+
const created = await harness.createTask(parent, {}, {prompts: [plan, {...plan}]});
|
|
394
|
+
|
|
395
|
+
const trace = await TraceModel.findExactlyOne({_id: created.traceId});
|
|
396
|
+
expect(plain(trace.prompts)).toEqual([plan]);
|
|
397
|
+
});
|
|
398
|
+
|
|
399
|
+
it("fails the caller at once for an instructions prompt without a body or a bad prompt ref", async () => {
|
|
400
|
+
const agent = summarizerWith();
|
|
401
|
+
const noBody = parentTask({
|
|
402
|
+
agent,
|
|
403
|
+
instructions: {name: "summary", version: 1} as HarnessPromptInstructions,
|
|
404
|
+
name: "test.noBody",
|
|
405
|
+
});
|
|
406
|
+
const badRef = parentTask({
|
|
407
|
+
agent,
|
|
408
|
+
name: "test.badRef",
|
|
409
|
+
prompts: [{name: "style", version: 0}],
|
|
410
|
+
});
|
|
411
|
+
const {calls, model} = scriptedModel([]);
|
|
412
|
+
const {harness} = await openProcess({model, registry: [agent, noBody, badRef]});
|
|
413
|
+
|
|
414
|
+
const first = await harness.createTask(noBody, {});
|
|
415
|
+
const second = await harness.createTask(badRef, {});
|
|
416
|
+
const failedNoBody = await harness.waitForTask(first._id, {timeout: {seconds: 10}});
|
|
417
|
+
const failedBadRef = await harness.waitForTask(second._id, {timeout: {seconds: 10}});
|
|
418
|
+
|
|
419
|
+
expect(failedNoBody.status).toBe("failed");
|
|
420
|
+
expect(failedNoBody.outcome?.error).toContain("instructions prompt needs a string body");
|
|
421
|
+
expect(failedBadRef.status).toBe("failed");
|
|
422
|
+
expect(failedBadRef.outcome?.error).toContain(
|
|
423
|
+
'Prompt ref "style" needs a positive integer version'
|
|
424
|
+
);
|
|
425
|
+
expect(calls).toHaveLength(0);
|
|
426
|
+
});
|
|
427
|
+
|
|
307
428
|
it("nests the subagent's AGENT span under the caller and its LLM and TOOL spans under it", async () => {
|
|
308
429
|
const lookup = defineTool({
|
|
309
430
|
description: "Look up a chart",
|
package/src/harness/subagent.ts
CHANGED
|
@@ -7,14 +7,23 @@ import type {
|
|
|
7
7
|
HarnessChildOutcome,
|
|
8
8
|
HarnessExtensionDefinition,
|
|
9
9
|
HarnessLeaseSettings,
|
|
10
|
+
HarnessPromptInstructions,
|
|
11
|
+
HarnessPromptRef,
|
|
10
12
|
HarnessTaskDefinition,
|
|
11
13
|
HarnessTaskDocument,
|
|
12
14
|
HarnessTurnResult,
|
|
13
15
|
} from "../types/harness";
|
|
14
16
|
import {HARNESS_CONVERSATION_STATUSES, HARNESS_MESSAGE_ROLES} from "../types/harness";
|
|
15
17
|
import type {AgentTurnInput} from "./agentLoop";
|
|
16
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
addTracePrompts,
|
|
20
|
+
createChildTaskRecords,
|
|
21
|
+
type HarnessModels,
|
|
22
|
+
tracePromptRefs,
|
|
23
|
+
} from "./commit";
|
|
17
24
|
import {conversationAgentSnapshot} from "./conversation";
|
|
25
|
+
import {HarnessDefinitionError} from "./definitionError";
|
|
26
|
+
import {errorMessage} from "./errors";
|
|
18
27
|
import {insertMessages} from "./events";
|
|
19
28
|
|
|
20
29
|
/** A subagent run by `rt.runAgent` did not produce a usable answer. */
|
|
@@ -56,6 +65,35 @@ export const subagentContent = (input: unknown): string | undefined => {
|
|
|
56
65
|
return JSON.stringify(input);
|
|
57
66
|
};
|
|
58
67
|
|
|
68
|
+
/**
|
|
69
|
+
* Split `rt.runAgent`'s instructions into the text the agent runs with and the prompt
|
|
70
|
+
* versions to record on the trace. A registry prompt passed as `instructions` is used by
|
|
71
|
+
* its `body` and recorded alongside `prompts`.
|
|
72
|
+
*/
|
|
73
|
+
export const subagentPrompts = ({
|
|
74
|
+
instructions,
|
|
75
|
+
prompts,
|
|
76
|
+
where,
|
|
77
|
+
}: {
|
|
78
|
+
instructions?: HarnessPromptInstructions | string;
|
|
79
|
+
prompts?: HarnessPromptRef[];
|
|
80
|
+
where: string;
|
|
81
|
+
}): {instructions?: string; prompts: HarnessPromptRef[]} => {
|
|
82
|
+
const prompt =
|
|
83
|
+
instructions !== undefined && typeof instructions !== "string" ? instructions : undefined;
|
|
84
|
+
if (prompt && typeof prompt.body !== "string") {
|
|
85
|
+
throw new HarnessDefinitionError(`${where} instructions prompt needs a string body`);
|
|
86
|
+
}
|
|
87
|
+
try {
|
|
88
|
+
return {
|
|
89
|
+
instructions: prompt ? prompt.body : (instructions as string | undefined),
|
|
90
|
+
prompts: tracePromptRefs(prompt ? [prompt, ...(prompts ?? [])] : prompts),
|
|
91
|
+
};
|
|
92
|
+
} catch (error: unknown) {
|
|
93
|
+
throw new HarnessDefinitionError(`${where}: ${errorMessage(error)}`);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
59
97
|
/**
|
|
60
98
|
* Serialize `schema` as JSON Schema with the AI SDK's own converter, so the turn requests
|
|
61
99
|
* exactly the structure `Output.object` would. Stored as a string: JSON Schema keys such
|
|
@@ -81,6 +119,7 @@ export const startSubagentTurn = async ({
|
|
|
81
119
|
models,
|
|
82
120
|
outputSchema,
|
|
83
121
|
parent,
|
|
122
|
+
prompts = [],
|
|
84
123
|
turn,
|
|
85
124
|
}: {
|
|
86
125
|
agent: HarnessAgentDefinition;
|
|
@@ -93,6 +132,8 @@ export const startSubagentTurn = async ({
|
|
|
93
132
|
models: HarnessModels;
|
|
94
133
|
outputSchema?: string;
|
|
95
134
|
parent: HarnessTaskDocument;
|
|
135
|
+
/** Prompt versions recorded on the parent's trace with the turn. */
|
|
136
|
+
prompts?: HarnessPromptRef[];
|
|
96
137
|
turn: HarnessTaskDefinition;
|
|
97
138
|
}): Promise<{conversationId: string; turnTask: HarnessTaskDocument}> => {
|
|
98
139
|
const key = `agent:${callIndex}`;
|
|
@@ -113,6 +154,7 @@ export const startSubagentTurn = async ({
|
|
|
113
154
|
name: agent.name,
|
|
114
155
|
},
|
|
115
156
|
writes: async ({session, task}) => {
|
|
157
|
+
await addTracePrompts({models, refs: prompts, session, traceId: parent.traceId});
|
|
116
158
|
await models.conversation.create(
|
|
117
159
|
[
|
|
118
160
|
{
|
package/src/index.ts
CHANGED
|
@@ -40,7 +40,7 @@ export {AIDataset} from "./models/aiDataset";
|
|
|
40
40
|
export {AIRequest} from "./models/aiRequest";
|
|
41
41
|
export {FileAttachment} from "./models/fileAttachment";
|
|
42
42
|
export {GptHistory} from "./models/gptHistory";
|
|
43
|
-
export {Project} from "./models/project";
|
|
43
|
+
export {getProjectModel, Project} from "./models/project";
|
|
44
44
|
export {AI_OBSERVABILITY_GROUP, observabilityAdminScreens} from "./observability/adminScreens";
|
|
45
45
|
export {LocalDatasetStore} from "./observability/local/datasetStore";
|
|
46
46
|
export {LocalEvaluatorStore} from "./observability/local/evaluatorStore";
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
describe("@terreno/ai root entry", () => {
|
|
5
|
+
it("does not register the Project model until it is used", async () => {
|
|
6
|
+
const ai = await import("../index");
|
|
7
|
+
expect(mongoose.models.Project).toBeUndefined();
|
|
8
|
+
|
|
9
|
+
expect(ai.Project.modelName).toBe("Project");
|
|
10
|
+
expect(mongoose.models.Project).toBe(ai.getProjectModel());
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it("forwards statics, construction and instanceof through the deprecated Project export", async () => {
|
|
14
|
+
const {Project} = await import("../index");
|
|
15
|
+
const userId = new mongoose.Types.ObjectId();
|
|
16
|
+
|
|
17
|
+
const project = new Project({name: "Roadmap", userId});
|
|
18
|
+
expect(project instanceof Project).toBe(true);
|
|
19
|
+
await project.save();
|
|
20
|
+
|
|
21
|
+
const found = await Project.findById(project._id);
|
|
22
|
+
expect(found?.name).toBe("Roadmap");
|
|
23
|
+
expect(await Project.countDocuments({userId})).toBe(1);
|
|
24
|
+
});
|
|
25
|
+
});
|
package/src/models/project.ts
CHANGED
|
@@ -51,4 +51,34 @@ projectSchema.virtual("ownerId").get(function () {
|
|
|
51
51
|
return this.userId;
|
|
52
52
|
});
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
let projectModel: ProjectModel | undefined;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The AI chat `Project` model, registered on first use rather than on import, so an app
|
|
58
|
+
* that imports `@terreno/ai` can own a model named `Project` as long as it does not also
|
|
59
|
+
* use AI projects.
|
|
60
|
+
*/
|
|
61
|
+
export const getProjectModel = (): ProjectModel => {
|
|
62
|
+
projectModel ??= mongoose.model<ProjectDocument, ProjectModel>("Project", projectSchema);
|
|
63
|
+
return projectModel;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Lazy stand-in for the `Project` model: every access registers it via `getProjectModel`.
|
|
68
|
+
* Statics, `new Project(...)` and `instanceof Project` work, but the proxy is not the model
|
|
69
|
+
* itself (`Project !== mongoose.models.Project`, `Object.keys(Project)` is empty, and reading
|
|
70
|
+
* `Project.prototype` throws).
|
|
71
|
+
* @deprecated Use `getProjectModel()`.
|
|
72
|
+
*/
|
|
73
|
+
// A class target keeps the proxy constructible, like a mongoose Model.
|
|
74
|
+
export const Project = new Proxy(class LazyProject {} as unknown as ProjectModel, {
|
|
75
|
+
construct: (_target, args) => Reflect.construct(getProjectModel(), args),
|
|
76
|
+
get: (_target, property) => {
|
|
77
|
+
const model = getProjectModel();
|
|
78
|
+
const value = Reflect.get(model, property, model);
|
|
79
|
+
return typeof value === "function" ? value.bind(model) : value;
|
|
80
|
+
},
|
|
81
|
+
getPrototypeOf: () => Reflect.getPrototypeOf(getProjectModel()),
|
|
82
|
+
has: (_target, property) => Reflect.has(getProjectModel(), property),
|
|
83
|
+
set: (_target, property, value) => Reflect.set(getProjectModel(), property, value),
|
|
84
|
+
});
|
|
@@ -22,13 +22,21 @@ const obsTraceSchema = new mongoose.Schema<ObsTraceDocument, ObsTraceModel>(
|
|
|
22
22
|
default: [],
|
|
23
23
|
description: "Prompt versions used during this trace",
|
|
24
24
|
type: [
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
// No subdocument _id, so `$addToSet` dedupes repeated refs from replayed phases.
|
|
26
|
+
new mongoose.Schema(
|
|
27
|
+
{
|
|
28
|
+
label: {description: "Label resolved at call time", type: String},
|
|
29
|
+
name: {description: "Prompt name", required: true, type: String},
|
|
30
|
+
version: {description: "Prompt version number", required: true, type: Number},
|
|
31
|
+
},
|
|
32
|
+
{_id: false}
|
|
33
|
+
),
|
|
30
34
|
],
|
|
31
35
|
},
|
|
36
|
+
scope: {
|
|
37
|
+
description: "App-defined scope (tenant, workspace) for metering and filtering traces",
|
|
38
|
+
type: String,
|
|
39
|
+
},
|
|
32
40
|
sensitive: {
|
|
33
41
|
default: false,
|
|
34
42
|
description: "When true, admin I/O disclosures start collapsed",
|
|
@@ -42,6 +50,11 @@ const obsTraceSchema = new mongoose.Schema<ObsTraceDocument, ObsTraceModel>(
|
|
|
42
50
|
required: true,
|
|
43
51
|
type: String,
|
|
44
52
|
},
|
|
53
|
+
tags: {
|
|
54
|
+
default: [],
|
|
55
|
+
description: "App-defined labels for grouping traces",
|
|
56
|
+
type: [String],
|
|
57
|
+
},
|
|
45
58
|
usage: {
|
|
46
59
|
description: "Aggregated token and cost usage for the trace",
|
|
47
60
|
type: mongoose.Schema.Types.Mixed,
|
|
@@ -81,6 +94,10 @@ const traceIndexSpecs: Array<Array<[string, -1 | 1]>> = [
|
|
|
81
94
|
["flaggedForDataset", 1],
|
|
82
95
|
["created", -1],
|
|
83
96
|
],
|
|
97
|
+
[
|
|
98
|
+
["scope", 1],
|
|
99
|
+
["created", -1],
|
|
100
|
+
],
|
|
84
101
|
];
|
|
85
102
|
for (const spec of traceIndexSpecs) {
|
|
86
103
|
obsTraceSchema.index(Object.fromEntries(spec));
|
|
@@ -159,6 +159,13 @@ describe("LocalTraceStore", () => {
|
|
|
159
159
|
const bySession = await store.list({sessionId: "sess-b"});
|
|
160
160
|
expect(bySession.data.map((row) => row.id)).toEqual([newer.id]);
|
|
161
161
|
|
|
162
|
+
// Harness tasks set scope and tags on the trace they create.
|
|
163
|
+
await registerObsTrace().updateOne({_id: newer.id}, {scope: "ws-1", tags: ["beta"]});
|
|
164
|
+
const byScope = await store.list({scope: "ws-1"});
|
|
165
|
+
expect(byScope.data.map((row) => ({id: row.id, scope: row.scope, tags: row.tags}))).toEqual([
|
|
166
|
+
{id: newer.id, scope: "ws-1", tags: ["beta"]},
|
|
167
|
+
]);
|
|
168
|
+
|
|
162
169
|
const bySensitive = await store.list({sensitive: true});
|
|
163
170
|
expect(bySensitive.data.map((row) => row.id)).toEqual([newer.id]);
|
|
164
171
|
|
|
@@ -15,6 +15,7 @@ export interface TraceListQuery {
|
|
|
15
15
|
page?: number;
|
|
16
16
|
prompt?: string;
|
|
17
17
|
promptVersion?: number;
|
|
18
|
+
scope?: string;
|
|
18
19
|
sensitive?: boolean;
|
|
19
20
|
sessionId?: string;
|
|
20
21
|
status?: "error" | "ok";
|
|
@@ -49,12 +50,14 @@ export interface TraceDetail {
|
|
|
49
50
|
name: string;
|
|
50
51
|
output?: unknown;
|
|
51
52
|
prompts: TraceRecord["prompts"];
|
|
53
|
+
scope?: string;
|
|
52
54
|
scores: ScoreRecord[];
|
|
53
55
|
sensitive: boolean;
|
|
54
56
|
sessionId?: string;
|
|
55
57
|
spans: TraceSpanNode[];
|
|
56
58
|
startedAt: string;
|
|
57
59
|
status: "error" | "ok";
|
|
60
|
+
tags: string[];
|
|
58
61
|
usage?: TraceRecord["usage"];
|
|
59
62
|
userId?: string;
|
|
60
63
|
}
|
|
@@ -66,12 +69,14 @@ export interface TraceListItem {
|
|
|
66
69
|
id: string;
|
|
67
70
|
name: string;
|
|
68
71
|
prompts: TraceRecord["prompts"];
|
|
72
|
+
scope?: string;
|
|
69
73
|
scoreCount: number;
|
|
70
74
|
sensitive: boolean;
|
|
71
75
|
sessionId?: string;
|
|
72
76
|
spanCount: number;
|
|
73
77
|
startedAt: string;
|
|
74
78
|
status: "error" | "ok";
|
|
79
|
+
tags: string[];
|
|
75
80
|
usage?: TraceRecord["usage"];
|
|
76
81
|
userId?: string;
|
|
77
82
|
}
|
|
@@ -221,6 +226,9 @@ export class LocalTraceStore {
|
|
|
221
226
|
if (query.sessionId) {
|
|
222
227
|
filter.sessionId = query.sessionId;
|
|
223
228
|
}
|
|
229
|
+
if (query.scope) {
|
|
230
|
+
filter.scope = query.scope;
|
|
231
|
+
}
|
|
224
232
|
if (query.promptVersion !== undefined && !query.prompt) {
|
|
225
233
|
throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
|
|
226
234
|
}
|
|
@@ -324,10 +332,12 @@ export class LocalTraceStore {
|
|
|
324
332
|
flaggedForDataset: boolean;
|
|
325
333
|
name: string;
|
|
326
334
|
prompts: TraceRecord["prompts"];
|
|
335
|
+
scope?: string;
|
|
327
336
|
sensitive: boolean;
|
|
328
337
|
sessionId?: string;
|
|
329
338
|
startedAt: Date;
|
|
330
339
|
status: "error" | "ok";
|
|
340
|
+
tags?: string[];
|
|
331
341
|
usage?: TraceRecord["usage"];
|
|
332
342
|
userId?: mongoose.Types.ObjectId;
|
|
333
343
|
},
|
|
@@ -340,12 +350,14 @@ export class LocalTraceStore {
|
|
|
340
350
|
id: String(row._id),
|
|
341
351
|
name: row.name,
|
|
342
352
|
prompts: row.prompts,
|
|
353
|
+
scope: row.scope,
|
|
343
354
|
scoreCount: counts.scoreCount,
|
|
344
355
|
sensitive: row.sensitive,
|
|
345
356
|
sessionId: row.sessionId,
|
|
346
357
|
spanCount: counts.spanCount,
|
|
347
358
|
startedAt: toIso(row.startedAt) ?? "",
|
|
348
359
|
status: row.status,
|
|
360
|
+
tags: row.tags ?? [],
|
|
349
361
|
usage: usageWithoutUndefinedCost(row.usage),
|
|
350
362
|
userId: row.userId ? String(row.userId) : undefined,
|
|
351
363
|
};
|
|
@@ -94,6 +94,21 @@ describe("observability trace routes", () => {
|
|
|
94
94
|
expect(listed.body.total).toBe(1);
|
|
95
95
|
});
|
|
96
96
|
|
|
97
|
+
it("filters the trace list by scope", async () => {
|
|
98
|
+
const startedAt = DateTime.utc().toISO() ?? "";
|
|
99
|
+
const base = {name: "scoped", prompts: [], sensitive: false, spans: [], startedAt};
|
|
100
|
+
const inScope = await store.exportTrace({...base, id: "a", status: "ok"});
|
|
101
|
+
await store.exportTrace({...base, id: "b", status: "ok"});
|
|
102
|
+
await registerObsTrace().updateOne({_id: inScope.id}, {scope: "ws-1"});
|
|
103
|
+
|
|
104
|
+
const agent = await authAsUser(app, "admin");
|
|
105
|
+
const listed = await agent.get("/ai/observability/traces?scope=ws-1");
|
|
106
|
+
expect(listed.status).toBe(200);
|
|
107
|
+
expect(
|
|
108
|
+
listed.body.data.map((row: {id: string; scope?: string}) => [row.id, row.scope])
|
|
109
|
+
).toEqual([[inScope.id, "ws-1"]]);
|
|
110
|
+
});
|
|
111
|
+
|
|
97
112
|
it("forbids non-admins from listing traces", async () => {
|
|
98
113
|
const agent = await authAsUser(app, "notAdmin");
|
|
99
114
|
const res = await agent.get("/ai/observability/traces");
|
|
@@ -71,6 +71,7 @@ export const addObservabilityTraceRoutes = (
|
|
|
71
71
|
.withSummary("List observability traces")
|
|
72
72
|
.withQueryParameter("prompt", {type: "string"}, {required: false})
|
|
73
73
|
.withQueryParameter("promptVersion", {type: "number"}, {required: false})
|
|
74
|
+
.withQueryParameter("scope", {type: "string"}, {required: false})
|
|
74
75
|
.withResponse(200, {data: {type: "array"}})
|
|
75
76
|
.build()
|
|
76
77
|
),
|
|
@@ -84,6 +85,7 @@ export const addObservabilityTraceRoutes = (
|
|
|
84
85
|
page: parsePositiveInt(req.query.page, 1),
|
|
85
86
|
prompt,
|
|
86
87
|
promptVersion: parsePromptVersion(req.query.promptVersion, prompt),
|
|
88
|
+
scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
|
|
87
89
|
sensitive: parseBoolean(req.query.sensitive),
|
|
88
90
|
sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
|
|
89
91
|
status:
|
package/src/routes/projects.ts
CHANGED
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
import type express from "express";
|
|
10
10
|
import type mongoose from "mongoose";
|
|
11
11
|
|
|
12
|
-
import {
|
|
12
|
+
import {getProjectModel} from "../models/project";
|
|
13
13
|
import type {ProjectDocument} from "../types";
|
|
14
14
|
|
|
15
15
|
export const addProjectRoutes = (
|
|
@@ -41,7 +41,7 @@ export const addProjectRoutes = (
|
|
|
41
41
|
throw new APIError({status: 400, title: "text is required"});
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
const project = await
|
|
44
|
+
const project = await getProjectModel().findById(id);
|
|
45
45
|
if (!project) {
|
|
46
46
|
throw new APIError({status: 404, title: "Project not found"});
|
|
47
47
|
}
|
|
@@ -73,7 +73,7 @@ export const addProjectRoutes = (
|
|
|
73
73
|
const {id, memoryId} = req.params;
|
|
74
74
|
const userId = (req.user as {_id?: mongoose.Types.ObjectId} | undefined)?._id;
|
|
75
75
|
|
|
76
|
-
const project = await
|
|
76
|
+
const project = await getProjectModel().findById(id);
|
|
77
77
|
if (!project) {
|
|
78
78
|
throw new APIError({status: 404, title: "Project not found"});
|
|
79
79
|
}
|
|
@@ -95,7 +95,7 @@ export const addProjectRoutes = (
|
|
|
95
95
|
|
|
96
96
|
router.use(
|
|
97
97
|
"/gpt/projects",
|
|
98
|
-
modelRouter(
|
|
98
|
+
modelRouter(getProjectModel(), {
|
|
99
99
|
...options?.openApiOptions,
|
|
100
100
|
permissions: {
|
|
101
101
|
create: [Permissions.IsAuthenticated],
|
package/src/service/chatTurn.ts
CHANGED
|
@@ -27,7 +27,7 @@ import type mongoose from "mongoose";
|
|
|
27
27
|
import {createTelemetryConfig, preparePromptForAI} from "../langfuseVercelAi";
|
|
28
28
|
import {AIRequest} from "../models/aiRequest";
|
|
29
29
|
import {GptHistory} from "../models/gptHistory";
|
|
30
|
-
import {
|
|
30
|
+
import {getProjectModel} from "../models/project";
|
|
31
31
|
import type {SpanRecord} from "../observability/types";
|
|
32
32
|
import type {
|
|
33
33
|
AskOrigin,
|
|
@@ -1120,7 +1120,7 @@ const buildSystemPrompt = async ({
|
|
|
1120
1120
|
let effectiveSystemPrompt = systemPrompt as string | undefined;
|
|
1121
1121
|
if (effectiveProjectId) {
|
|
1122
1122
|
try {
|
|
1123
|
-
const project = await
|
|
1123
|
+
const project = await getProjectModel().findById(effectiveProjectId);
|
|
1124
1124
|
if (project && project.userId.toString() === userId?.toString()) {
|
|
1125
1125
|
const parts: string[] = [];
|
|
1126
1126
|
if (project.systemContext) {
|
package/src/types/harness.ts
CHANGED
|
@@ -4,6 +4,26 @@ import type {Duration, DurationLike} from "luxon";
|
|
|
4
4
|
import type mongoose from "mongoose";
|
|
5
5
|
|
|
6
6
|
import type {ExecutionEnv} from "../harness/executionEnv";
|
|
7
|
+
import type {ObsPromptRef} from "./observability";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
|
|
11
|
+
* from `promptRegistry.get` fits as-is; its `body` is ignored.
|
|
12
|
+
*/
|
|
13
|
+
export type HarnessPromptRef = ObsPromptRef;
|
|
14
|
+
|
|
15
|
+
/** A registry prompt passed as instructions: its `body` is used and its version recorded. */
|
|
16
|
+
export interface HarnessPromptInstructions extends HarnessPromptRef {
|
|
17
|
+
body: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Fields written onto the task's `ObsTrace`, shared by every child task and subagent. */
|
|
21
|
+
export interface HarnessTraceOptions {
|
|
22
|
+
/** App-defined scope (tenant, workspace) for metering; indexed with `created`. */
|
|
23
|
+
scope?: string;
|
|
24
|
+
/** App-defined labels for grouping traces. */
|
|
25
|
+
tags?: string[];
|
|
26
|
+
}
|
|
7
27
|
|
|
8
28
|
/** Every lifecycle status a harness task can hold. */
|
|
9
29
|
export const HARNESS_TASK_STATUSES = {
|
|
@@ -374,8 +394,13 @@ export interface HarnessMemo {
|
|
|
374
394
|
export interface HarnessRunAgentOptions<Result> {
|
|
375
395
|
/** The subagent's user message. A non-string value is sent as JSON. */
|
|
376
396
|
input: unknown;
|
|
377
|
-
/**
|
|
378
|
-
|
|
397
|
+
/**
|
|
398
|
+
* Replaces the agent's instructions for this subagent conversation only. A registry
|
|
399
|
+
* prompt (`promptRegistry.get`) is used by its `body` and recorded on the trace.
|
|
400
|
+
*/
|
|
401
|
+
instructions?: HarnessPromptInstructions | string;
|
|
402
|
+
/** Prompt versions this subagent used, recorded on the task's `ObsTrace.prompts`. */
|
|
403
|
+
prompts?: HarnessPromptRef[];
|
|
379
404
|
/**
|
|
380
405
|
* Structured output schema; defaults to the agent's own `output`. Requested through the
|
|
381
406
|
* AI SDK's `Output.object`, then validated with this schema before it is returned.
|
|
@@ -442,8 +467,12 @@ export interface HarnessAbortOptions {
|
|
|
442
467
|
}
|
|
443
468
|
|
|
444
469
|
export interface HarnessCreateTaskOptions {
|
|
470
|
+
/** Prompt versions the task uses, recorded on its `ObsTrace.prompts`. */
|
|
471
|
+
prompts?: HarnessPromptRef[];
|
|
445
472
|
/** Idempotency key: a second create with the same id returns the existing task. */
|
|
446
473
|
requestId?: string;
|
|
474
|
+
/** Scope and tags for the task's `ObsTrace`; child tasks and subagents share it. */
|
|
475
|
+
trace?: HarnessTraceOptions;
|
|
447
476
|
userId?: mongoose.Types.ObjectId | string;
|
|
448
477
|
}
|
|
449
478
|
|
|
@@ -89,10 +89,13 @@ export interface ObsTraceDocument extends mongoose.Document<mongoose.Types.Objec
|
|
|
89
89
|
name: string;
|
|
90
90
|
output?: unknown;
|
|
91
91
|
prompts: ObsPromptRef[];
|
|
92
|
+
/** App-defined scope (tenant, workspace) set by `harness.createTask({trace: {scope}})`. */
|
|
93
|
+
scope?: string;
|
|
92
94
|
sensitive: boolean;
|
|
93
95
|
sessionId?: string;
|
|
94
96
|
startedAt: Date;
|
|
95
97
|
status: "error" | "ok";
|
|
98
|
+
tags: string[];
|
|
96
99
|
updated: Date;
|
|
97
100
|
usage?: ObsPromptUsage;
|
|
98
101
|
userId?: mongoose.Types.ObjectId;
|