@ema.co/mcp-toolkit 2026.3.25-4 → 2026.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/login.js +1 -1
- package/dist/config/profile.js +1 -1
- package/dist/config/tool-guidance.js +7 -3
- package/dist/knowledge/extractors/openapi-endpoints.js +160 -2
- package/dist/knowledge/guidance-cache.js +4 -3
- package/dist/knowledge/search-client.js +57 -47
- package/dist/knowledge/search-config.js +2 -1
- package/dist/mcp/handlers/config/index.js +124 -8
- package/dist/mcp/handlers/feedback/index.js +32 -0
- package/dist/mcp/handlers/feedback/store.js +4 -0
- package/dist/mcp/handlers/knowledge/confidence-loop.js +10 -5
- package/dist/mcp/handlers/knowledge/index.js +25 -7
- package/dist/mcp/handlers/knowledge/outcome-feedback.js +205 -0
- package/dist/mcp/handlers/knowledge/session-state.js +110 -0
- package/dist/mcp/handlers/workflow/deploy.js +33 -0
- package/dist/mcp/knowledge-guidance-topics.js +25 -1
- package/dist/mcp/knowledge.js +1 -1
- package/dist/mcp/resources-dynamic.js +3 -3
- package/dist/mcp/tools.js +15 -3
- package/dist/sdk/generated/agent-catalog.js +8 -4
- package/dist/sdk/generated/deprecated-actions.js +19 -19
- package/dist/sdk/generated/proto-fields.js +1 -1
- package/dist/sdk/generated/protos/service/agent_qa/v1/agent_qa_pb.js +223 -15
- package/dist/sdk/generated/protos/service/common/v1/common_pb.js +51 -1
- package/dist/sdk/generated/protos/service/conversation_review/v1/conversation_review_pb.js +63 -16
- package/dist/sdk/generated/protos/service/dataingest/v1/dataingest_pb.js +65 -9
- package/dist/sdk/generated/protos/service/eval-tool/v1/evaluation_pb.js +114 -1
- package/dist/sdk/generated/protos/service/eval-tool/v1/testbed_pb.js +1 -1
- package/dist/sdk/generated/protos/service/external_access_service/v1/external_bot_pb.js +5 -1
- package/dist/sdk/generated/protos/service/external_tool_connection/v1/connection_manager_pb.js +114 -35
- package/dist/sdk/generated/protos/service/external_tool_connection/v1/ema_connector_pb.js +120 -0
- package/dist/sdk/generated/protos/service/identity/v1/identity_pb.js +188 -0
- package/dist/sdk/generated/protos/service/llmservice/v1/llmservice_pb.js +41 -2
- package/dist/sdk/generated/protos/service/permissions/permissions_pb.js +327 -24
- package/dist/sdk/generated/protos/service/persona/v1/chatbot_pb.js +41 -3
- package/dist/sdk/generated/protos/service/persona/v1/persona_config_pb.js +141 -89
- package/dist/sdk/generated/protos/service/persona/v1/persona_pb.js +283 -133
- package/dist/sdk/generated/protos/service/persona/v1/shared_widgets/widget_types_pb.js +24 -3
- package/dist/sdk/generated/protos/service/persona/v1/voicebot_widgets/widget_types_pb.js +114 -14
- package/dist/sdk/generated/protos/service/proposal/v1/proposal_pb.js +57 -1
- package/dist/sdk/generated/protos/service/script_executor/v1/script_executor_pb.js +96 -0
- package/dist/sdk/generated/protos/service/transform/v1/transform_pb.js +19 -1
- package/dist/sdk/generated/protos/service/user/v1/user_pb.js +1 -1
- package/dist/sdk/generated/protos/service/voice/v1/external_voice_pb.js +132 -0
- package/dist/sdk/generated/protos/service/voice/v1/voice_pb.js +1 -1
- package/dist/sdk/generated/protos/service/webcrawl/v1/webcrawl_pb.js +7 -2
- package/dist/sdk/generated/protos/service/workflows/v1/action_registry_pb.js +86 -49
- package/dist/sdk/generated/protos/service/workflows/v1/action_runner_pb.js +12 -7
- package/dist/sdk/generated/protos/service/workflows/v1/action_type_pb.js +27 -17
- package/dist/sdk/generated/protos/service/workflows/v1/agentic_search_pb.js +27 -0
- package/dist/sdk/generated/protos/service/workflows/v1/chatbot_pb.js +30 -21
- package/dist/sdk/generated/protos/service/workflows/v1/dashboards_pb.js +35 -13
- package/dist/sdk/generated/protos/service/workflows/v1/embedded_persona_runner_pb.js +6 -1
- package/dist/sdk/generated/protos/service/workflows/v1/external_actions_pb.js +19 -1
- package/dist/sdk/generated/protos/service/workflows/v1/log_type_pb.js +6 -1
- package/dist/sdk/generated/protos/service/workflows/v1/rpc/workflow_rpc_pb.js +71 -14
- package/dist/sdk/generated/protos/service/workflows/v1/values_pb.js +8 -2
- package/dist/sdk/generated/protos/service/workflows/v1/well_known_pb.js +2 -1
- package/dist/sdk/generated/protos/service/workflows/v1/workflow_pb.js +31 -16
- package/dist/sdk/generated/template-fallbacks.js +1 -1
- package/dist/sdk/generated/well-known-types.js +4 -1
- package/package.json +1 -1
|
@@ -7,6 +7,57 @@
|
|
|
7
7
|
import { loadConfig, saveConfig, addProfile, removeProfile, setCurrentProfile, listProfiles, getActiveProfile, slugify, profileName, } from "../../../config/profile.js";
|
|
8
8
|
import { invalidateEnvCache } from "../env/config.js";
|
|
9
9
|
import { tryGcloudAuth } from "../../../auth/gcloud.js";
|
|
10
|
+
/**
|
|
11
|
+
* Derive API URL, app URL, and environment name from a custom app URL.
|
|
12
|
+
* Supports any `*.ema.co` hostname. Returns null for non-ema.co domains.
|
|
13
|
+
*/
|
|
14
|
+
function deriveFromAppUrl(appUrl) {
|
|
15
|
+
try {
|
|
16
|
+
const url = new URL(appUrl.includes("://") ? appUrl : `https://${appUrl}`);
|
|
17
|
+
const hostname = url.hostname;
|
|
18
|
+
// Match *.ema.co pattern
|
|
19
|
+
const match = hostname.match(/^(.+)\.ema\.co$/);
|
|
20
|
+
if (!match)
|
|
21
|
+
return null;
|
|
22
|
+
const subdomain = match[1]; // e.g., "staging.acme", "app", "demo"
|
|
23
|
+
// Well-known frontends map to standard envs
|
|
24
|
+
if (subdomain === "app") {
|
|
25
|
+
return { apiUrl: "https://api.ema.co", appUrl: "https://app.ema.co", envName: "prod" };
|
|
26
|
+
}
|
|
27
|
+
if (["demo", "staging", "dev"].includes(subdomain)) {
|
|
28
|
+
return {
|
|
29
|
+
apiUrl: `https://api.${subdomain}.ema.co`,
|
|
30
|
+
appUrl: `https://${subdomain}.ema.co`,
|
|
31
|
+
envName: subdomain,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
// Custom subdomain: derive API URL by prepending "api."
|
|
35
|
+
return {
|
|
36
|
+
apiUrl: `https://api.${subdomain}.ema.co`,
|
|
37
|
+
appUrl: `https://${subdomain}.ema.co`,
|
|
38
|
+
envName: subdomain,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Probe an API URL to check reachability. Returns true if the endpoint
|
|
47
|
+
* responds (even with 401/403 — means it's reachable but auth-gated).
|
|
48
|
+
*/
|
|
49
|
+
async function probeApiUrl(apiUrl) {
|
|
50
|
+
try {
|
|
51
|
+
const resp = await fetch(`${apiUrl.replace(/\/$/, "")}/api/health`, {
|
|
52
|
+
method: "HEAD",
|
|
53
|
+
signal: AbortSignal.timeout(5000),
|
|
54
|
+
});
|
|
55
|
+
return resp.ok || resp.status === 401 || resp.status === 403;
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
10
61
|
export async function handleConfig(args) {
|
|
11
62
|
const method = args.method;
|
|
12
63
|
switch (method) {
|
|
@@ -161,14 +212,56 @@ async function handleStatus() {
|
|
|
161
212
|
// Login
|
|
162
213
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
163
214
|
async function handleLogin(args) {
|
|
164
|
-
|
|
165
|
-
const
|
|
215
|
+
const customAppUrl = args.app_url;
|
|
216
|
+
const customApiUrl = args.api_url;
|
|
166
217
|
const tenantHint = args.tenant;
|
|
167
218
|
const directToken = args.token;
|
|
219
|
+
// Resolve URLs and env name from custom app_url or standard env
|
|
220
|
+
let envName;
|
|
221
|
+
let resolvedApiUrl;
|
|
222
|
+
let resolvedAppUrl;
|
|
223
|
+
let probeWarning;
|
|
224
|
+
if (customAppUrl) {
|
|
225
|
+
const derived = deriveFromAppUrl(customAppUrl);
|
|
226
|
+
if (derived) {
|
|
227
|
+
envName = args.env || derived.envName;
|
|
228
|
+
resolvedApiUrl = customApiUrl?.replace(/\/$/, "") ?? derived.apiUrl;
|
|
229
|
+
resolvedAppUrl = derived.appUrl;
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
// Non-ema.co domain — derive env from hostname, require api_url if not provided
|
|
233
|
+
const parsed = new URL(customAppUrl.includes("://") ? customAppUrl : `https://${customAppUrl}`);
|
|
234
|
+
envName = args.env || slugify(parsed.hostname.split(".")[0]);
|
|
235
|
+
resolvedAppUrl = `${parsed.protocol}//${parsed.host}`;
|
|
236
|
+
if (customApiUrl) {
|
|
237
|
+
resolvedApiUrl = customApiUrl.replace(/\/$/, "");
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
// Best guess: api.{hostname}
|
|
241
|
+
resolvedApiUrl = `${parsed.protocol}//api.${parsed.host}`;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
// Probe the API URL — warn but don't block if unreachable
|
|
245
|
+
const reachable = await probeApiUrl(resolvedApiUrl);
|
|
246
|
+
if (!reachable) {
|
|
247
|
+
probeWarning =
|
|
248
|
+
`Derived API URL (${resolvedApiUrl}) did not respond. ` +
|
|
249
|
+
`Login will proceed with this URL, but if auth fails, verify the correct API URL ` +
|
|
250
|
+
`and retry with: config(method="login", app_url="${customAppUrl}", api_url="<correct_url>")`;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
else {
|
|
254
|
+
// Standard flow: explicit env > preference default_env > "prod"
|
|
255
|
+
envName = args.env || loadConfig().preferences.default_env || "prod";
|
|
256
|
+
resolvedApiUrl = customApiUrl?.replace(/\/$/, "")
|
|
257
|
+
?? (envName === "prod" ? "https://api.ema.co" : `https://api.${envName}.ema.co`);
|
|
258
|
+
}
|
|
168
259
|
try {
|
|
169
260
|
const { login } = await import("../../../auth/login.js");
|
|
170
261
|
const result = await login({
|
|
171
262
|
environment: envName,
|
|
263
|
+
baseUrl: resolvedApiUrl,
|
|
264
|
+
...(resolvedAppUrl ? { appUrl: resolvedAppUrl } : {}),
|
|
172
265
|
tenantId: tenantHint,
|
|
173
266
|
token: directToken,
|
|
174
267
|
});
|
|
@@ -183,7 +276,8 @@ async function handleLogin(args) {
|
|
|
183
276
|
},
|
|
184
277
|
environment: {
|
|
185
278
|
name: envName,
|
|
186
|
-
url:
|
|
279
|
+
url: resolvedApiUrl,
|
|
280
|
+
...(resolvedAppUrl ? { app_url: resolvedAppUrl } : {}),
|
|
187
281
|
},
|
|
188
282
|
user: { email: result.userEmail ?? "" },
|
|
189
283
|
created_at: new Date().toISOString(),
|
|
@@ -225,6 +319,7 @@ async function handleLogin(args) {
|
|
|
225
319
|
profile_name: profileName(slugify(t.company_name), envName),
|
|
226
320
|
active: t.tenant_id === result.tenantId,
|
|
227
321
|
})),
|
|
322
|
+
...(probeWarning ? { _warning: probeWarning } : {}),
|
|
228
323
|
_tip: "Token stored. All tools will now use this profile automatically.",
|
|
229
324
|
_next_step: `persona(method="list", profile="${name}")`,
|
|
230
325
|
};
|
|
@@ -362,7 +457,7 @@ async function handleUse(args) {
|
|
|
362
457
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
363
458
|
// Set / Get preferences
|
|
364
459
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
365
|
-
const ALLOWED_KEYS = ["debug", "default_env", "resource_source"];
|
|
460
|
+
const ALLOWED_KEYS = ["debug", "default_env", "resource_source", "api_url", "app_url"];
|
|
366
461
|
async function handleSet(args) {
|
|
367
462
|
const key = args.key;
|
|
368
463
|
const value = args.value;
|
|
@@ -372,11 +467,14 @@ async function handleSet(args) {
|
|
|
372
467
|
error: `Unknown preference key: ${key}. Available: ${ALLOWED_KEYS.join(", ")}`,
|
|
373
468
|
};
|
|
374
469
|
}
|
|
375
|
-
// Validate default_env values
|
|
470
|
+
// Validate default_env values — accept well-known envs + any env from existing profiles
|
|
376
471
|
if (key === "default_env") {
|
|
377
|
-
const
|
|
378
|
-
|
|
379
|
-
|
|
472
|
+
const config = loadConfig();
|
|
473
|
+
const wellKnown = ["prod", "staging", "dev", "demo"];
|
|
474
|
+
const profileEnvNames = Object.values(config.profiles).map(p => p.environment.name);
|
|
475
|
+
const allValid = [...new Set([...wellKnown, ...profileEnvNames])];
|
|
476
|
+
if (!allValid.includes(value)) {
|
|
477
|
+
return { error: `Invalid default_env: ${value}. Must be one of: ${allValid.join(", ")}` };
|
|
380
478
|
}
|
|
381
479
|
}
|
|
382
480
|
// Validate resource_source values
|
|
@@ -386,6 +484,24 @@ async function handleSet(args) {
|
|
|
386
484
|
return { error: `Invalid resource_source: ${value}. Must be one of: ${valid.join(", ")}` };
|
|
387
485
|
}
|
|
388
486
|
}
|
|
487
|
+
// api_url and app_url update the current profile's environment URLs
|
|
488
|
+
if (key === "api_url" || key === "app_url") {
|
|
489
|
+
const config = loadConfig();
|
|
490
|
+
const profile = config.profiles[config.current_profile];
|
|
491
|
+
if (!profile) {
|
|
492
|
+
return { error: "No active profile. Log in first.", _tip: 'config(method="login")' };
|
|
493
|
+
}
|
|
494
|
+
const previous = key === "api_url" ? profile.environment.url : profile.environment.app_url;
|
|
495
|
+
if (key === "api_url") {
|
|
496
|
+
profile.environment.url = value;
|
|
497
|
+
}
|
|
498
|
+
else {
|
|
499
|
+
profile.environment.app_url = value;
|
|
500
|
+
}
|
|
501
|
+
saveConfig(config);
|
|
502
|
+
invalidateEnvCache();
|
|
503
|
+
return { set: { key, value, previous, profile: config.current_profile } };
|
|
504
|
+
}
|
|
389
505
|
const config = loadConfig();
|
|
390
506
|
const previous = config.preferences[key];
|
|
391
507
|
config.preferences[key] = value;
|
|
@@ -16,6 +16,9 @@ import { submitFeedback, listFeedback, listTelemetry, analyzeFeedback, rotateLog
|
|
|
16
16
|
import { markProbeResponded } from "./probes.js";
|
|
17
17
|
import { appendToOutbox, flushOutbox, getOutboxStats, readLocalMessages } from "./outbox.js";
|
|
18
18
|
import { isRemoteEnabled } from "./remote-store.js";
|
|
19
|
+
import { writeUserEvent } from "../../../knowledge/search-client.js";
|
|
20
|
+
import { getOrCreateClientId } from "./client-id.js";
|
|
21
|
+
import { getAttributionToken } from "../knowledge/session-state.js";
|
|
19
22
|
import { analyzeGlobal } from "./global-analysis.js";
|
|
20
23
|
import { TOOLKIT_VERSION } from "../env/config.js";
|
|
21
24
|
const VALID_CATEGORIES = ALL_CATEGORIES;
|
|
@@ -141,6 +144,35 @@ async function handleSubmit(args) {
|
|
|
141
144
|
// Best-effort — don't block feedback submission
|
|
142
145
|
}
|
|
143
146
|
}
|
|
147
|
+
// UserEvent emission: fire DE conversion/view-item for positive feedback with knowledge_ref.
|
|
148
|
+
// Independent of confidence loop — no guards, no cooldown. Fire-and-forget.
|
|
149
|
+
if (knowledgeRef) {
|
|
150
|
+
const isSuccess = category === "success";
|
|
151
|
+
const isHighQuality = category === "quality"
|
|
152
|
+
&& (qualityData?.accuracy ?? 0) >= 4
|
|
153
|
+
&& (qualityData?.usefulness ?? 0) >= 4;
|
|
154
|
+
const isInteraction = category === "interaction";
|
|
155
|
+
if (isSuccess || isHighQuality || isInteraction) {
|
|
156
|
+
const conversionType = isSuccess ? "knowledge-success"
|
|
157
|
+
: isHighQuality ? "knowledge-quality-high"
|
|
158
|
+
: undefined; // interaction → view-item, no conversionType
|
|
159
|
+
getOrCreateClientId()
|
|
160
|
+
.then((clientId) => {
|
|
161
|
+
const token = getAttributionToken(knowledgeRef);
|
|
162
|
+
writeUserEvent({
|
|
163
|
+
eventType: conversionType ? "conversion" : "view-item",
|
|
164
|
+
userPseudoId: clientId,
|
|
165
|
+
...(token ? { attributionToken: token } : {}),
|
|
166
|
+
documents: [{
|
|
167
|
+
id: knowledgeRef,
|
|
168
|
+
...(conversionType ? { conversionValue: isSuccess ? 1.0 : 0.8 } : {}),
|
|
169
|
+
}],
|
|
170
|
+
...(conversionType ? { conversionType } : {}),
|
|
171
|
+
}).catch(() => { });
|
|
172
|
+
})
|
|
173
|
+
.catch(() => { });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
144
176
|
return {
|
|
145
177
|
success: true,
|
|
146
178
|
feedback_id: entry.id,
|
|
@@ -12,6 +12,7 @@ import { promises as fs } from "node:fs";
|
|
|
12
12
|
import { join } from "node:path";
|
|
13
13
|
import { randomUUID } from "node:crypto";
|
|
14
14
|
import { getToolkitRoot } from "../../../sdk/paths.js";
|
|
15
|
+
import { userEventCounters } from "../../../knowledge/search-client.js";
|
|
15
16
|
import { appendToOutbox } from "./outbox.js";
|
|
16
17
|
import { isRemoteEnabled } from "./remote-store.js";
|
|
17
18
|
import { SESSION_ID } from "./session.js";
|
|
@@ -384,6 +385,8 @@ export async function analyzeFeedback(rootOverride) {
|
|
|
384
385
|
qualityEntries.reduce((sum, e) => sum + (e.quality_data.accuracy ?? 0), 0) /
|
|
385
386
|
qualityEntries.length;
|
|
386
387
|
}
|
|
388
|
+
// UserEvent pipeline counters (in-memory, this session only)
|
|
389
|
+
const hasEventActivity = userEventCounters.sent > 0 || userEventCounters.failed > 0;
|
|
387
390
|
return {
|
|
388
391
|
summary: {
|
|
389
392
|
total_feedback: feedback.length,
|
|
@@ -394,6 +397,7 @@ export async function analyzeFeedback(rootOverride) {
|
|
|
394
397
|
telemetry_period: telemetry.length > 0
|
|
395
398
|
? { from: telemetry[0].ts, to: telemetry[telemetry.length - 1].ts }
|
|
396
399
|
: null,
|
|
400
|
+
...(hasEventActivity ? { user_events: { ...userEventCounters } } : {}),
|
|
397
401
|
},
|
|
398
402
|
category_breakdown: categoryBreakdown,
|
|
399
403
|
hot_spots: {
|
|
@@ -40,13 +40,18 @@ const cooldownMap = new Map();
|
|
|
40
40
|
let sessionUpdateCount = 0;
|
|
41
41
|
/** Per-document feedback history for graduated scoring */
|
|
42
42
|
const feedbackHistoryMap = new Map();
|
|
43
|
+
/** Recognized outcome suffixes from the outcome-feedback module */
|
|
44
|
+
const OUTCOME_SUFFIXES = ["_success", "_failure", "_partial", "_misaligned", "_accepted"];
|
|
43
45
|
/** Classify feedback strength based on context */
|
|
44
46
|
export function classifyEvidence(category, context) {
|
|
45
|
-
//
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
// Any tool outcome with a structured context suffix is hard evidence.
|
|
48
|
+
// Context format from outcome-feedback: "{tool}_{operation}_{quality}"
|
|
49
|
+
if (context) {
|
|
50
|
+
for (const suffix of OUTCOME_SUFFIXES) {
|
|
51
|
+
if (context.endsWith(suffix))
|
|
52
|
+
return "hard";
|
|
53
|
+
}
|
|
54
|
+
}
|
|
50
55
|
// Explicit corrections with knowledge_ref are medium-hard
|
|
51
56
|
if (category === "correction")
|
|
52
57
|
return "hard";
|
|
@@ -15,6 +15,8 @@ import { inferSourceType } from "../../../knowledge/pipeline/types.js";
|
|
|
15
15
|
import { computeConfidenceScore } from "../../../knowledge/pipeline/confidence.js";
|
|
16
16
|
import { actionsForSearchResults, actionsForNoResults, actionsForPublish } from "../response-actions.js";
|
|
17
17
|
import { generateRelatedQueries } from "./related-queries.js";
|
|
18
|
+
import { getOrCreateClientId } from "../feedback/client-id.js";
|
|
19
|
+
import { recordSearchResults } from "./session-state.js";
|
|
18
20
|
const GCS_BUCKET = "em1-knowledge";
|
|
19
21
|
async function checkSupersedeGuard(supersedes) {
|
|
20
22
|
if (!supersedes || supersedes.length === 0)
|
|
@@ -568,7 +570,8 @@ async function handleQuery(args) {
|
|
|
568
570
|
if (response.generativeAnswer) {
|
|
569
571
|
result.generative_answer = response.generativeAnswer;
|
|
570
572
|
result.citations = response.citations;
|
|
571
|
-
result._warning = "Generative answer is AI-synthesized.
|
|
573
|
+
result._warning = "Generative answer is AI-synthesized. Code examples may be fabricated or simplified — NEVER copy JSON without verification.";
|
|
574
|
+
result._next_step = 'Use workflow(mode="get") for canonical workflow_def format, or knowledge("schema/workflow-def", detail="excerpts") for validated examples.';
|
|
572
575
|
}
|
|
573
576
|
// Follow-up token for multi-turn answer conversations
|
|
574
577
|
if (response.answerQueryToken) {
|
|
@@ -599,6 +602,8 @@ async function handleQuery(args) {
|
|
|
599
602
|
if (related.length > 0) {
|
|
600
603
|
result._related_queries = related;
|
|
601
604
|
}
|
|
605
|
+
// Record results in session state for attribution cache + consultedDocs tracking
|
|
606
|
+
recordSearchResults(results.map((r) => ({ id: r.id })), response.attributionToken);
|
|
602
607
|
fireSearchEvent(query, response.attributionToken);
|
|
603
608
|
return result;
|
|
604
609
|
}
|
|
@@ -649,6 +654,8 @@ function contextualNextStep(results) {
|
|
|
649
654
|
return "Follow the guide steps relevant to your task";
|
|
650
655
|
return "Review results and refine search if needed";
|
|
651
656
|
}
|
|
657
|
+
/** Cached client ID for UserEvent pseudoId — resolved once, then sync. */
|
|
658
|
+
let _cachedPseudoId;
|
|
652
659
|
function fireSearchEvent(query, attributionToken) {
|
|
653
660
|
recordTelemetry({
|
|
654
661
|
type: "search_event",
|
|
@@ -657,10 +664,21 @@ function fireSearchEvent(query, attributionToken) {
|
|
|
657
664
|
ok: true,
|
|
658
665
|
resource_uri: attributionToken,
|
|
659
666
|
}).catch(() => { });
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
667
|
+
// Resolve pseudoId: use cached value if available, otherwise resolve async (first call only)
|
|
668
|
+
const emit = (pseudoId) => {
|
|
669
|
+
writeUserEvent({
|
|
670
|
+
eventType: "search",
|
|
671
|
+
userPseudoId: pseudoId,
|
|
672
|
+
attributionToken,
|
|
673
|
+
searchInfo: { searchQuery: query },
|
|
674
|
+
}).catch(() => { });
|
|
675
|
+
};
|
|
676
|
+
if (_cachedPseudoId) {
|
|
677
|
+
emit(_cachedPseudoId);
|
|
678
|
+
}
|
|
679
|
+
else {
|
|
680
|
+
getOrCreateClientId()
|
|
681
|
+
.then((id) => { _cachedPseudoId = id; emit(id); })
|
|
682
|
+
.catch(() => { emit("mcp-agent"); }); // fallback if client-id resolution fails
|
|
683
|
+
}
|
|
666
684
|
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Outcome Feedback — Layered signal emitter for knowledge quality feedback.
|
|
3
|
+
*
|
|
4
|
+
* Captures the full lifecycle of an agent interaction:
|
|
5
|
+
*
|
|
6
|
+
* Intent stated → work done → outcome assessed at multiple layers
|
|
7
|
+
*
|
|
8
|
+
* Signal layers (each carries different weight):
|
|
9
|
+
* 1. API acceptance (0.1) — system accepted the input structurally
|
|
10
|
+
* 2. Agent assessment (0.3) — agent's self-evaluation of alignment
|
|
11
|
+
* 3. Agent validation (0.5) — functional testing (conversation, debug)
|
|
12
|
+
* 4. End-user signal (1.0) — real user responded, kept using it, or abandoned
|
|
13
|
+
*
|
|
14
|
+
* Intent tracking:
|
|
15
|
+
* - Original intent is recorded at the start of the interaction
|
|
16
|
+
* - If intent pivots mid-journey, that's knowledge (not failure):
|
|
17
|
+
* "Users with intent X often pivot to Y" → proactive suggestion for next agent
|
|
18
|
+
* - Outcome is assessed against the FINAL intent, not the original
|
|
19
|
+
* - But the pivot itself is published as a pattern signal
|
|
20
|
+
*/
|
|
21
|
+
import { getConsultedDocs } from "./session-state.js";
|
|
22
|
+
import { processConfidenceFeedback } from "./confidence-loop.js";
|
|
23
|
+
import { writeUserEvent } from "../../../knowledge/search-client.js";
|
|
24
|
+
import { getOrCreateClientId } from "../feedback/client-id.js";
|
|
25
|
+
/** Weight multiplier per signal layer */
|
|
26
|
+
const LAYER_WEIGHTS = {
|
|
27
|
+
system: 0.1, // API accepted it — weakest signal
|
|
28
|
+
agent: 0.3, // Agent self-assessment — has context but may be biased
|
|
29
|
+
validation: 0.5, // Functional testing — objective but synthetic
|
|
30
|
+
user: 1.0, // End-user response — ground truth
|
|
31
|
+
};
|
|
32
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
33
|
+
// Outcome Resolution
|
|
34
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
35
|
+
function resolveOutcome(success) {
|
|
36
|
+
if (typeof success === "boolean") {
|
|
37
|
+
return success
|
|
38
|
+
? { quality: "success", isPositive: true, suffix: "success", category: "success" }
|
|
39
|
+
: { quality: "failure", isPositive: false, suffix: "failure", category: "correction" };
|
|
40
|
+
}
|
|
41
|
+
switch (success) {
|
|
42
|
+
case "success":
|
|
43
|
+
return { quality: "success", isPositive: true, suffix: "success", category: "success" };
|
|
44
|
+
case "partial":
|
|
45
|
+
return { quality: "partial", isPositive: true, suffix: "partial", category: "success" };
|
|
46
|
+
case "accepted":
|
|
47
|
+
return { quality: "accepted", isPositive: true, suffix: "accepted", category: "interaction" };
|
|
48
|
+
case "misaligned":
|
|
49
|
+
return { quality: "misaligned", isPositive: false, suffix: "misaligned", category: "correction" };
|
|
50
|
+
case "failure":
|
|
51
|
+
return { quality: "failure", isPositive: false, suffix: "failure", category: "correction" };
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
55
|
+
// Intent Pivot Tracking
|
|
56
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
57
|
+
const MAX_PIVOTS = 100;
|
|
58
|
+
/** In-memory pivot accumulator — aggregated by the journey/feedback adapter */
|
|
59
|
+
const intentPivots = [];
|
|
60
|
+
/** Get accumulated intent pivots (for journey reporting) */
|
|
61
|
+
export function getIntentPivots() {
|
|
62
|
+
return intentPivots;
|
|
63
|
+
}
|
|
64
|
+
/** Reset pivot state (for test isolation) */
|
|
65
|
+
export function _resetIntentPivots() {
|
|
66
|
+
intentPivots.length = 0;
|
|
67
|
+
}
|
|
68
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
69
|
+
// Main Entry Point
|
|
70
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
71
|
+
/**
|
|
72
|
+
* Emit outcome feedback for all consulted knowledge docs in the current session.
|
|
73
|
+
*
|
|
74
|
+
* Best-effort: never throws. Failures are logged but don't block the calling handler.
|
|
75
|
+
*
|
|
76
|
+
* @example
|
|
77
|
+
* // Layer 1: Deploy accepted (system signal, low weight)
|
|
78
|
+
* await emitOutcomeFeedback({
|
|
79
|
+
* tool: "workflow", operation: "deploy", success: "accepted", layer: "system",
|
|
80
|
+
* });
|
|
81
|
+
*
|
|
82
|
+
* // Layer 2: Agent thinks it looks right
|
|
83
|
+
* await emitOutcomeFeedback({
|
|
84
|
+
* tool: "workflow", operation: "deploy", success: "partial", layer: "agent",
|
|
85
|
+
* intent: "route billing questions to billing team",
|
|
86
|
+
* });
|
|
87
|
+
*
|
|
88
|
+
* // Layer 3: Conversation test confirms behavior
|
|
89
|
+
* await emitOutcomeFeedback({
|
|
90
|
+
* tool: "conversation", operation: "test", success: "success", layer: "validation",
|
|
91
|
+
* intent: "route billing questions to billing team",
|
|
92
|
+
* });
|
|
93
|
+
*
|
|
94
|
+
* // Layer 4: End-user kept using it (highest weight)
|
|
95
|
+
* await emitOutcomeFeedback({
|
|
96
|
+
* tool: "conversation", operation: "usage", success: true, layer: "user",
|
|
97
|
+
* });
|
|
98
|
+
*
|
|
99
|
+
* // Intent pivot (knowledge, not failure):
|
|
100
|
+
* await emitOutcomeFeedback({
|
|
101
|
+
* tool: "workflow", operation: "deploy", success: "success", layer: "validation",
|
|
102
|
+
* intent: {
|
|
103
|
+
* original: "route billing questions to billing team",
|
|
104
|
+
* final: "route billing questions and auto-generate invoice summaries",
|
|
105
|
+
* pivotReason: "user realized they also need invoice summaries during testing",
|
|
106
|
+
* },
|
|
107
|
+
* });
|
|
108
|
+
*/
|
|
109
|
+
export async function emitOutcomeFeedback(event) {
|
|
110
|
+
const docs = getConsultedDocs();
|
|
111
|
+
const outcome = resolveOutcome(event.success);
|
|
112
|
+
const layer = event.layer ?? "system";
|
|
113
|
+
const weight = LAYER_WEIGHTS[layer];
|
|
114
|
+
const eventType = `${event.tool}-${event.operation}-${outcome.suffix}`;
|
|
115
|
+
// Track intent pivots — these are knowledge signals, not failures
|
|
116
|
+
if (event.intent && typeof event.intent === "object" && event.intent.final && event.intent.final !== event.intent.original) {
|
|
117
|
+
intentPivots.push({
|
|
118
|
+
original: event.intent.original,
|
|
119
|
+
final: event.intent.final,
|
|
120
|
+
reason: event.intent.pivotReason,
|
|
121
|
+
tool: event.tool,
|
|
122
|
+
timestamp: new Date().toISOString(),
|
|
123
|
+
});
|
|
124
|
+
// Evict oldest entries to prevent unbounded growth
|
|
125
|
+
while (intentPivots.length > MAX_PIVOTS)
|
|
126
|
+
intentPivots.shift();
|
|
127
|
+
console.error(`[INTENT-PIVOT] "${event.intent.original}" → "${event.intent.final}"` +
|
|
128
|
+
(event.intent.pivotReason ? ` (${event.intent.pivotReason})` : ""));
|
|
129
|
+
}
|
|
130
|
+
if (docs.size === 0) {
|
|
131
|
+
return { docs_processed: 0, confidence_updates: 0, event_type: eventType };
|
|
132
|
+
}
|
|
133
|
+
// "accepted" = system accepted it, no intent validation → skip confidence updates.
|
|
134
|
+
// This applies at ALL layers: "accepted" means structural acceptance, not functional
|
|
135
|
+
// success. Use "partial" or "success" to indicate functional validation.
|
|
136
|
+
if (outcome.quality === "accepted") {
|
|
137
|
+
return { docs_processed: docs.size, confidence_updates: 0, event_type: eventType };
|
|
138
|
+
}
|
|
139
|
+
const defaultCategory = outcome.category;
|
|
140
|
+
// Context encodes tool, operation, quality, AND layer for evidence classification
|
|
141
|
+
const context = `${event.tool}_${event.operation}_${outcome.suffix}`;
|
|
142
|
+
let updates = 0;
|
|
143
|
+
// Scale the quality signal for lower layers: system/agent produce weaker
|
|
144
|
+
// quality data, so downgrade "success" to "interaction" (neutral) at system layer.
|
|
145
|
+
// Higher layers (validation, user) keep the original category.
|
|
146
|
+
const effectiveDefaultCategory = (layer === "system" && defaultCategory === "success")
|
|
147
|
+
? "interaction" // System-layer positive is neutral — API acceptance isn't validation
|
|
148
|
+
: defaultCategory;
|
|
149
|
+
const assessment = event.agentAssessment;
|
|
150
|
+
const helpfulSet = new Set(assessment?.helpful ?? []);
|
|
151
|
+
const misleadingSet = new Set(assessment?.misleading ?? []);
|
|
152
|
+
for (const docId of docs) {
|
|
153
|
+
try {
|
|
154
|
+
// Agent assessment overrides the blanket signal per doc
|
|
155
|
+
let category = effectiveDefaultCategory;
|
|
156
|
+
if (helpfulSet.has(docId)) {
|
|
157
|
+
category = "success";
|
|
158
|
+
}
|
|
159
|
+
else if (misleadingSet.has(docId)) {
|
|
160
|
+
category = "correction";
|
|
161
|
+
}
|
|
162
|
+
const result = await processConfidenceFeedback(category, docId, undefined, context);
|
|
163
|
+
if (result)
|
|
164
|
+
updates++;
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
// Best-effort
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
// Emit gap signals for missing knowledge
|
|
171
|
+
if (assessment?.missing) {
|
|
172
|
+
for (const topic of assessment.missing) {
|
|
173
|
+
try {
|
|
174
|
+
await processConfidenceFeedback("gap", `missing:${topic}`, undefined, context);
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
// Best-effort
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
// Fire UserEvents (fire-and-forget)
|
|
182
|
+
getOrCreateClientId()
|
|
183
|
+
.then((clientId) => {
|
|
184
|
+
// Conversion value = outcome quality × layer weight
|
|
185
|
+
const baseValue = outcome.quality === "success" ? 1.0
|
|
186
|
+
: outcome.quality === "partial" ? 0.5
|
|
187
|
+
: 0;
|
|
188
|
+
const conversionValue = baseValue * weight;
|
|
189
|
+
const isConversion = outcome.isPositive && outcome.quality !== "accepted";
|
|
190
|
+
const documents = [...docs].map((docId) => ({
|
|
191
|
+
id: docId,
|
|
192
|
+
...(isConversion ? { conversionValue } : {}),
|
|
193
|
+
}));
|
|
194
|
+
writeUserEvent({
|
|
195
|
+
eventType: isConversion ? "conversion" : "view-item",
|
|
196
|
+
userPseudoId: clientId,
|
|
197
|
+
...(isConversion ? { conversionType: eventType } : {}),
|
|
198
|
+
documents,
|
|
199
|
+
}).catch(() => { });
|
|
200
|
+
})
|
|
201
|
+
.catch(() => { });
|
|
202
|
+
console.error(`[OUTCOME-FEEDBACK] ${eventType} (layer=${layer}, weight=${weight}): ` +
|
|
203
|
+
`${docs.size} consulted docs, ${updates} confidence updates`);
|
|
204
|
+
return { docs_processed: docs.size, confidence_updates: updates, event_type: eventType };
|
|
205
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Session State — Shared tracking for knowledge search sessions.
|
|
3
|
+
*
|
|
4
|
+
* Tracks which documents were served (consultedDocs), their attribution tokens,
|
|
5
|
+
* and DE resource names. Used by:
|
|
6
|
+
* - Task 2.2: consultedDocs for deploy outcome feedback
|
|
7
|
+
* - Task 2.9: attribution token cache for UserEvent correlation
|
|
8
|
+
* - Task 2.10: conversion events from positive feedback
|
|
9
|
+
* - Task 2.12: auto-citation tracking + dedup for answer mode
|
|
10
|
+
*/
|
|
11
|
+
const ATTRIBUTION_TTL_MS = 30 * 60 * 1000; // 30 minutes
|
|
12
|
+
const MAX_CACHE_SIZE = 500;
|
|
13
|
+
const SWEEP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
|
14
|
+
/** Documents consulted in the current deploy cycle. Reset before each deploy attempt. */
|
|
15
|
+
const consultedDocs = new Set();
|
|
16
|
+
/** Attribution token cache: artifactId → {token, ts, deResourceName}. */
|
|
17
|
+
const attributionCache = new Map();
|
|
18
|
+
/** Per-session dedup for L1 auto-citation view-item events. */
|
|
19
|
+
const citationDedupeSet = new Set();
|
|
20
|
+
/** Deploy attempt counter for retry-aware feedback. */
|
|
21
|
+
let deployAttempts = 0;
|
|
22
|
+
// ─── Periodic sweep to prevent memory leak ───────────────────────────────────
|
|
23
|
+
let sweepTimer;
|
|
24
|
+
function startSweepTimer() {
|
|
25
|
+
if (sweepTimer)
|
|
26
|
+
return;
|
|
27
|
+
sweepTimer = setInterval(() => {
|
|
28
|
+
const now = Date.now();
|
|
29
|
+
for (const [id, entry] of attributionCache) {
|
|
30
|
+
if (now - entry.ts > ATTRIBUTION_TTL_MS)
|
|
31
|
+
attributionCache.delete(id);
|
|
32
|
+
}
|
|
33
|
+
// Evict oldest if over max size
|
|
34
|
+
if (attributionCache.size > MAX_CACHE_SIZE) {
|
|
35
|
+
const sorted = [...attributionCache.entries()].sort((a, b) => a[1].ts - b[1].ts);
|
|
36
|
+
const toRemove = sorted.slice(0, sorted.length - MAX_CACHE_SIZE);
|
|
37
|
+
for (const [id] of toRemove)
|
|
38
|
+
attributionCache.delete(id);
|
|
39
|
+
}
|
|
40
|
+
}, SWEEP_INTERVAL_MS);
|
|
41
|
+
// Don't prevent process exit
|
|
42
|
+
if (sweepTimer && typeof sweepTimer === "object" && "unref" in sweepTimer) {
|
|
43
|
+
sweepTimer.unref();
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// ─── Public API ──────────────────────────────────────────────────────────────
|
|
47
|
+
/**
|
|
48
|
+
* Record that documents were served in a search response.
|
|
49
|
+
* Call after each knowledge() search returns results.
|
|
50
|
+
*/
|
|
51
|
+
export function recordSearchResults(results, attributionToken) {
|
|
52
|
+
startSweepTimer();
|
|
53
|
+
const now = Date.now();
|
|
54
|
+
for (const r of results) {
|
|
55
|
+
if (!r.id)
|
|
56
|
+
continue;
|
|
57
|
+
consultedDocs.add(r.id);
|
|
58
|
+
attributionCache.set(r.id, {
|
|
59
|
+
token: attributionToken,
|
|
60
|
+
ts: now,
|
|
61
|
+
deResourceName: r.deResourceName,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** Get the attribution token for a document (if cached and not expired). */
|
|
66
|
+
export function getAttributionToken(docId) {
|
|
67
|
+
const entry = attributionCache.get(docId);
|
|
68
|
+
if (!entry)
|
|
69
|
+
return undefined;
|
|
70
|
+
if (Date.now() - entry.ts > ATTRIBUTION_TTL_MS) {
|
|
71
|
+
attributionCache.delete(docId);
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
return entry.token;
|
|
75
|
+
}
|
|
76
|
+
/** Get all documents consulted in the current deploy cycle. */
|
|
77
|
+
export function getConsultedDocs() {
|
|
78
|
+
return consultedDocs;
|
|
79
|
+
}
|
|
80
|
+
/** Reset consultedDocs for a new deploy cycle. Call before each deploy attempt. */
|
|
81
|
+
export function resetConsultedDocs() {
|
|
82
|
+
consultedDocs.clear();
|
|
83
|
+
}
|
|
84
|
+
/** Increment deploy attempts counter. Call at start of each deploy. */
|
|
85
|
+
export function incrementDeployAttempts() {
|
|
86
|
+
deployAttempts++;
|
|
87
|
+
}
|
|
88
|
+
/** Get current deploy attempt count. */
|
|
89
|
+
export function getDeployAttempts() {
|
|
90
|
+
return deployAttempts;
|
|
91
|
+
}
|
|
92
|
+
/** Check if a citation has already been tracked this session (for L1 dedup). */
|
|
93
|
+
export function hasEmittedCitation(docId) {
|
|
94
|
+
return citationDedupeSet.has(docId);
|
|
95
|
+
}
|
|
96
|
+
/** Mark a citation as emitted this session. */
|
|
97
|
+
export function markCitationEmitted(docId) {
|
|
98
|
+
citationDedupeSet.add(docId);
|
|
99
|
+
}
|
|
100
|
+
/** Reset all state (for test isolation). */
|
|
101
|
+
export function _resetSessionState() {
|
|
102
|
+
consultedDocs.clear();
|
|
103
|
+
attributionCache.clear();
|
|
104
|
+
citationDedupeSet.clear();
|
|
105
|
+
deployAttempts = 0;
|
|
106
|
+
if (sweepTimer) {
|
|
107
|
+
clearInterval(sweepTimer);
|
|
108
|
+
sweepTimer = undefined;
|
|
109
|
+
}
|
|
110
|
+
}
|