@ema.co/mcp-toolkit 2026.3.25-3 → 2026.3.29-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/login.js +26 -26
- package/dist/config/tool-guidance.js +2 -2
- package/dist/knowledge/extractors/agent-catalog.js +14 -8
- package/dist/knowledge/pipeline/confidence.js +137 -25
- package/dist/knowledge/search-client.js +29 -5
- package/dist/knowledge/search-config.js +2 -1
- package/dist/mcp/domain/workflow-def-schema.js +1 -1
- package/dist/mcp/domain/workflow-def-validator.js +2 -2
- package/dist/mcp/guidance/classify.js +5 -4
- package/dist/mcp/guidance/defaults.js +2 -1
- package/dist/mcp/guidance.js +2 -1
- package/dist/mcp/handlers/feedback/index.js +32 -0
- package/dist/mcp/handlers/feedback/store.js +4 -0
- package/dist/mcp/handlers/knowledge/confidence-loop.js +85 -11
- package/dist/mcp/handlers/knowledge/index.js +23 -6
- package/dist/mcp/handlers/knowledge/outcome-feedback.js +205 -0
- package/dist/mcp/handlers/knowledge/session-state.js +110 -0
- package/dist/mcp/handlers/persona/create.js +119 -7
- package/dist/mcp/handlers/utils.js +5 -1
- package/dist/mcp/handlers/workflow/adapter.js +2 -0
- package/dist/mcp/handlers/workflow/deploy.js +33 -0
- package/dist/mcp/handlers/workflow/index.js +23 -0
- package/dist/mcp/handlers/workflow/validation.js +29 -5
- package/dist/mcp/knowledge-guidance-topics.js +8 -3
- package/dist/mcp/resources-dynamic.js +41 -11
- package/dist/mcp/tools.js +5 -0
- package/package.json +1 -1
package/dist/auth/login.js
CHANGED
|
@@ -145,14 +145,14 @@ async function loginWithPasteToken(appUrl) {
|
|
|
145
145
|
await new Promise((resolve) => {
|
|
146
146
|
exec(cmd, () => resolve());
|
|
147
147
|
});
|
|
148
|
-
console.
|
|
149
|
-
console.
|
|
150
|
-
console.
|
|
151
|
-
console.
|
|
152
|
-
console.
|
|
153
|
-
console.
|
|
154
|
-
console.
|
|
155
|
-
const rl = createInterface({ input: process.stdin, output: process.
|
|
148
|
+
console.error(`\nBrowser opened to: ${appUrl}`);
|
|
149
|
+
console.error("Log in normally, then grab your bearer token:\n");
|
|
150
|
+
console.error(" 1. Open DevTools (F12 or Cmd+Opt+I)");
|
|
151
|
+
console.error(" 2. Go to Network tab");
|
|
152
|
+
console.error(" 3. Filter for 'generate_token_from_code'");
|
|
153
|
+
console.error(" 4. Click the request → Response tab");
|
|
154
|
+
console.error(" 5. Copy the access_token value (starts with eyJ...)\n");
|
|
155
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
156
156
|
const token = await new Promise((resolve) => {
|
|
157
157
|
rl.question("Paste bearer token here: ", (answer) => {
|
|
158
158
|
rl.close();
|
|
@@ -205,8 +205,8 @@ export async function loginGoogleOAuth(opts = {}) {
|
|
|
205
205
|
});
|
|
206
206
|
});
|
|
207
207
|
// Navigate to Cloud Console — triggers Google auth
|
|
208
|
-
console.
|
|
209
|
-
console.
|
|
208
|
+
console.error("Opening browser for Google authentication...");
|
|
209
|
+
console.error("Sign in with your Google account — the token will be captured automatically.\n");
|
|
210
210
|
await page.goto("https://console.cloud.google.com/");
|
|
211
211
|
const token = await tokenPromise;
|
|
212
212
|
return token;
|
|
@@ -232,15 +232,15 @@ async function isPlaywrightAvailable() {
|
|
|
232
232
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
233
233
|
async function promptTenantSelection(tenants) {
|
|
234
234
|
const { createInterface } = await import("node:readline");
|
|
235
|
-
console.
|
|
235
|
+
console.error(`\nMultiple tenants available:`);
|
|
236
236
|
for (let i = 0; i < tenants.length; i++) {
|
|
237
237
|
const t = tenants[i];
|
|
238
238
|
const marker = t.is_current ? " ← current" : "";
|
|
239
|
-
console.
|
|
239
|
+
console.error(` [${i + 1}] ${t.company_name} (${t.domain})${marker}`);
|
|
240
240
|
}
|
|
241
241
|
const defaultIdx = tenants.findIndex((t) => t.is_current);
|
|
242
242
|
const defaultNum = defaultIdx >= 0 ? defaultIdx + 1 : 1;
|
|
243
|
-
const rl = createInterface({ input: process.stdin, output: process.
|
|
243
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
244
244
|
const answer = await new Promise((resolve) => {
|
|
245
245
|
rl.question(`\nSelect tenant [${defaultNum}]: `, (ans) => {
|
|
246
246
|
rl.close();
|
|
@@ -250,11 +250,11 @@ async function promptTenantSelection(tenants) {
|
|
|
250
250
|
const choice = answer === "" ? defaultNum : parseInt(answer, 10);
|
|
251
251
|
if (choice >= 1 && choice <= tenants.length) {
|
|
252
252
|
const selected = tenants[choice - 1];
|
|
253
|
-
console.
|
|
253
|
+
console.error(`Selected: ${selected.company_name}`);
|
|
254
254
|
return selected.tenant_id;
|
|
255
255
|
}
|
|
256
256
|
// Invalid input — use default
|
|
257
|
-
console.
|
|
257
|
+
console.error(`Invalid choice, using default: ${tenants[defaultNum - 1].company_name}`);
|
|
258
258
|
return tenants[defaultNum - 1].tenant_id;
|
|
259
259
|
}
|
|
260
260
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -281,22 +281,22 @@ export async function login(opts = {}) {
|
|
|
281
281
|
else if (await isPlaywrightAvailable()) {
|
|
282
282
|
// Primary: Playwright-controlled browser, with paste-token fallback on failure
|
|
283
283
|
try {
|
|
284
|
-
console.
|
|
285
|
-
console.
|
|
284
|
+
console.error("Opening browser for login...");
|
|
285
|
+
console.error("Authenticate normally — the token will be captured automatically.\n");
|
|
286
286
|
tokenResponse = await loginWithPlaywright(appUrl, apiUrl, timeoutMs);
|
|
287
287
|
}
|
|
288
288
|
catch (err) {
|
|
289
289
|
const msg = err instanceof Error ? err.message : String(err);
|
|
290
|
-
console.
|
|
291
|
-
console.
|
|
290
|
+
console.error(`\nBrowser login failed: ${msg}`);
|
|
291
|
+
console.error("Falling back to manual token entry...\n");
|
|
292
292
|
tokenResponse = await loginWithPasteToken(appUrl);
|
|
293
293
|
}
|
|
294
294
|
}
|
|
295
295
|
else {
|
|
296
296
|
// Fallback: system browser + paste URL
|
|
297
|
-
console.
|
|
298
|
-
console.
|
|
299
|
-
console.
|
|
297
|
+
console.error("(Playwright not installed — using paste-URL fallback)\n");
|
|
298
|
+
console.error("For automatic login, install playwright-chromium:");
|
|
299
|
+
console.error(" npm install -g playwright-chromium\n");
|
|
300
300
|
tokenResponse = await loginWithPasteToken(appUrl);
|
|
301
301
|
}
|
|
302
302
|
const token = tokenResponse.access_token;
|
|
@@ -318,10 +318,10 @@ export async function login(opts = {}) {
|
|
|
318
318
|
const match = accessibleTenants.find((t) => t.tenant_id === opts.tenantId);
|
|
319
319
|
if (match) {
|
|
320
320
|
selectedTenantId = match.tenant_id;
|
|
321
|
-
console.
|
|
321
|
+
console.error(`\nUsing tenant: ${match.company_name} (${match.domain})`);
|
|
322
322
|
}
|
|
323
323
|
else {
|
|
324
|
-
console.
|
|
324
|
+
console.error(`\nWarning: tenant ${opts.tenantId} not in accessible list. Using default.`);
|
|
325
325
|
}
|
|
326
326
|
}
|
|
327
327
|
else if (process.stdin.isTTY) {
|
|
@@ -330,10 +330,10 @@ export async function login(opts = {}) {
|
|
|
330
330
|
}
|
|
331
331
|
else {
|
|
332
332
|
// Non-interactive: log what's available
|
|
333
|
-
console.
|
|
333
|
+
console.error(`\nAccessible tenants (${accessibleTenants.length}):`);
|
|
334
334
|
for (const t of accessibleTenants) {
|
|
335
335
|
const marker = t.is_current ? " ← active" : "";
|
|
336
|
-
console.
|
|
336
|
+
console.error(` ${t.company_name} (${t.domain})${marker}`);
|
|
337
337
|
}
|
|
338
338
|
}
|
|
339
339
|
}
|
|
@@ -176,13 +176,13 @@ export const TOOL_GUIDANCE = {
|
|
|
176
176
|
operations: [
|
|
177
177
|
{ name: "Get", description: "Fetch current workflow_def + generation schema", example: 'workflow(mode="get", persona_id="...")' },
|
|
178
178
|
{ name: "Get (slim)", description: "Fetch slimmed workflow_def for large workflows (strips displaySettings, truncates long values, ~60-70% smaller)", example: 'workflow(mode="get", persona_id="...", slim=true)' },
|
|
179
|
-
{ name: "Deploy", description: "Deploy LLM-generated workflow_def", example: 'workflow(mode="deploy", persona_id="...", workflow_def={...})' },
|
|
179
|
+
{ name: "Deploy", description: "Deploy LLM-generated workflow_def. Deploy outcomes automatically feed knowledge quality — failures demote consulted docs. Test with conversation() after deploy to validate intent alignment.", example: 'workflow(mode="deploy", persona_id="...", workflow_def={...})' },
|
|
180
180
|
{ name: "Validate", description: "Static validation with path enumeration", example: 'workflow(mode="validate", persona_id="...")' },
|
|
181
181
|
{ name: "Optimize", description: "Structural graph optimization", example: 'workflow(mode="optimize", persona_id="...")' },
|
|
182
182
|
],
|
|
183
183
|
nextSteps: {
|
|
184
184
|
get: "Build a workflow_def based on the generation_schema and deploy it.",
|
|
185
|
-
deploy: "
|
|
185
|
+
deploy: "Test with conversation() to validate intent alignment. Deploy success only means the API accepted it — conversation testing validates the persona actually works.",
|
|
186
186
|
validate: "Fix any reported issues, then deploy.",
|
|
187
187
|
optimize: "Review optimized workflow_def, then deploy if acceptable.",
|
|
188
188
|
},
|
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
export async function extractAgentCatalog(config) {
|
|
2
2
|
const { AGENT_CATALOG } = await import("../../sdk/generated/agent-catalog.js");
|
|
3
3
|
const documents = AGENT_CATALOG.map((action) => {
|
|
4
|
-
const inputs = (action.inputs ?? []).map((i) => `${i.name}
|
|
5
|
-
const outputs = (action.outputs ?? []).map((o) => `${o.name}
|
|
4
|
+
const inputs = (action.inputs ?? []).map((i) => `${i.name} (${i.type})${i.description ? `: ${i.description}` : ""}`).join("; ");
|
|
5
|
+
const outputs = (action.outputs ?? []).map((o) => `${o.name} (${o.type})${o.description ? `: ${o.description}` : ""}`).join("; ");
|
|
6
|
+
const criticalRules = (action.criticalRules ?? []).map((r) => `- ${r}`).join("\n");
|
|
7
|
+
// Build prose content optimized for semantic search.
|
|
8
|
+
// Repeat the actionName in natural language so DE embeddings match queries like "call_llm".
|
|
6
9
|
const content = [
|
|
7
|
-
action.displayName
|
|
8
|
-
|
|
9
|
-
action.
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
`# ${action.actionName} — ${action.displayName}`,
|
|
11
|
+
"",
|
|
12
|
+
`The ${action.actionName} action (also known as "${action.displayName}") ${action.description?.toLowerCase() ?? ""}`,
|
|
13
|
+
action.whenToUse ? `\nUse ${action.actionName} when: ${action.whenToUse}` : "",
|
|
14
|
+
action.whenNotToUse ? `\nDo NOT use ${action.actionName} when: ${action.whenNotToUse}` : "",
|
|
15
|
+
inputs ? `\n## Inputs for ${action.actionName}\n${inputs}` : "",
|
|
16
|
+
outputs ? `\n## Outputs from ${action.actionName}\n${outputs}` : "",
|
|
17
|
+
criticalRules ? `\n## Critical rules for ${action.actionName}\n${criticalRules}` : "",
|
|
18
|
+
action.aliases?.length ? `\nAlso known as: ${action.aliases.join(", ")}` : "",
|
|
13
19
|
].filter(Boolean).join("\n");
|
|
14
20
|
return {
|
|
15
21
|
id: `entity:${action.actionName}`,
|
|
@@ -53,7 +53,7 @@ export function computeConfidenceScore(provenance, feedbackDelta, boost) {
|
|
|
53
53
|
const floor = Math.max(0, base - MAX_NEGATIVE_DRIFT);
|
|
54
54
|
return Math.max(floor, Math.min(1.0, adjusted));
|
|
55
55
|
}
|
|
56
|
-
/** Per-event feedback deltas (used by runtime confidence-loop) */
|
|
56
|
+
/** Per-event feedback deltas (used by runtime confidence-loop for backward compat) */
|
|
57
57
|
export const FEEDBACK_DELTA_NEGATIVE = -0.08;
|
|
58
58
|
export const FEEDBACK_DELTA_POSITIVE = 0.04;
|
|
59
59
|
/** Maximum boost above provenance base from positive feedback */
|
|
@@ -62,6 +62,12 @@ export const MAX_POSITIVE_BOOST = 0.15;
|
|
|
62
62
|
export const MAX_NEGATIVE_DRIFT = 0.30;
|
|
63
63
|
/** Minimum score delta to trigger a DE update (avoids churn) */
|
|
64
64
|
export const MIN_SCORE_DELTA = 0.05;
|
|
65
|
+
/** Minimum unique clients required before score changes take effect */
|
|
66
|
+
export const MIN_CORROBORATION_CLIENTS = 2;
|
|
67
|
+
/** Evidence-based feedback gets stronger delta multiplier */
|
|
68
|
+
export const EVIDENCE_MULTIPLIER = 1.5; // deploy failure = hard evidence
|
|
69
|
+
/** Soft feedback gets weaker delta multiplier */
|
|
70
|
+
export const SOFT_MULTIPLIER = 0.5; // "this seems wrong" without evidence
|
|
65
71
|
/** Score thresholds for label assignment — single source of truth */
|
|
66
72
|
export const LABEL_THRESHOLDS = {
|
|
67
73
|
verified: 0.80, // >= 0.80
|
|
@@ -79,19 +85,134 @@ export function scoreToLabel(score) {
|
|
|
79
85
|
return "inferred";
|
|
80
86
|
return "low-confidence";
|
|
81
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* Compute confidence adjustment based on the ratio of negative to total feedback.
|
|
90
|
+
*
|
|
91
|
+
* Key principle: a doc with 500 positives and 5 negatives (1% negative) should NOT
|
|
92
|
+
* be downgraded — the 5 are likely confused agents, not a real problem.
|
|
93
|
+
*
|
|
94
|
+
* The ratio determines the direction. The total count determines the strength.
|
|
95
|
+
* Both matter: low ratio + high count = strong confidence. High ratio + low count = weak signal.
|
|
96
|
+
*
|
|
97
|
+
* @param negativeCount - Total negative feedback events
|
|
98
|
+
* @param positiveCount - Total positive feedback events
|
|
99
|
+
* @param uniqueClients - Number of distinct clients (for corroboration)
|
|
100
|
+
* @returns Confidence delta to apply (negative = downgrade, positive = upgrade)
|
|
101
|
+
*/
|
|
102
|
+
export function computeFeedbackDelta(negativeCount, positiveCount, uniqueClients = 1) {
|
|
103
|
+
const total = negativeCount + positiveCount;
|
|
104
|
+
if (total === 0)
|
|
105
|
+
return 0;
|
|
106
|
+
const negativeRatio = negativeCount / total;
|
|
107
|
+
// Confidence bands based on negative ratio
|
|
108
|
+
// High negative ratio = downgrade, low ratio = upgrade, middle = neutral
|
|
109
|
+
let baseDelta;
|
|
110
|
+
if (negativeRatio >= 0.80) {
|
|
111
|
+
// Overwhelmingly negative — strong downgrade
|
|
112
|
+
baseDelta = -0.15;
|
|
113
|
+
}
|
|
114
|
+
else if (negativeRatio >= 0.60) {
|
|
115
|
+
// Mostly negative — moderate downgrade
|
|
116
|
+
baseDelta = -0.10;
|
|
117
|
+
}
|
|
118
|
+
else if (negativeRatio >= 0.40) {
|
|
119
|
+
// Mixed signals — slight downgrade (benefit of doubt to negative)
|
|
120
|
+
baseDelta = -0.05;
|
|
121
|
+
}
|
|
122
|
+
else if (negativeRatio >= 0.20) {
|
|
123
|
+
// Mostly positive with some complaints — neutral/slight upgrade
|
|
124
|
+
baseDelta = 0.02;
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
// Overwhelmingly positive — upgrade
|
|
128
|
+
baseDelta = 0.05;
|
|
129
|
+
}
|
|
130
|
+
// Volume amplifier — more total feedback = more confidence in the signal
|
|
131
|
+
// But with diminishing returns (log scale)
|
|
132
|
+
const volumeMultiplier = Math.min(2.0, 1.0 + Math.log2(Math.max(1, total)) * 0.15);
|
|
133
|
+
// Corroboration amplifier — multiple independent clients agreeing is stronger
|
|
134
|
+
const corroborationMultiplier = Math.min(1.5, 1.0 + Math.max(0, uniqueClients - 1) * 0.1);
|
|
135
|
+
return baseDelta * volumeMultiplier * corroborationMultiplier;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Compute effective floor — allows breakthrough under sustained, high-ratio negative feedback.
|
|
139
|
+
*
|
|
140
|
+
* The base floor (provenance - MAX_NEGATIVE_DRIFT) protects against noise.
|
|
141
|
+
* The floor only gives way when: high negative ratio AND sufficient volume.
|
|
142
|
+
* This prevents a single bad feedback from breaking the floor.
|
|
143
|
+
*/
|
|
144
|
+
export function effectiveFloor(provenanceBase, negativeRatio, totalCount) {
|
|
145
|
+
const baseFloor = Math.max(0, provenanceBase - MAX_NEGATIVE_DRIFT);
|
|
146
|
+
// Floor only gives way when: high negative ratio AND sufficient volume
|
|
147
|
+
if (negativeRatio < 0.70 || totalCount < 5)
|
|
148
|
+
return baseFloor;
|
|
149
|
+
// Beyond threshold: floor lowers proportionally to how negative the ratio is
|
|
150
|
+
const floorReduction = (negativeRatio - 0.70) * totalCount * 0.01;
|
|
151
|
+
return Math.max(0.10, baseFloor - floorReduction);
|
|
152
|
+
}
|
|
153
|
+
// ── Legacy graduated functions (kept for backward compat imports) ────────────
|
|
154
|
+
/** @deprecated Use computeFeedbackDelta instead */
|
|
155
|
+
export function graduatedNegativeDelta(negativeCount) {
|
|
156
|
+
if (negativeCount >= 8)
|
|
157
|
+
return -0.14;
|
|
158
|
+
if (negativeCount >= 5)
|
|
159
|
+
return -0.12;
|
|
160
|
+
if (negativeCount >= 3)
|
|
161
|
+
return -0.08;
|
|
162
|
+
if (negativeCount >= 2)
|
|
163
|
+
return -0.06;
|
|
164
|
+
return -0.04;
|
|
165
|
+
}
|
|
166
|
+
/** @deprecated Use computeFeedbackDelta instead */
|
|
167
|
+
export function graduatedPositiveDelta(positiveCount) {
|
|
168
|
+
if (positiveCount >= 5)
|
|
169
|
+
return 0.06;
|
|
170
|
+
if (positiveCount >= 3)
|
|
171
|
+
return 0.04;
|
|
172
|
+
return 0.03;
|
|
173
|
+
}
|
|
82
174
|
/**
|
|
83
175
|
* Compute a per-event feedback delta and apply to current score.
|
|
84
176
|
* Used by the runtime confidence loop when a single feedback event arrives.
|
|
85
177
|
*
|
|
86
|
-
*
|
|
178
|
+
* When feedbackHistory is provided, uses graduated deltas that accelerate
|
|
179
|
+
* with corroboration. Without feedbackHistory, uses flat deltas for
|
|
180
|
+
* backward compatibility.
|
|
181
|
+
*
|
|
182
|
+
* @returns { newScore, label } — clamped to [floor, provenanceBase + MAX_POSITIVE_BOOST]
|
|
87
183
|
*/
|
|
88
|
-
export function applyFeedbackDelta(currentScore, provenance, isNegative) {
|
|
89
|
-
const delta = isNegative ? FEEDBACK_DELTA_NEGATIVE : FEEDBACK_DELTA_POSITIVE;
|
|
184
|
+
export function applyFeedbackDelta(currentScore, provenance, isNegative, feedbackHistory) {
|
|
90
185
|
const provenanceBase = PROVENANCE_BASE_SCORES[provenance] ?? PROVENANCE_BASE_SCORES["inferred"];
|
|
91
186
|
const maxScore = provenanceBase + MAX_POSITIVE_BOOST;
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
187
|
+
let delta;
|
|
188
|
+
let minScore;
|
|
189
|
+
if (feedbackHistory) {
|
|
190
|
+
// Ratio-based model — direction from ratio, strength from volume
|
|
191
|
+
const { negativeCount, positiveCount, uniqueClients } = feedbackHistory;
|
|
192
|
+
// Add the current event to history for calculation
|
|
193
|
+
const adjNeg = isNegative ? negativeCount + 1 : negativeCount;
|
|
194
|
+
const adjPos = isNegative ? positiveCount : positiveCount + 1;
|
|
195
|
+
const total = adjNeg + adjPos;
|
|
196
|
+
const negRatio = total > 0 ? adjNeg / total : 0;
|
|
197
|
+
delta = computeFeedbackDelta(adjNeg, adjPos, uniqueClients);
|
|
198
|
+
// Apply delta relative to provenance base, not current score
|
|
199
|
+
const targetScore = provenanceBase + delta;
|
|
200
|
+
// Move current score toward target (don't jump, converge)
|
|
201
|
+
const moveRate = 0.3; // converge 30% toward target per event
|
|
202
|
+
const newScore = currentScore + (targetScore - currentScore) * moveRate;
|
|
203
|
+
minScore = effectiveFloor(provenanceBase, negRatio, total);
|
|
204
|
+
return {
|
|
205
|
+
newScore: Math.max(minScore, Math.min(maxScore, newScore)),
|
|
206
|
+
label: scoreToLabel(Math.max(minScore, Math.min(maxScore, newScore))),
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
// Legacy flat model for backward compatibility
|
|
211
|
+
delta = isNegative ? FEEDBACK_DELTA_NEGATIVE : FEEDBACK_DELTA_POSITIVE;
|
|
212
|
+
minScore = Math.max(0, provenanceBase - MAX_NEGATIVE_DRIFT);
|
|
213
|
+
const newScore = Math.max(minScore, Math.min(maxScore, currentScore + delta));
|
|
214
|
+
return { newScore, label: scoreToLabel(newScore) };
|
|
215
|
+
}
|
|
95
216
|
}
|
|
96
217
|
/**
|
|
97
218
|
* Feedback signal classification — single source of truth.
|
|
@@ -281,25 +402,16 @@ function buildReport(source, totalEntries, correlated, signalMap) {
|
|
|
281
402
|
const signals = [];
|
|
282
403
|
const lowConfidence = [];
|
|
283
404
|
for (const [docId, signal] of signalMap) {
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
}
|
|
290
|
-
else if (netNegative >= 3) {
|
|
291
|
-
delta = -0.25;
|
|
292
|
-
}
|
|
293
|
-
else if (netNegative >= 1) {
|
|
294
|
-
delta = -0.10;
|
|
295
|
-
}
|
|
296
|
-
else {
|
|
297
|
-
delta = 0;
|
|
298
|
-
}
|
|
299
|
-
// Estimate label from score using "curated" as a conservative baseline.
|
|
405
|
+
// Ratio-based delta — direction from ratio, strength from volume
|
|
406
|
+
const total = signal.negative + signal.positive;
|
|
407
|
+
const negRatio = total > 0 ? signal.negative / total : 0;
|
|
408
|
+
const delta = computeFeedbackDelta(signal.negative, signal.positive);
|
|
409
|
+
// Estimate label using "curated" baseline + adaptive floor.
|
|
300
410
|
// This is advisory — applyConfidenceSignals() recomputes with actual provenance.
|
|
301
|
-
|
|
302
|
-
const
|
|
411
|
+
const provenanceBase = PROVENANCE_BASE_SCORES["curated"];
|
|
412
|
+
const maxScore = provenanceBase + MAX_POSITIVE_BOOST;
|
|
413
|
+
const minScore = effectiveFloor(provenanceBase, negRatio, total);
|
|
414
|
+
const estimatedScore = Math.max(minScore, Math.min(maxScore, provenanceBase + delta));
|
|
303
415
|
const label = scoreToLabel(estimatedScore);
|
|
304
416
|
if (label === "low-confidence") {
|
|
305
417
|
lowConfidence.push(docId);
|
|
@@ -478,9 +478,22 @@ async function searchDirect(query, options) {
|
|
|
478
478
|
// Dynamic domain boost — if query signals a specific platform, boost its domain
|
|
479
479
|
// and demote the other. DE serves both platforms; this keeps results focused.
|
|
480
480
|
const queryBoost = buildQueryBoostSpec(query, filters);
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
481
|
+
// Confidence boost — always applied. Verified docs rank higher, low-confidence lower.
|
|
482
|
+
// This makes the feedback loop visible at search time: downgraded docs get demoted
|
|
483
|
+
// regardless of relevance. DE boost values are additive to relevance score.
|
|
484
|
+
// Values calibrated against signal viewer: semantic relevance spreads 0.07-0.99,
|
|
485
|
+
// so boosts must be large enough to move docs across that range.
|
|
486
|
+
const confidenceBoosts = [
|
|
487
|
+
{ condition: 'confidence: ANY("verified")', boost: 0.5 },
|
|
488
|
+
{ condition: 'confidence: ANY("inferred")', boost: -0.2 },
|
|
489
|
+
{ condition: 'confidence: ANY("low-confidence")', boost: -0.8 },
|
|
490
|
+
];
|
|
491
|
+
const querySpecs = (queryBoost?.conditionBoostSpecs ?? []);
|
|
492
|
+
const allBoosts = [
|
|
493
|
+
...querySpecs,
|
|
494
|
+
...confidenceBoosts,
|
|
495
|
+
];
|
|
496
|
+
body.boostSpec = { conditionBoostSpecs: allBoosts };
|
|
484
497
|
// Always request snippets — works with chunked datastores.
|
|
485
498
|
// (Extractive answers do NOT work with chunking, only snippets.)
|
|
486
499
|
// For answer mode, also request summary with citations.
|
|
@@ -640,6 +653,8 @@ export async function browseDocuments(options = {}) {
|
|
|
640
653
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
641
654
|
// User Event Tracking
|
|
642
655
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
656
|
+
/** Counters for UserEvent pipeline health — exposed via feedback(method="analyze"). */
|
|
657
|
+
export const userEventCounters = { sent: 0, failed: 0 };
|
|
643
658
|
export async function writeUserEvent(event) {
|
|
644
659
|
if (!isVertexEventsEnabled())
|
|
645
660
|
return;
|
|
@@ -648,15 +663,24 @@ export async function writeUserEvent(event) {
|
|
|
648
663
|
if (!headers)
|
|
649
664
|
return;
|
|
650
665
|
try {
|
|
651
|
-
await fetch(`${de.baseUrl}/${de.datastorePath}/userEvents:write`, {
|
|
666
|
+
const resp = await fetch(`${de.baseUrl}/${de.datastorePath}/userEvents:write`, {
|
|
652
667
|
method: "POST",
|
|
653
668
|
headers,
|
|
654
669
|
body: JSON.stringify(event),
|
|
655
670
|
signal: AbortSignal.timeout(5_000),
|
|
656
671
|
});
|
|
672
|
+
if (!resp.ok) {
|
|
673
|
+
userEventCounters.failed++;
|
|
674
|
+
const detail = await resp.text().catch(() => "");
|
|
675
|
+
console.error(`[SEARCH-CLIENT] UserEvent write failed: ${resp.status} — ${detail.slice(0, 200)}`);
|
|
676
|
+
}
|
|
677
|
+
else {
|
|
678
|
+
userEventCounters.sent++;
|
|
679
|
+
}
|
|
657
680
|
}
|
|
658
681
|
catch {
|
|
659
|
-
|
|
682
|
+
userEventCounters.failed++;
|
|
683
|
+
// Fire-and-forget — timeout or network error
|
|
660
684
|
}
|
|
661
685
|
}
|
|
662
686
|
/**
|
|
@@ -38,8 +38,9 @@ export function getSearchBackend() {
|
|
|
38
38
|
export function isDiscoveryEngineEnabled() {
|
|
39
39
|
return getSearchBackend() === "discovery-engine";
|
|
40
40
|
}
|
|
41
|
+
/** UserEvent tracking is ON by default. Set EMA_VERTEX_EVENTS=false to opt out. */
|
|
41
42
|
export function isVertexEventsEnabled() {
|
|
42
|
-
return process.env.EMA_VERTEX_EVENTS?.trim().toLowerCase()
|
|
43
|
+
return process.env.EMA_VERTEX_EVENTS?.trim().toLowerCase() !== "false";
|
|
43
44
|
}
|
|
44
45
|
export function getDeConfig() {
|
|
45
46
|
const project = process.env.EMA_GCP_PROJECT?.trim() || DEFAULT_PROJECT;
|
|
@@ -35,7 +35,7 @@ export const WORKFLOW_DEF_SCHEMA = {
|
|
|
35
35
|
namespaces: {
|
|
36
36
|
type: "array",
|
|
37
37
|
items: { type: "string" },
|
|
38
|
-
description: "Namespace path
|
|
38
|
+
description: "Namespace path — MUST be copied exactly from workflow(mode='get') response. Do NOT construct manually.",
|
|
39
39
|
},
|
|
40
40
|
name: {
|
|
41
41
|
type: "string",
|
|
@@ -149,8 +149,8 @@ function validateEnumTypes(wf, issues) {
|
|
|
149
149
|
}
|
|
150
150
|
enumNames.add(name.name);
|
|
151
151
|
}
|
|
152
|
-
// Validate options array
|
|
153
|
-
const options = et.options;
|
|
152
|
+
// Validate options/values array — proto uses "options", compiled proto uses "values"
|
|
153
|
+
const options = (et.options ?? et.values);
|
|
154
154
|
if (!Array.isArray(options) || options.length === 0) {
|
|
155
155
|
issues.push({
|
|
156
156
|
path: `${prefix}.options`,
|
|
@@ -40,16 +40,17 @@ export function classifyResult(result, unfilteredCount) {
|
|
|
40
40
|
return "error_500";
|
|
41
41
|
return "error";
|
|
42
42
|
}
|
|
43
|
-
// Success shapes
|
|
43
|
+
// Success shapes — order matters: check deploy before created,
|
|
44
|
+
// because deploy results also carry persona_id but aren't "created".
|
|
45
|
+
if (result.deployed === true || result.workflow_deployed === true || (result.mode === "deploy" && status === "deployed")) {
|
|
46
|
+
return "deployed";
|
|
47
|
+
}
|
|
44
48
|
if (result.success === true || result.persona_id) {
|
|
45
49
|
// Created entity
|
|
46
50
|
if (result.persona_id && !result.workflow_def) {
|
|
47
51
|
return "created";
|
|
48
52
|
}
|
|
49
53
|
}
|
|
50
|
-
if (result.deployed === true || (result.mode === "deploy" && !error)) {
|
|
51
|
-
return "deployed";
|
|
52
|
-
}
|
|
53
54
|
// List shapes — check count
|
|
54
55
|
const count = typeof result.count === "number" ? result.count : undefined;
|
|
55
56
|
if (count !== undefined) {
|
|
@@ -30,7 +30,8 @@ export function getDefaultGuidance(shape, ctx) {
|
|
|
30
30
|
};
|
|
31
31
|
case "deployed":
|
|
32
32
|
return {
|
|
33
|
-
_next_step: "
|
|
33
|
+
_next_step: "Test your deployed workflow: conversation(method='create', persona_id='{persona_id}') for chat, or upload documents via persona(id='{persona_id}', data={method:'upload', path:'/path/to/doc.pdf'}) for dashboard.",
|
|
34
|
+
_tip: "Deployed successfully. The workflow is now active.",
|
|
34
35
|
};
|
|
35
36
|
case "deploy_failed":
|
|
36
37
|
return {
|
package/dist/mcp/guidance.js
CHANGED
|
@@ -119,6 +119,7 @@ function generateDecisionFlow(tools) {
|
|
|
119
119
|
2. \`knowledge("workflow patterns for <your use case>")\` → learn the correct workflow pattern
|
|
120
120
|
3. \`${createPersona}\` → creates persona
|
|
121
121
|
4. \`${getWorkflow}\` → get starter workflow + generation schema (FULL input/output specs from API) + fingerprint
|
|
122
|
+
Use \`compact=true\` for a smaller response (workflowName + fingerprint + workflow_def only, no schema).
|
|
122
123
|
5. Build a complete workflow_def using the generation schema — it shows ALL required inputs per action
|
|
123
124
|
6. Upload data sources if needed — \`persona(id="<new_id>", data={method:"upload", path:"/path/to/doc.pdf"})\`
|
|
124
125
|
7. \`workflow(mode="validate", persona_id="...", workflow_def={...})\` → catch errors BEFORE deploying
|
|
@@ -131,7 +132,7 @@ function generateDecisionFlow(tools) {
|
|
|
131
132
|
const get = opExample("workflow", "Get");
|
|
132
133
|
const deploy = opExample("workflow", "Deploy");
|
|
133
134
|
sections.push(`**Modifying an existing AI Employee's workflow?**
|
|
134
|
-
1. \`${get}\` → get current workflow_def + schema + fingerprint
|
|
135
|
+
1. \`${get}\` → get current workflow_def + schema + fingerprint (use \`compact=true\` for smaller response)
|
|
135
136
|
2. LLM modifies the workflow_def JSON (use the returned workflow_def as format reference)
|
|
136
137
|
3. \`workflow(mode="validate", persona_id="...", workflow_def={...})\` → catch errors before deploying
|
|
137
138
|
4. \`${deploy}\``);
|
|
@@ -16,6 +16,9 @@ import { submitFeedback, listFeedback, listTelemetry, analyzeFeedback, rotateLog
|
|
|
16
16
|
import { markProbeResponded } from "./probes.js";
|
|
17
17
|
import { appendToOutbox, flushOutbox, getOutboxStats, readLocalMessages } from "./outbox.js";
|
|
18
18
|
import { isRemoteEnabled } from "./remote-store.js";
|
|
19
|
+
import { writeUserEvent } from "../../../knowledge/search-client.js";
|
|
20
|
+
import { getOrCreateClientId } from "./client-id.js";
|
|
21
|
+
import { getAttributionToken } from "../knowledge/session-state.js";
|
|
19
22
|
import { analyzeGlobal } from "./global-analysis.js";
|
|
20
23
|
import { TOOLKIT_VERSION } from "../env/config.js";
|
|
21
24
|
const VALID_CATEGORIES = ALL_CATEGORIES;
|
|
@@ -141,6 +144,35 @@ async function handleSubmit(args) {
|
|
|
141
144
|
// Best-effort — don't block feedback submission
|
|
142
145
|
}
|
|
143
146
|
}
|
|
147
|
+
// UserEvent emission: fire DE conversion/view-item for positive feedback with knowledge_ref.
|
|
148
|
+
// Independent of confidence loop — no guards, no cooldown. Fire-and-forget.
|
|
149
|
+
if (knowledgeRef) {
|
|
150
|
+
const isSuccess = category === "success";
|
|
151
|
+
const isHighQuality = category === "quality"
|
|
152
|
+
&& (qualityData?.accuracy ?? 0) >= 4
|
|
153
|
+
&& (qualityData?.usefulness ?? 0) >= 4;
|
|
154
|
+
const isInteraction = category === "interaction";
|
|
155
|
+
if (isSuccess || isHighQuality || isInteraction) {
|
|
156
|
+
const conversionType = isSuccess ? "knowledge-success"
|
|
157
|
+
: isHighQuality ? "knowledge-quality-high"
|
|
158
|
+
: undefined; // interaction → view-item, no conversionType
|
|
159
|
+
getOrCreateClientId()
|
|
160
|
+
.then((clientId) => {
|
|
161
|
+
const token = getAttributionToken(knowledgeRef);
|
|
162
|
+
writeUserEvent({
|
|
163
|
+
eventType: conversionType ? "conversion" : "view-item",
|
|
164
|
+
userPseudoId: clientId,
|
|
165
|
+
...(token ? { attributionToken: token } : {}),
|
|
166
|
+
documents: [{
|
|
167
|
+
id: knowledgeRef,
|
|
168
|
+
...(conversionType ? { conversionValue: isSuccess ? 1.0 : 0.8 } : {}),
|
|
169
|
+
}],
|
|
170
|
+
...(conversionType ? { conversionType } : {}),
|
|
171
|
+
}).catch(() => { });
|
|
172
|
+
})
|
|
173
|
+
.catch(() => { });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
144
176
|
return {
|
|
145
177
|
success: true,
|
|
146
178
|
feedback_id: entry.id,
|
|
@@ -12,6 +12,7 @@ import { promises as fs } from "node:fs";
|
|
|
12
12
|
import { join } from "node:path";
|
|
13
13
|
import { randomUUID } from "node:crypto";
|
|
14
14
|
import { getToolkitRoot } from "../../../sdk/paths.js";
|
|
15
|
+
import { userEventCounters } from "../../../knowledge/search-client.js";
|
|
15
16
|
import { appendToOutbox } from "./outbox.js";
|
|
16
17
|
import { isRemoteEnabled } from "./remote-store.js";
|
|
17
18
|
import { SESSION_ID } from "./session.js";
|
|
@@ -384,6 +385,8 @@ export async function analyzeFeedback(rootOverride) {
|
|
|
384
385
|
qualityEntries.reduce((sum, e) => sum + (e.quality_data.accuracy ?? 0), 0) /
|
|
385
386
|
qualityEntries.length;
|
|
386
387
|
}
|
|
388
|
+
// UserEvent pipeline counters (in-memory, this session only)
|
|
389
|
+
const hasEventActivity = userEventCounters.sent > 0 || userEventCounters.failed > 0;
|
|
387
390
|
return {
|
|
388
391
|
summary: {
|
|
389
392
|
total_feedback: feedback.length,
|
|
@@ -394,6 +397,7 @@ export async function analyzeFeedback(rootOverride) {
|
|
|
394
397
|
telemetry_period: telemetry.length > 0
|
|
395
398
|
? { from: telemetry[0].ts, to: telemetry[telemetry.length - 1].ts }
|
|
396
399
|
: null,
|
|
400
|
+
...(hasEventActivity ? { user_events: { ...userEventCounters } } : {}),
|
|
397
401
|
},
|
|
398
402
|
category_breakdown: categoryBreakdown,
|
|
399
403
|
hot_spots: {
|