groove-dev 0.27.102 → 0.27.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +0 -7
- package/moe-training/client/domain-tagger.js +205 -0
- package/moe-training/client/edit-normalizer.js +188 -0
- package/moe-training/client/envelope-builder.js +1 -1
- package/moe-training/client/parsers/claude-code.js +56 -9
- package/moe-training/client/parsers/codex.js +25 -5
- package/moe-training/client/parsers/gemini.js +21 -2
- package/moe-training/client/parsers/grok.js +18 -0
- package/moe-training/client/trajectory-capture.js +95 -3
- package/moe-training/server/routes/ingest.js +26 -0
- package/moe-training/server/verifier.js +34 -0
- package/moe-training/shared/constants.js +9 -0
- package/moe-training/shared/envelope-schema.js +128 -2
- package/moe-training/test/client/domain-tagger.test.js +203 -0
- package/moe-training/test/client/edit-normalizer.test.js +376 -0
- package/moe-training/test/client/envelope-builder.test.js +28 -0
- package/moe-training/test/client/parsers/claude-code.test.js +248 -38
- package/moe-training/test/client/parsers/codex.test.js +2 -0
- package/moe-training/test/client/parsers/gemini.test.js +2 -0
- package/moe-training/test/client/trajectory-capture.test.js +345 -0
- package/moe-training/test/server/verifier.test.js +94 -0
- package/moe-training/test/shared/envelope-schema.test.js +291 -0
- package/node_modules/@groove-dev/cli/package.json +1 -1
- package/node_modules/@groove-dev/daemon/package.json +1 -1
- package/node_modules/@groove-dev/gui/package.json +1 -1
- package/node_modules/moe-training/client/domain-tagger.js +205 -0
- package/node_modules/moe-training/client/edit-normalizer.js +188 -0
- package/node_modules/moe-training/client/envelope-builder.js +1 -1
- package/node_modules/moe-training/client/parsers/claude-code.js +56 -9
- package/node_modules/moe-training/client/parsers/codex.js +25 -5
- package/node_modules/moe-training/client/parsers/gemini.js +21 -2
- package/node_modules/moe-training/client/parsers/grok.js +18 -0
- package/node_modules/moe-training/client/trajectory-capture.js +95 -3
- package/node_modules/moe-training/server/routes/ingest.js +26 -0
- package/node_modules/moe-training/server/verifier.js +34 -0
- package/node_modules/moe-training/shared/constants.js +9 -0
- package/node_modules/moe-training/shared/envelope-schema.js +128 -2
- package/node_modules/moe-training/test/client/domain-tagger.test.js +203 -0
- package/node_modules/moe-training/test/client/edit-normalizer.test.js +376 -0
- package/node_modules/moe-training/test/client/envelope-builder.test.js +28 -0
- package/node_modules/moe-training/test/client/parsers/claude-code.test.js +248 -38
- package/node_modules/moe-training/test/client/parsers/codex.test.js +2 -0
- package/node_modules/moe-training/test/client/parsers/gemini.test.js +2 -0
- package/node_modules/moe-training/test/client/trajectory-capture.test.js +345 -0
- package/node_modules/moe-training/test/server/verifier.test.js +94 -0
- package/node_modules/moe-training/test/shared/envelope-schema.test.js +291 -0
- package/package.json +1 -1
- package/packages/cli/package.json +1 -1
- package/packages/daemon/package.json +1 -1
- package/packages/gui/package.json +1 -1
|
@@ -8,7 +8,16 @@ import { StepClassifier } from './step-classifier.js';
|
|
|
8
8
|
import { EnvelopeBuilder } from './envelope-builder.js';
|
|
9
9
|
import { SessionAttestation } from './session-attestation.js';
|
|
10
10
|
import { TransmissionQueue } from './transmission-queue.js';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
CHUNK_TIMEOUT_MS,
|
|
13
|
+
CENTRAL_COMMAND_URL,
|
|
14
|
+
TIER_A_MIN_QUALITY,
|
|
15
|
+
TIER_B_MIN_QUALITY,
|
|
16
|
+
TRAINING_MIN_STEPS,
|
|
17
|
+
TRAINING_MIN_TOKENS,
|
|
18
|
+
TRAINING_MIN_DURATION,
|
|
19
|
+
TRAINING_EXCLUSION_REASONS,
|
|
20
|
+
} from '../shared/constants.js';
|
|
12
21
|
|
|
13
22
|
const OFFLINE_RETRY_INTERVAL_MS = 60_000;
|
|
14
23
|
|
|
@@ -56,6 +65,7 @@ export class TrajectoryCapture {
|
|
|
56
65
|
team_size: teamSize || 1,
|
|
57
66
|
session_quality: 0,
|
|
58
67
|
groove_version: this._grooveVersion,
|
|
68
|
+
leaf_context: null,
|
|
59
69
|
};
|
|
60
70
|
|
|
61
71
|
const builder = new EnvelopeBuilder(sessionId, contributorId, metadata);
|
|
@@ -78,6 +88,7 @@ export class TrajectoryCapture {
|
|
|
78
88
|
startTime,
|
|
79
89
|
chunkTimer: null,
|
|
80
90
|
allSteps: [],
|
|
91
|
+
revisionRounds: 0,
|
|
81
92
|
};
|
|
82
93
|
|
|
83
94
|
ctx.chunkTimer = setInterval(() => {
|
|
@@ -126,6 +137,8 @@ export class TrajectoryCapture {
|
|
|
126
137
|
const ctx = this._contexts.get(agentId);
|
|
127
138
|
if (!ctx) return;
|
|
128
139
|
|
|
140
|
+
ctx.revisionRounds++;
|
|
141
|
+
|
|
129
142
|
const classified = ctx.classifier.classifyUserMessage(text);
|
|
130
143
|
if (!classified) return;
|
|
131
144
|
|
|
@@ -238,22 +251,34 @@ export class TrajectoryCapture {
|
|
|
238
251
|
|
|
239
252
|
ctx.metadata.session_quality = this._computeQuality(ctx);
|
|
240
253
|
|
|
254
|
+
const userInterventions = StepClassifier.countUserInterventions(ctx.allSteps);
|
|
255
|
+
const durationSeconds = Math.round((Date.now() - ctx.startTime) / 1000);
|
|
256
|
+
|
|
257
|
+
const { tier, reason: tierReason } = this._computeQualityTier(ctx, status, userInterventions);
|
|
258
|
+
const { eligible, exclusionReason } = this._computeTrainingEligibility(ctx, durationSeconds);
|
|
259
|
+
|
|
241
260
|
const closeEnvelope = ctx.builder.buildSessionClose({
|
|
242
261
|
status,
|
|
243
262
|
session_quality: ctx.metadata.session_quality,
|
|
244
|
-
|
|
263
|
+
quality_tier: tier,
|
|
264
|
+
quality_tier_reason: tierReason,
|
|
265
|
+
user_interventions: userInterventions,
|
|
245
266
|
total_steps: ctx.stepCount,
|
|
246
267
|
total_chunks: ctx.chunkCount,
|
|
247
268
|
total_tokens: ctx.totalTokens,
|
|
248
|
-
duration_seconds:
|
|
269
|
+
duration_seconds: durationSeconds,
|
|
249
270
|
files_modified: extra?.files_modified || ctx.filesModified,
|
|
250
271
|
errors_encountered: ctx.errorsEncountered,
|
|
251
272
|
errors_recovered: ctx.errorsRecovered,
|
|
252
273
|
coordination_events: ctx.coordinationEvents,
|
|
274
|
+
training_eligible: eligible,
|
|
275
|
+
training_exclusion_reason: exclusionReason,
|
|
253
276
|
});
|
|
254
277
|
|
|
255
278
|
this._signAndTransmit(ctx.sessionId, closeEnvelope);
|
|
256
279
|
|
|
280
|
+
this._emitUserFeedback(ctx, status, userInterventions);
|
|
281
|
+
|
|
257
282
|
try {
|
|
258
283
|
await this._transmissionQueue.waitForDrain();
|
|
259
284
|
} catch {
|
|
@@ -269,6 +294,73 @@ export class TrajectoryCapture {
|
|
|
269
294
|
this._contexts.delete(agentId);
|
|
270
295
|
}
|
|
271
296
|
|
|
297
|
+
_computeQualityTier(ctx, status, userInterventions) {
|
|
298
|
+
const quality = ctx.metadata.session_quality;
|
|
299
|
+
if (quality >= TIER_A_MIN_QUALITY && ctx.errorsEncountered === 0 && userInterventions === 0 && status === 'SUCCESS') {
|
|
300
|
+
return { tier: 'TIER_A', reason: 'high_quality_no_errors' };
|
|
301
|
+
}
|
|
302
|
+
if (status !== 'SUCCESS') {
|
|
303
|
+
return { tier: 'TIER_C', reason: 'non_success_status' };
|
|
304
|
+
}
|
|
305
|
+
if (quality >= TIER_B_MIN_QUALITY || (ctx.errorsEncountered > 0 && ctx.errorsRecovered === ctx.errorsEncountered)) {
|
|
306
|
+
return { tier: 'TIER_B', reason: quality >= TIER_B_MIN_QUALITY ? 'moderate_quality' : 'errors_recovered' };
|
|
307
|
+
}
|
|
308
|
+
return { tier: 'TIER_C', reason: quality < TIER_B_MIN_QUALITY ? 'low_quality' : 'unrecovered_errors' };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
_computeTrainingEligibility(ctx, durationSeconds) {
|
|
312
|
+
if (ctx.stepCount < TRAINING_MIN_STEPS) {
|
|
313
|
+
return { eligible: false, exclusionReason: 'too_few_steps' };
|
|
314
|
+
}
|
|
315
|
+
const hasAction = ctx.allSteps.some((s) => s.type === 'action' && s.tool);
|
|
316
|
+
if (!hasAction) {
|
|
317
|
+
return { eligible: false, exclusionReason: 'no_actions' };
|
|
318
|
+
}
|
|
319
|
+
const hasObservation = ctx.allSteps.some((s) => s.type === 'observation' && s.content);
|
|
320
|
+
if (!hasObservation) {
|
|
321
|
+
return { eligible: false, exclusionReason: 'no_observations' };
|
|
322
|
+
}
|
|
323
|
+
if (ctx.totalTokens < TRAINING_MIN_TOKENS) {
|
|
324
|
+
return { eligible: false, exclusionReason: 'insufficient_tokens' };
|
|
325
|
+
}
|
|
326
|
+
if (durationSeconds < TRAINING_MIN_DURATION) {
|
|
327
|
+
return { eligible: false, exclusionReason: 'too_short' };
|
|
328
|
+
}
|
|
329
|
+
return { eligible: true, exclusionReason: null };
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
_emitUserFeedback(ctx, status, userInterventions) {
|
|
333
|
+
let signal;
|
|
334
|
+
let context;
|
|
335
|
+
|
|
336
|
+
if (status === 'SUCCESS' && userInterventions === 0 && ctx.revisionRounds === 0) {
|
|
337
|
+
signal = 'accepted';
|
|
338
|
+
context = 'session completed successfully with no user interventions';
|
|
339
|
+
} else if (status === 'SUCCESS' && ctx.revisionRounds > 0) {
|
|
340
|
+
signal = 'iterated';
|
|
341
|
+
context = `user requested ${ctx.revisionRounds} revision(s) before completion`;
|
|
342
|
+
} else {
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const feedbackEnvelope = {
|
|
347
|
+
envelope_id: `env_${randomUUID()}`,
|
|
348
|
+
session_id: ctx.sessionId,
|
|
349
|
+
type: 'USER_FEEDBACK',
|
|
350
|
+
attestation: { session_hmac: '', sequence: 0, app_version_hash: '' },
|
|
351
|
+
feedback: {
|
|
352
|
+
signal,
|
|
353
|
+
timestamp: Date.now() / 1000,
|
|
354
|
+
context,
|
|
355
|
+
target_step: ctx.stepCount,
|
|
356
|
+
revision_rounds: ctx.revisionRounds,
|
|
357
|
+
delta_summary: null,
|
|
358
|
+
},
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
this._signAndTransmit(ctx.sessionId, feedbackEnvelope);
|
|
362
|
+
}
|
|
363
|
+
|
|
272
364
|
async _retryOfflineQueue() {
|
|
273
365
|
if (!this._enabled || !this._transmissionQueue || this._transmissionQueue.offlineQueueSize === 0) return;
|
|
274
366
|
try {
|
|
@@ -15,6 +15,32 @@ export function createIngestRoutes(verifier, storage, stitcher, scorer, enrichme
|
|
|
15
15
|
return res.status(400).json({ accepted: false, reason: 'malformed request: missing envelope or session_id' });
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
if (envelope.type === 'USER_FEEDBACK') {
|
|
19
|
+
const feedbackResult = verifier.verifyFeedback(envelope);
|
|
20
|
+
if (!feedbackResult.valid) {
|
|
21
|
+
return res.json({ accepted: false, reason: feedbackResult.reason });
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const envelopeId = `env_${uuidv4()}`;
|
|
25
|
+
envelope.envelope_id = envelopeId;
|
|
26
|
+
|
|
27
|
+
if (sessionRegistry.isEnvelopeProcessed(envelopeId)) {
|
|
28
|
+
return res.json({ accepted: false, reason: 'duplicate envelope' });
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
try {
|
|
32
|
+
storage.store(envelope);
|
|
33
|
+
} catch (err) {
|
|
34
|
+
if (err.message === 'STORAGE_QUOTA_EXCEEDED') {
|
|
35
|
+
return res.status(507).json({ accepted: false, reason: 'storage quota exceeded' });
|
|
36
|
+
}
|
|
37
|
+
throw err;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
sessionRegistry.recordProcessedEnvelope(envelopeId, envelope.session_id);
|
|
41
|
+
return res.json({ accepted: true, envelope_id: envelopeId });
|
|
42
|
+
}
|
|
43
|
+
|
|
18
44
|
if (envelope.type === 'SESSION_CLOSE') {
|
|
19
45
|
const closeResult = verifier.verifyClose(envelope);
|
|
20
46
|
if (!closeResult.valid) {
|
|
@@ -99,4 +99,38 @@ export class EnvelopeVerifier {
|
|
|
99
99
|
this.sessionRegistry.closeSession(sessionId);
|
|
100
100
|
return { valid: true };
|
|
101
101
|
}
|
|
102
|
+
|
|
103
|
+
verifyFeedback(envelope) {
|
|
104
|
+
const sessionId = envelope.session_id;
|
|
105
|
+
if (!sessionId) return { valid: false, reason: 'missing session_id' };
|
|
106
|
+
|
|
107
|
+
const session = this.sessionRegistry.getSession(sessionId);
|
|
108
|
+
if (!session) return { valid: false, reason: 'unknown session_id' };
|
|
109
|
+
|
|
110
|
+
const attestation = envelope.attestation;
|
|
111
|
+
if (!attestation) return { valid: false, reason: 'missing attestation' };
|
|
112
|
+
|
|
113
|
+
if (!attestation.session_hmac || typeof attestation.session_hmac !== 'string' || attestation.session_hmac.length === 0) {
|
|
114
|
+
return { valid: false, reason: 'empty or missing HMAC' };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const envelopeForHmac = { ...envelope };
|
|
118
|
+
delete envelopeForHmac.attestation;
|
|
119
|
+
const envelopeBytes = JSON.stringify(envelopeForHmac);
|
|
120
|
+
|
|
121
|
+
const hmacValid = verifyEnvelope(
|
|
122
|
+
session.shared_secret,
|
|
123
|
+
envelopeBytes,
|
|
124
|
+
attestation.sequence,
|
|
125
|
+
attestation.session_hmac
|
|
126
|
+
);
|
|
127
|
+
if (!hmacValid) return { valid: false, reason: 'HMAC verification failed' };
|
|
128
|
+
|
|
129
|
+
const schemaResult = validateEnvelope(envelope);
|
|
130
|
+
if (!schemaResult.valid) {
|
|
131
|
+
return { valid: false, reason: `schema validation failed: ${schemaResult.reason || schemaResult.errors?.join(', ')}` };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return { valid: true };
|
|
135
|
+
}
|
|
102
136
|
}
|
|
@@ -13,6 +13,7 @@ export const MODEL_TIERS = {
|
|
|
13
13
|
'claude-opus-4-7': 5,
|
|
14
14
|
'claude-sonnet-4-6': 3,
|
|
15
15
|
'gpt-4.5': 5,
|
|
16
|
+
'gpt-5.5': 5,
|
|
16
17
|
'o3': 5,
|
|
17
18
|
'o4-mini': 2,
|
|
18
19
|
'gemini-2.5-pro': 3,
|
|
@@ -27,4 +28,12 @@ export const QUALITY_MULTIPLIERS = {
|
|
|
27
28
|
highQuality: 1.5,
|
|
28
29
|
};
|
|
29
30
|
|
|
31
|
+
export const OBSERVATION_TOKEN_LIMIT = 4096;
|
|
32
|
+
export const TIER_A_MIN_QUALITY = 70;
|
|
33
|
+
export const TIER_B_MIN_QUALITY = 50;
|
|
34
|
+
export const TRAINING_MIN_STEPS = 5;
|
|
35
|
+
export const TRAINING_MIN_TOKENS = 500;
|
|
36
|
+
export const TRAINING_MIN_DURATION = 10;
|
|
37
|
+
export const TRAINING_EXCLUSION_REASONS = ['too_few_steps', 'no_actions', 'no_observations', 'insufficient_tokens', 'too_short'];
|
|
38
|
+
|
|
30
39
|
export const CENTRAL_COMMAND_URL = process.env.GROOVE_CENTRAL_URL || 'https://api.groovedev.ai';
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
// FSL-1.1-Apache-2.0 — see LICENSE
|
|
2
2
|
|
|
3
|
-
import { SUPPORTED_PROVIDERS, MODEL_TIERS } from './constants.js';
|
|
3
|
+
import { SUPPORTED_PROVIDERS, MODEL_TIERS, TRAINING_EXCLUSION_REASONS } from './constants.js';
|
|
4
4
|
|
|
5
|
-
export const STEP_TYPES = ['thought', 'action', 'observation', 'correction', 'resolution', 'error', 'coordination'];
|
|
5
|
+
export const STEP_TYPES = ['thought', 'action', 'observation', 'correction', 'resolution', 'error', 'coordination', 'edit'];
|
|
6
|
+
const VALID_QUALITY_TIERS = ['TIER_A', 'TIER_B', 'TIER_C'];
|
|
7
|
+
const VALID_FEEDBACK_SIGNALS = ['accepted', 'modified', 'rejected', 'iterated'];
|
|
6
8
|
|
|
7
9
|
const VALID_MODEL_ENGINES = Object.keys(MODEL_TIERS);
|
|
8
10
|
const VALID_COMPLEXITIES = ['light', 'medium', 'heavy'];
|
|
@@ -61,6 +63,10 @@ export function validateEnvelope(envelope) {
|
|
|
61
63
|
return validateSessionClose(envelope);
|
|
62
64
|
}
|
|
63
65
|
|
|
66
|
+
if (envelope.type === 'USER_FEEDBACK') {
|
|
67
|
+
return validateUserFeedback(envelope);
|
|
68
|
+
}
|
|
69
|
+
|
|
64
70
|
if (!envelope.session_id || typeof envelope.session_id !== 'string') {
|
|
65
71
|
errors.push('Missing or invalid session_id');
|
|
66
72
|
}
|
|
@@ -123,6 +129,43 @@ export function validateEnvelope(envelope) {
|
|
|
123
129
|
}
|
|
124
130
|
}
|
|
125
131
|
// session_quality is ignored from client — server derives quality
|
|
132
|
+
if (m.domain_tags !== undefined && m.domain_tags !== null) {
|
|
133
|
+
if (typeof m.domain_tags !== 'object') {
|
|
134
|
+
errors.push('metadata.domain_tags must be an object or null');
|
|
135
|
+
} else {
|
|
136
|
+
for (const level of ['primary', 'secondary', 'tertiary']) {
|
|
137
|
+
const tag = m.domain_tags[level];
|
|
138
|
+
if (!tag || typeof tag !== 'object') {
|
|
139
|
+
errors.push(`metadata.domain_tags.${level} must be an object`);
|
|
140
|
+
} else {
|
|
141
|
+
if (typeof tag.domain !== 'string' || tag.domain.length === 0) {
|
|
142
|
+
errors.push(`metadata.domain_tags.${level}.domain must be a non-empty string`);
|
|
143
|
+
}
|
|
144
|
+
if (typeof tag.confidence !== 'number' || tag.confidence < 0 || tag.confidence > 1) {
|
|
145
|
+
errors.push(`metadata.domain_tags.${level}.confidence must be a number 0-1`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (m.leaf_context !== undefined && m.leaf_context !== null) {
|
|
152
|
+
if (typeof m.leaf_context !== 'object') {
|
|
153
|
+
errors.push('metadata.leaf_context must be an object or null');
|
|
154
|
+
} else {
|
|
155
|
+
if (!m.leaf_context.leaf_id || typeof m.leaf_context.leaf_id !== 'string') {
|
|
156
|
+
errors.push('metadata.leaf_context.leaf_id must be a non-empty string');
|
|
157
|
+
}
|
|
158
|
+
if (!m.leaf_context.leaf_version || typeof m.leaf_context.leaf_version !== 'string') {
|
|
159
|
+
errors.push('metadata.leaf_context.leaf_version must be a non-empty string');
|
|
160
|
+
}
|
|
161
|
+
if (typeof m.leaf_context.confidence_at_route !== 'number' || m.leaf_context.confidence_at_route < 0 || m.leaf_context.confidence_at_route > 1) {
|
|
162
|
+
errors.push('metadata.leaf_context.confidence_at_route must be a number 0-1');
|
|
163
|
+
}
|
|
164
|
+
if (!m.leaf_context.chassis_model || typeof m.leaf_context.chassis_model !== 'string') {
|
|
165
|
+
errors.push('metadata.leaf_context.chassis_model must be a non-empty string');
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
126
169
|
}
|
|
127
170
|
|
|
128
171
|
// Trajectory log validation
|
|
@@ -161,6 +204,14 @@ export function validateEnvelope(envelope) {
|
|
|
161
204
|
errors.push(`step.token_count must be 0-${MAX_TOKEN_COUNT} at index ${i}`);
|
|
162
205
|
}
|
|
163
206
|
}
|
|
207
|
+
if (step.truncated !== undefined && typeof step.truncated !== 'boolean') {
|
|
208
|
+
errors.push(`step.truncated must be a boolean at index ${i}`);
|
|
209
|
+
}
|
|
210
|
+
if (step.original_token_count !== undefined) {
|
|
211
|
+
if (typeof step.original_token_count !== 'number' || step.original_token_count < 0 || step.original_token_count > MAX_TOKEN_COUNT) {
|
|
212
|
+
errors.push(`step.original_token_count must be 0-${MAX_TOKEN_COUNT} at index ${i}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
164
215
|
}
|
|
165
216
|
}
|
|
166
217
|
|
|
@@ -214,6 +265,81 @@ function validateSessionClose(envelope) {
|
|
|
214
265
|
}
|
|
215
266
|
}
|
|
216
267
|
}
|
|
268
|
+
if (envelope.outcome.quality_tier !== undefined && envelope.outcome.quality_tier !== null) {
|
|
269
|
+
if (!VALID_QUALITY_TIERS.includes(envelope.outcome.quality_tier)) {
|
|
270
|
+
errors.push(`outcome.quality_tier must be one of: ${VALID_QUALITY_TIERS.join(', ')}`);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
if (envelope.outcome.quality_tier_reason !== undefined && envelope.outcome.quality_tier_reason !== null) {
|
|
274
|
+
if (typeof envelope.outcome.quality_tier_reason !== 'string' || envelope.outcome.quality_tier_reason.length > 200) {
|
|
275
|
+
errors.push('outcome.quality_tier_reason must be a string, max 200 characters');
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
if (envelope.outcome.training_eligible !== undefined && envelope.outcome.training_eligible !== null) {
|
|
279
|
+
if (typeof envelope.outcome.training_eligible !== 'boolean') {
|
|
280
|
+
errors.push('outcome.training_eligible must be a boolean');
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
if (envelope.outcome.training_exclusion_reason !== undefined && envelope.outcome.training_exclusion_reason !== null) {
|
|
284
|
+
if (!TRAINING_EXCLUSION_REASONS.includes(envelope.outcome.training_exclusion_reason)) {
|
|
285
|
+
errors.push(`outcome.training_exclusion_reason must be one of: ${TRAINING_EXCLUSION_REASONS.join(', ')}`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
return { valid: errors.length === 0, errors };
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function validateUserFeedback(envelope) {
|
|
294
|
+
const errors = [];
|
|
295
|
+
|
|
296
|
+
if (!envelope.session_id || typeof envelope.session_id !== 'string') {
|
|
297
|
+
errors.push('Missing or invalid session_id');
|
|
298
|
+
}
|
|
299
|
+
if (envelope.type !== 'USER_FEEDBACK') {
|
|
300
|
+
errors.push('Invalid type for USER_FEEDBACK envelope');
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
if (!envelope.attestation || typeof envelope.attestation !== 'object') {
|
|
304
|
+
errors.push('Missing attestation');
|
|
305
|
+
} else {
|
|
306
|
+
if (typeof envelope.attestation.session_hmac !== 'string' || !HEX_64.test(envelope.attestation.session_hmac)) {
|
|
307
|
+
errors.push('attestation.session_hmac must be exactly 64 hex characters');
|
|
308
|
+
}
|
|
309
|
+
if (typeof envelope.attestation.sequence !== 'number' || !Number.isInteger(envelope.attestation.sequence) || envelope.attestation.sequence < 0 || envelope.attestation.sequence > 1_000_000) {
|
|
310
|
+
errors.push('attestation.sequence must be a non-negative integer, max 1000000');
|
|
311
|
+
}
|
|
312
|
+
if (typeof envelope.attestation.app_version_hash !== 'string' || !HEX_64.test(envelope.attestation.app_version_hash)) {
|
|
313
|
+
errors.push('attestation.app_version_hash must be exactly 64 hex characters');
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
if (!envelope.feedback || typeof envelope.feedback !== 'object') {
|
|
318
|
+
errors.push('Missing feedback');
|
|
319
|
+
} else {
|
|
320
|
+
const f = envelope.feedback;
|
|
321
|
+
if (!VALID_FEEDBACK_SIGNALS.includes(f.signal)) {
|
|
322
|
+
errors.push(`feedback.signal must be one of: ${VALID_FEEDBACK_SIGNALS.join(', ')}`);
|
|
323
|
+
}
|
|
324
|
+
if (typeof f.timestamp !== 'number') {
|
|
325
|
+
errors.push('feedback.timestamp must be a number');
|
|
326
|
+
}
|
|
327
|
+
if (f.context !== undefined && f.context !== null && typeof f.context !== 'string') {
|
|
328
|
+
errors.push('feedback.context must be a string or null');
|
|
329
|
+
}
|
|
330
|
+
if (f.target_step !== undefined && f.target_step !== null) {
|
|
331
|
+
if (typeof f.target_step !== 'number' || !Number.isInteger(f.target_step) || f.target_step < 0) {
|
|
332
|
+
errors.push('feedback.target_step must be a non-negative integer');
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
if (f.revision_rounds !== undefined && f.revision_rounds !== null) {
|
|
336
|
+
if (typeof f.revision_rounds !== 'number' || !Number.isInteger(f.revision_rounds) || f.revision_rounds < 0) {
|
|
337
|
+
errors.push('feedback.revision_rounds must be a non-negative integer');
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
if (f.delta_summary !== undefined && f.delta_summary !== null && typeof f.delta_summary !== 'string') {
|
|
341
|
+
errors.push('feedback.delta_summary must be a string or null');
|
|
342
|
+
}
|
|
217
343
|
}
|
|
218
344
|
|
|
219
345
|
return { valid: errors.length === 0, errors };
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
// FSL-1.1-Apache-2.0 — see LICENSE
|
|
2
|
+
|
|
3
|
+
import { describe, it, beforeEach } from 'node:test';
|
|
4
|
+
import assert from 'node:assert/strict';
|
|
5
|
+
import { DomainTagger, cosineSimilarity } from '../../client/domain-tagger.js';
|
|
6
|
+
|
|
7
|
+
describe('DomainTagger', () => {
|
|
8
|
+
let tagger;
|
|
9
|
+
|
|
10
|
+
beforeEach(async () => {
|
|
11
|
+
tagger = new DomainTagger();
|
|
12
|
+
await tagger.init();
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it('initializes in keyword mode when no embedding service is configured', async () => {
|
|
16
|
+
assert.equal(tagger.ready, true);
|
|
17
|
+
assert.equal(tagger.mode, 'keyword');
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it('tags Python-related routing text', async () => {
|
|
21
|
+
const text = DomainTagger.buildRoutingText(
|
|
22
|
+
'Fix Python unit tests',
|
|
23
|
+
'The pytest suite is failing on the Django model validators',
|
|
24
|
+
[{ content: 'I need to check the pytest output and fix the failing tests' }]
|
|
25
|
+
);
|
|
26
|
+
const result = await tagger.tag(text);
|
|
27
|
+
assert.notEqual(result, null);
|
|
28
|
+
assert.equal(result.primary.domain, 'python');
|
|
29
|
+
assert.ok(result.primary.confidence > 0);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it('tags React/frontend routing text', async () => {
|
|
33
|
+
const text = DomainTagger.buildRoutingText(
|
|
34
|
+
'Build a React component with Tailwind CSS',
|
|
35
|
+
'Create a new JSX component using hooks and styled with Tailwind',
|
|
36
|
+
[{ content: 'I will use useState and useEffect hooks for this component' }]
|
|
37
|
+
);
|
|
38
|
+
const result = await tagger.tag(text);
|
|
39
|
+
assert.notEqual(result, null);
|
|
40
|
+
assert.equal(result.primary.domain, 'react_frontend');
|
|
41
|
+
assert.ok(result.primary.confidence > 0);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('tags Rust routing text', async () => {
|
|
45
|
+
const text = DomainTagger.buildRoutingText(
|
|
46
|
+
'Fix Rust ownership error',
|
|
47
|
+
'The cargo build fails with a borrow checker lifetime error in main.rs',
|
|
48
|
+
[{ content: 'I need to check the struct impl and fix the lifetime annotation' }]
|
|
49
|
+
);
|
|
50
|
+
const result = await tagger.tag(text);
|
|
51
|
+
assert.notEqual(result, null);
|
|
52
|
+
assert.equal(result.primary.domain, 'rust');
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('tags DevOps/Docker routing text', async () => {
|
|
56
|
+
const text = DomainTagger.buildRoutingText(
|
|
57
|
+
'Fix Dockerfile build',
|
|
58
|
+
'The Docker compose deployment fails on Kubernetes with nginx config errors',
|
|
59
|
+
[{ content: 'Let me check the Dockerfile and the helm chart' }]
|
|
60
|
+
);
|
|
61
|
+
const result = await tagger.tag(text);
|
|
62
|
+
assert.notEqual(result, null);
|
|
63
|
+
assert.equal(result.primary.domain, 'devops_docker');
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('tags database-related routing text', async () => {
|
|
67
|
+
const text = DomainTagger.buildRoutingText(
|
|
68
|
+
'Optimize SQL query',
|
|
69
|
+
'The PostgreSQL SELECT query with JOIN is slow, need to add an index',
|
|
70
|
+
[{ content: 'I will check the query plan and add the missing index' }]
|
|
71
|
+
);
|
|
72
|
+
const result = await tagger.tag(text);
|
|
73
|
+
assert.notEqual(result, null);
|
|
74
|
+
assert.equal(result.primary.domain, 'postgresql_database');
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('tags ML/data science routing text', async () => {
|
|
78
|
+
const text = DomainTagger.buildRoutingText(
|
|
79
|
+
'Train neural network model',
|
|
80
|
+
'Fine-tune the PyTorch transformer model on the new dataset with huggingface',
|
|
81
|
+
[{ content: 'I will set up the training loop with epoch tracking and loss metrics' }]
|
|
82
|
+
);
|
|
83
|
+
const result = await tagger.tag(text);
|
|
84
|
+
assert.notEqual(result, null);
|
|
85
|
+
assert.equal(result.primary.domain, 'data_science_ml');
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('returns all three tag levels with correct structure', async () => {
|
|
89
|
+
const text = 'Build a React component with TypeScript and Tailwind CSS for the frontend';
|
|
90
|
+
const result = await tagger.tag(text);
|
|
91
|
+
assert.notEqual(result, null);
|
|
92
|
+
assert.ok(result.primary);
|
|
93
|
+
assert.ok(result.secondary);
|
|
94
|
+
assert.ok(result.tertiary);
|
|
95
|
+
assert.equal(typeof result.primary.domain, 'string');
|
|
96
|
+
assert.equal(typeof result.primary.confidence, 'number');
|
|
97
|
+
assert.equal(typeof result.secondary.domain, 'string');
|
|
98
|
+
assert.equal(typeof result.secondary.confidence, 'number');
|
|
99
|
+
assert.equal(typeof result.tertiary.domain, 'string');
|
|
100
|
+
assert.equal(typeof result.tertiary.confidence, 'number');
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it('primary confidence >= secondary >= tertiary', async () => {
|
|
104
|
+
const text = 'Build a React JSX component with hooks and CSS styling for the frontend DOM';
|
|
105
|
+
const result = await tagger.tag(text);
|
|
106
|
+
assert.notEqual(result, null);
|
|
107
|
+
assert.ok(result.primary.confidence >= result.secondary.confidence);
|
|
108
|
+
assert.ok(result.secondary.confidence >= result.tertiary.confidence);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('returns null for empty or missing input', async () => {
|
|
112
|
+
assert.equal(await tagger.tag(''), null);
|
|
113
|
+
assert.equal(await tagger.tag(null), null);
|
|
114
|
+
assert.equal(await tagger.tag(undefined), null);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('returns null for unrecognizable content', async () => {
|
|
118
|
+
const result = await tagger.tag('asdfghjkl 12345 zzzzz');
|
|
119
|
+
assert.equal(result, null);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it('returns null when not initialized', async () => {
|
|
123
|
+
const uninit = new DomainTagger();
|
|
124
|
+
const result = await uninit.tag('Build a Python Django app');
|
|
125
|
+
assert.equal(result, null);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('gracefully degrades when embedding service is unreachable', async () => {
|
|
129
|
+
const taggerHttp = new DomainTagger({ serviceUrl: 'http://localhost:99999/v1/embeddings' });
|
|
130
|
+
await taggerHttp.init();
|
|
131
|
+
assert.equal(taggerHttp.ready, true);
|
|
132
|
+
assert.equal(taggerHttp.mode, 'keyword');
|
|
133
|
+
const result = await taggerHttp.tag('Build a Python Flask API');
|
|
134
|
+
assert.notEqual(result, null);
|
|
135
|
+
assert.equal(result.primary.domain, 'python');
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it('does not block on tag failure — returns null', async () => {
|
|
139
|
+
const start = Date.now();
|
|
140
|
+
const brokenTagger = new DomainTagger({ serviceUrl: 'http://localhost:99999/v1/embeddings' });
|
|
141
|
+
await brokenTagger.init();
|
|
142
|
+
const result = await brokenTagger.tag('anything');
|
|
143
|
+
const elapsed = Date.now() - start;
|
|
144
|
+
assert.ok(elapsed < 10_000, `tag() took ${elapsed}ms, should not block`);
|
|
145
|
+
assert.ok(result !== undefined);
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
describe('DomainTagger.buildRoutingText', () => {
|
|
150
|
+
it('combines task title, prompt, and thought steps', () => {
|
|
151
|
+
const text = DomainTagger.buildRoutingText(
|
|
152
|
+
'Fix bug',
|
|
153
|
+
'The server crashes on startup',
|
|
154
|
+
[{ content: 'Let me check the logs' }, { content: 'I see the error' }]
|
|
155
|
+
);
|
|
156
|
+
assert.ok(text.includes('Fix bug'));
|
|
157
|
+
assert.ok(text.includes('The server crashes on startup'));
|
|
158
|
+
assert.ok(text.includes('Let me check the logs'));
|
|
159
|
+
assert.ok(text.includes('I see the error'));
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it('limits to first 2 thought steps', () => {
|
|
163
|
+
const text = DomainTagger.buildRoutingText(
|
|
164
|
+
'Task',
|
|
165
|
+
'Prompt',
|
|
166
|
+
[{ content: 'Step 1' }, { content: 'Step 2' }, { content: 'Step 3' }]
|
|
167
|
+
);
|
|
168
|
+
assert.ok(text.includes('Step 1'));
|
|
169
|
+
assert.ok(text.includes('Step 2'));
|
|
170
|
+
assert.ok(!text.includes('Step 3'));
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it('handles missing parts gracefully', () => {
|
|
174
|
+
assert.equal(DomainTagger.buildRoutingText(null, null, []), '');
|
|
175
|
+
assert.ok(DomainTagger.buildRoutingText('Title', null, []).includes('Title'));
|
|
176
|
+
assert.ok(DomainTagger.buildRoutingText(null, 'Prompt', []).includes('Prompt'));
|
|
177
|
+
});
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
describe('cosineSimilarity', () => {
|
|
181
|
+
it('returns 1 for identical vectors', () => {
|
|
182
|
+
const v = [1, 2, 3, 4, 5];
|
|
183
|
+
assert.ok(Math.abs(cosineSimilarity(v, v) - 1) < 0.0001);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it('returns 0 for orthogonal vectors', () => {
|
|
187
|
+
assert.ok(Math.abs(cosineSimilarity([1, 0], [0, 1])) < 0.0001);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it('returns -1 for opposite vectors', () => {
|
|
191
|
+
assert.ok(Math.abs(cosineSimilarity([1, 0], [-1, 0]) - (-1)) < 0.0001);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
it('returns 0 for mismatched lengths', () => {
|
|
195
|
+
assert.equal(cosineSimilarity([1, 2], [1, 2, 3]), 0);
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('returns 0 for null/empty inputs', () => {
|
|
199
|
+
assert.equal(cosineSimilarity(null, [1, 2]), 0);
|
|
200
|
+
assert.equal(cosineSimilarity([1, 2], null), 0);
|
|
201
|
+
assert.equal(cosineSimilarity([], []), 0);
|
|
202
|
+
});
|
|
203
|
+
});
|