groove-dev 0.27.101 → 0.27.103

package/moe-training/client/domain-tagger.js

@@ -0,0 +1,205 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+
+const DEFAULT_DOMAINS = [
+  'python', 'typescript_node', 'react_frontend', 'postgresql_database',
+  'devops_docker', 'rust', 'data_science_ml', 'security_pentest',
+  'mobile_swift', 'system_design',
+];
+
+const DEFAULT_MODEL = 'sentence-transformers/all-MiniLM-L6-v2';
+const DEFAULT_TOP_K = 3;
+
+const DOMAIN_KEYWORDS = {
+  python: ['python', 'pip', 'pytest', 'django', 'flask', 'fastapi', '.py', 'pandas', 'numpy', 'venv', 'poetry', 'pyproject', '__init__', 'def ', 'import '],
+  typescript_node: ['typescript', 'node', 'npm', 'express', '.ts', 'tsconfig', 'package.json', 'nestjs', 'prisma', 'tsc', 'deno', 'bun'],
+  react_frontend: ['react', 'jsx', 'tsx', 'component', 'hook', 'usestate', 'useeffect', 'tailwind', 'css', 'vite', 'nextjs', 'styled', 'frontend', 'dom', 'html'],
+  postgresql_database: ['postgresql', 'postgres', 'sql', 'database', 'query', 'schema', 'migration', 'table', 'index', 'select', 'insert', 'join', 'foreign key', 'sequelize', 'knex'],
+  devops_docker: ['docker', 'kubernetes', 'k8s', 'ci/cd', 'github actions', 'deployment', 'terraform', 'ansible', 'nginx', 'dockerfile', 'compose', 'helm', 'aws', 'gcp', 'pipeline'],
+  rust: ['rust', 'cargo', 'ownership', 'lifetime', 'borrow', '.rs', 'impl ', 'fn ', 'struct ', 'enum ', 'trait ', 'crate', 'tokio'],
+  data_science_ml: ['machine learning', 'pytorch', 'tensorflow', 'ml', 'training', 'dataset', 'neural', 'deep learning', 'transformer', 'huggingface', 'sklearn', 'prediction', 'epoch', 'loss'],
+  security_pentest: ['security', 'vulnerability', 'cve', 'authentication', 'authorization', 'encryption', 'xss', 'sql injection', 'pentest', 'exploit', 'firewall', 'oauth', 'csrf'],
+  mobile_swift: ['swift', 'ios', 'swiftui', 'xcode', 'cocoapod', 'uikit', 'storyboard', 'watchos', 'macos', 'apple', 'carthage', 'spm'],
+  system_design: ['architecture', 'system design', 'scalability', 'microservice', 'distributed', 'load balancer', 'cache', 'message queue', 'api gateway', 'monorepo', 'design pattern', 'event driven'],
+};
+
+export class DomainTagger {
+  constructor(options = {}) {
+    this._serviceUrl = options.serviceUrl || process.env.EMBEDDING_SERVICE_URL || null;
+    this._model = options.model || DEFAULT_MODEL;
+    this._topK = options.topK || DEFAULT_TOP_K;
+    this._domains = options.domains || DEFAULT_DOMAINS;
+    this._ready = false;
+    this._mode = null;
+    this._centroids = null;
+    this._lastError = null;
+  }
+
+  async init() {
+    this._lastError = null;
+
+    if (this._serviceUrl) {
+      try {
+        const res = await fetch(this._serviceUrl, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ input: 'health check', model: this._model }),
+          signal: AbortSignal.timeout(5_000),
+        });
+        if (res.ok) {
+          this._mode = 'http';
+          await this._buildCentroids();
+          this._ready = true;
+          return;
+        }
+      } catch {
+        // HTTP service unavailable
+      }
+    }
+
+    this._mode = 'keyword';
+    this._ready = true;
+  }
+
+  async tag(routingText) {
+    if (!this._ready || !routingText || typeof routingText !== 'string') return null;
+
+    this._lastError = null;
+    try {
+      if (this._mode === 'http') {
+        return await this._tagWithEmbeddings(routingText);
+      }
+      return this._tagWithKeywords(routingText);
+    } catch (err) {
+      this._lastError = err.message || String(err);
+      return null;
+    }
+  }
+
+  get lastError() {
+    return this._lastError;
+  }
+
+  get ready() {
+    return this._ready;
+  }
+
+  get mode() {
+    return this._mode;
+  }
+
+  static buildRoutingText(taskTitle, firstPrompt, thoughtSteps = []) {
+    const parts = [];
+    if (taskTitle) parts.push(taskTitle);
+    if (firstPrompt) parts.push(firstPrompt);
+    for (const step of thoughtSteps.slice(0, 2)) {
+      if (step?.content) parts.push(step.content);
+    }
+    return parts.join('\n');
+  }
+
+  async _tagWithEmbeddings(routingText) {
+    const embedding = await this._embed(routingText);
+    if (!embedding) return null;
+
+    const scores = [];
+    for (const [domain, centroid] of Object.entries(this._centroids)) {
+      scores.push({ domain, confidence: cosineSimilarity(embedding, centroid) });
+    }
+
+    scores.sort((a, b) => b.confidence - a.confidence);
+    const top = scores.slice(0, this._topK);
+
+    if (top.length < 3) return null;
+
+    return {
+      primary: { domain: top[0].domain, confidence: round4(top[0].confidence) },
+      secondary: { domain: top[1].domain, confidence: round4(top[1].confidence) },
+      tertiary: { domain: top[2].domain, confidence: round4(top[2].confidence) },
+    };
+  }
+
+  _tagWithKeywords(routingText) {
+    const text = routingText.toLowerCase();
+    const scores = [];
+
+    for (const domain of this._domains) {
+      const keywords = DOMAIN_KEYWORDS[domain];
+      if (!keywords) {
+        scores.push({ domain, confidence: 0 });
+        continue;
+      }
+
+      let hits = 0;
+      for (const kw of keywords) {
+        if (text.includes(kw.toLowerCase())) hits++;
+      }
+      scores.push({ domain, confidence: keywords.length > 0 ? hits / keywords.length : 0 });
+    }
+
+    scores.sort((a, b) => b.confidence - a.confidence);
+
+    if (scores[0].confidence === 0) return null;
+
+    const top = scores.slice(0, this._topK);
+    return {
+      primary: { domain: top[0].domain, confidence: round4(top[0].confidence) },
+      secondary: { domain: top[1].domain, confidence: round4(top[1].confidence) },
+      tertiary: { domain: top[2].domain, confidence: round4(top[2].confidence) },
+    };
+  }
+
+  async _buildCentroids() {
+    this._centroids = {};
+    for (const domain of this._domains) {
+      const kws = DOMAIN_KEYWORDS[domain];
+      const description = kws ? `${domain}: ${kws.join(', ')}` : domain;
+      const embedding = await this._embed(description);
+      if (embedding) {
+        this._centroids[domain] = embedding;
+      }
+    }
+  }
+
+  async _embed(text) {
+    try {
+      const res = await fetch(this._serviceUrl, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ input: text, model: this._model }),
+        signal: AbortSignal.timeout(10_000),
+      });
+
+      if (!res.ok) {
+        this._lastError = `Embedding service returned ${res.status}`;
+        return null;
+      }
+
+      const data = await res.json();
+      const embedding = data?.data?.[0]?.embedding;
+      if (!Array.isArray(embedding)) {
+        this._lastError = 'Invalid embedding response format';
+        return null;
+      }
+      return embedding;
+    } catch (err) {
+      this._lastError = err.message || String(err);
+      return null;
+    }
+  }
+}
+
+export function cosineSimilarity(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0, magA = 0, magB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+function round4(n) {
+  return Math.round(n * 10000) / 10000;
+}

package/moe-training/client/edit-normalizer.js

@@ -0,0 +1,188 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+
+const APPLY_PATCH_RE = /apply_patch/;
+const BEGIN_PATCH_RE = /^\*{3}\s*Begin Patch\s*$/;
+const END_PATCH_RE = /^\*{3}\s*End Patch\s*$/;
+const ADD_FILE_RE = /^\*{3}\s*Add File:\s*(.+)$/;
+const UPDATE_FILE_RE = /^\*{3}\s*Update File:\s*(.+)$/;
+const DELETE_FILE_RE = /^\*{3}\s*Delete File:\s*(.+)$/;
+const HUNK_HEADER_RE = /^@@@.*@@@/;
+
+export class EditNormalizer {
+  detectApplyPatch(actionContent) {
+    const text = extractText(actionContent);
+    if (!text) return false;
+    return APPLY_PATCH_RE.test(text);
+  }
+
+  normalize(actionContent, timestamp, startStep) {
+    const text = extractText(actionContent);
+    if (!text) return [];
+
+    const patchBody = extractPatchBody(text);
+    if (!patchBody) return [];
+
+    const sections = parseSections(patchBody);
+    const edits = [];
+    let step = startStep || 1;
+
+    for (const section of sections) {
+      if (section.type === 'add') {
+        edits.push({
+          step: step++,
+          type: 'edit',
+          timestamp: timestamp || Date.now() / 1000,
+          file_path: section.filePath,
+          edit_type: 'create',
+          content: section.content,
+          old_string: null,
+          new_string: null,
+          token_count: estimateTokens(section.content),
+        });
+      } else if (section.type === 'update') {
+        const hunks = parseHunks(section.lines);
+        for (const hunk of hunks) {
+          edits.push({
+            step: step++,
+            type: 'edit',
+            timestamp: timestamp || Date.now() / 1000,
+            file_path: section.filePath,
+            edit_type: 'modify',
+            content: null,
+            old_string: hunk.oldString,
+            new_string: hunk.newString,
+            token_count: estimateTokens(hunk.oldString + hunk.newString),
+          });
+        }
+      } else if (section.type === 'delete') {
+        edits.push({
+          step: step++,
+          type: 'edit',
+          timestamp: timestamp || Date.now() / 1000,
+          file_path: section.filePath,
+          edit_type: 'delete',
+          content: null,
+          old_string: section.content || null,
+          new_string: null,
+          token_count: estimateTokens(section.content || ''),
+        });
+      }
+    }
+
+    return edits;
+  }
+}
+
+function extractText(actionContent) {
+  if (typeof actionContent === 'string') return actionContent;
+  if (actionContent?.arguments?.command) return actionContent.arguments.command;
+  if (actionContent?.content) return actionContent.content;
+  return null;
+}
+
+function extractPatchBody(text) {
+  const lines = text.split('\n');
+  let startIdx = -1;
+  let endIdx = -1;
+
+  for (let i = 0; i < lines.length; i++) {
+    if (BEGIN_PATCH_RE.test(lines[i].trim())) {
+      startIdx = i + 1;
+    }
+    if (END_PATCH_RE.test(lines[i].trim())) {
+      endIdx = i;
+    }
+  }
+
+  if (startIdx === -1) return null;
+  if (endIdx === -1) endIdx = lines.length;
+
+  return lines.slice(startIdx, endIdx).join('\n');
+}
+
+function parseSections(body) {
+  const lines = body.split('\n');
+  const sections = [];
+  let current = null;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const trimmed = line.trim();
+
+    let match;
+    if ((match = ADD_FILE_RE.exec(trimmed))) {
+      if (current) sections.push(current);
+      current = { type: 'add', filePath: match[1].trim(), lines: [], content: '' };
+    } else if ((match = UPDATE_FILE_RE.exec(trimmed))) {
+      if (current) sections.push(current);
+      current = { type: 'update', filePath: match[1].trim(), lines: [] };
+    } else if ((match = DELETE_FILE_RE.exec(trimmed))) {
+      if (current) sections.push(current);
+      current = { type: 'delete', filePath: match[1].trim(), lines: [], content: '' };
+    } else if (current) {
+      current.lines.push(line);
+    }
+  }
+
+  if (current) sections.push(current);
+
+  for (const section of sections) {
+    if (section.type === 'add' || section.type === 'delete') {
+      section.content = section.lines.join('\n').trim();
+    }
+  }
+
+  return sections;
+}
+
+function parseHunks(lines) {
+  const hunks = [];
+  let inHunk = false;
+  let oldLines = [];
+  let newLines = [];
+
+  for (const line of lines) {
+    if (HUNK_HEADER_RE.test(line.trim())) {
+      if (inHunk && (oldLines.length > 0 || newLines.length > 0)) {
+        hunks.push(buildHunk(oldLines, newLines));
+      }
+      inHunk = true;
+      oldLines = [];
+      newLines = [];
+      continue;
+    }
+
+    if (!inHunk && (line.startsWith('-') || line.startsWith('+') || line.startsWith(' '))) {
+      inHunk = true;
+    }
+
+    if (!inHunk) continue;
+
+    if (line.startsWith('-')) {
+      oldLines.push(line.slice(1));
+    } else if (line.startsWith('+')) {
+      newLines.push(line.slice(1));
+    } else if (line.startsWith(' ')) {
+      oldLines.push(line.slice(1));
+      newLines.push(line.slice(1));
+    }
+  }
+
+  if (inHunk && (oldLines.length > 0 || newLines.length > 0)) {
+    hunks.push(buildHunk(oldLines, newLines));
+  }
+
+  return hunks;
+}
+
+function buildHunk(oldLines, newLines) {
+  return {
+    oldString: oldLines.join('\n'),
+    newString: newLines.join('\n'),
+  };
+}
+
+export function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.max(1, Math.ceil(text.length / 4));
+}

package/moe-training/client/envelope-builder.js

@@ -7,7 +7,7 @@ export class EnvelopeBuilder {
   constructor(sessionId, contributorId, metadata) {
     this._sessionId = sessionId;
     this._contributorId = contributorId;
-    this._metadata = metadata;
+    this._metadata = { ...metadata, leaf_context: metadata.leaf_context ?? null };
     this._buffer = [];
     this._chunkSequence = 0;
   }

package/moe-training/client/parsers/claude-code.js

@@ -1,15 +1,21 @@
 // FSL-1.1-Apache-2.0 — see LICENSE
 
-import { OBSERVATION_TRUNCATE_HEAD, OBSERVATION_TRUNCATE_TAIL } from '../../shared/constants.js';
+import { OBSERVATION_TOKEN_LIMIT } from '../../shared/constants.js';
+
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
 
 function truncateObservation(text) {
-  if (!text || typeof text !== 'string') return text;
-  const lines = text.split('\n');
-  if (lines.length <= OBSERVATION_TRUNCATE_HEAD + OBSERVATION_TRUNCATE_TAIL) return text;
-  const head = lines.slice(0, OBSERVATION_TRUNCATE_HEAD);
-  const tail = lines.slice(-OBSERVATION_TRUNCATE_TAIL);
-  const omitted = lines.length - OBSERVATION_TRUNCATE_HEAD - OBSERVATION_TRUNCATE_TAIL;
-  return [...head, `[... ${omitted} lines omitted ...]`, ...tail].join('\n');
+  if (!text || typeof text !== 'string') return { content: text, truncated: false, original_token_count: estimateTokens(text) };
+  const originalTokens = estimateTokens(text);
+  if (originalTokens <= OBSERVATION_TOKEN_LIMIT) {
+    return { content: text, truncated: false, original_token_count: originalTokens };
+  }
+  const charLimit = OBSERVATION_TOKEN_LIMIT * 4;
+  const truncated = text.slice(0, charLimit) + `\n[TRUNCATED — original output was ${originalTokens} tokens]`;
+  return { content: truncated, truncated: true, original_token_count: originalTokens };
 }
 
 export class ClaudeCodeParser {
@@ -50,10 +56,51 @@ export class ClaudeCodeParser {
           if (block.is_error) {
             results.push({ type: 'error', content: resultContent, is_error: true });
           } else {
+            const obs = truncateObservation(resultContent);
+            results.push({
+              type: 'observation',
+              content: obs.content,
+              truncated: obs.truncated,
+              original_token_count: obs.original_token_count,
+              is_error: false,
+            });
+          }
+        }
+      }
+
+      return results.length === 1 ? results[0] : results.length > 1 ? results : null;
+    }
+
+    if (jsonEvent.type === 'user') {
+      const contentBlocks = jsonEvent.message?.content;
+      if (!Array.isArray(contentBlocks)) return null;
+
+      const results = [];
+      for (const block of contentBlocks) {
+        if (block.type === 'tool_result') {
+          const toolUse = this._pendingToolUse.get(block.tool_use_id);
+          if (toolUse) this._pendingToolUse.delete(block.tool_use_id);
+
+          const resultContent = Array.isArray(block.content)
+            ? block.content.map((c) => c.text || '').join('\n')
+            : (typeof block.content === 'string' ? block.content : '');
+
+          if (block.is_error) {
+            results.push({
+              type: 'error',
+              content: resultContent,
+              is_error: true,
+              tool: toolUse?.name,
+            });
+          } else {
+            const obs = truncateObservation(resultContent);
             results.push({
               type: 'observation',
-              content: truncateObservation(resultContent),
+              content: obs.content,
+              truncated: obs.truncated,
+              original_token_count: obs.original_token_count,
               is_error: false,
+              tool: toolUse?.name,
             });
           }
         }

package/moe-training/client/parsers/codex.js

@@ -1,5 +1,23 @@
 // FSL-1.1-Apache-2.0 — see LICENSE
 
+import { OBSERVATION_TOKEN_LIMIT } from '../../shared/constants.js';
+
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+function truncateObservation(text) {
+  if (!text || typeof text !== 'string') return { content: text || '', truncated: false, original_token_count: estimateTokens(text) };
+  const originalTokens = estimateTokens(text);
+  if (originalTokens <= OBSERVATION_TOKEN_LIMIT) {
+    return { content: text, truncated: false, original_token_count: originalTokens };
+  }
+  const charLimit = OBSERVATION_TOKEN_LIMIT * 4;
+  const truncated = text.slice(0, charLimit) + `\n[TRUNCATED — original output was ${originalTokens} tokens]`;
+  return { content: truncated, truncated: true, original_token_count: originalTokens };
+}
+
 export class CodexParser {
   constructor() {
     this._sessionId = null;
@@ -37,15 +55,17 @@ export class CodexParser {
           return { type: 'thought', content: item.text || '' };
         }
         if (item.type === 'command_execution') {
-          const output = (item.aggregated_output || '').slice(0, 2000);
+          const rawOutput = item.aggregated_output || '';
           if (item.exit_code !== 0) {
-            return { type: 'error', content: output || `Exit code: ${item.exit_code}` };
+            return { type: 'error', content: rawOutput.slice(0, 2000) || `Exit code: ${item.exit_code}` };
           }
-          return { type: 'observation', content: output };
+          const obs = truncateObservation(rawOutput);
+          return { type: 'observation', content: obs.content, truncated: obs.truncated, original_token_count: obs.original_token_count };
         }
         if (item.type === 'file_edit' || item.type === 'file_write' || item.type === 'file_read') {
-          const output = (item.output || item.content || '').slice(0, 2000);
-          return { type: 'observation', content: output };
+          const rawOutput = item.output || item.content || '';
+          const obs = truncateObservation(rawOutput);
+          return { type: 'observation', content: obs.content, truncated: obs.truncated, original_token_count: obs.original_token_count };
         }
         return null;
       }

package/moe-training/client/parsers/gemini.js

@@ -1,5 +1,23 @@
 // FSL-1.1-Apache-2.0 — see LICENSE
 
+import { OBSERVATION_TOKEN_LIMIT } from '../../shared/constants.js';
+
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+function truncateObservation(text) {
+  if (!text || typeof text !== 'string') return { content: text || '', truncated: false, original_token_count: estimateTokens(text) };
+  const originalTokens = estimateTokens(text);
+  if (originalTokens <= OBSERVATION_TOKEN_LIMIT) {
+    return { content: text, truncated: false, original_token_count: originalTokens };
+  }
+  const charLimit = OBSERVATION_TOKEN_LIMIT * 4;
+  const truncated = text.slice(0, charLimit) + `\n[TRUNCATED — original output was ${originalTokens} tokens]`;
+  return { content: truncated, truncated: true, original_token_count: originalTokens };
+}
+
 export class GeminiParser {
   constructor() {
     this._sessionId = null;
@@ -44,8 +62,9 @@ export class GeminiParser {
       case 'tool_response': {
         const rawContent = jsonEvent.content;
         const contentParts = Array.isArray(rawContent) ? rawContent : (typeof rawContent === 'string' ? [{ text: rawContent }] : rawContent ? [rawContent] : []);
-        const content = contentParts.map((p) => p.text || '').join('').slice(0, 2000);
-        return { type: 'observation', content };
+        const rawText = contentParts.map((p) => p.text || '').join('');
+        const obs = truncateObservation(rawText);
+        return { type: 'observation', content: obs.content, truncated: obs.truncated, original_token_count: obs.original_token_count };
       }
 
       case 'error': {

package/moe-training/client/parsers/grok.js

@@ -1,5 +1,23 @@
 // FSL-1.1-Apache-2.0 — see LICENSE
 
+import { OBSERVATION_TOKEN_LIMIT } from '../../shared/constants.js';
+
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+function truncateObservation(text) {
+  if (!text || typeof text !== 'string') return { content: text || '', truncated: false, original_token_count: estimateTokens(text) };
+  const originalTokens = estimateTokens(text);
+  if (originalTokens <= OBSERVATION_TOKEN_LIMIT) {
+    return { content: text, truncated: false, original_token_count: originalTokens };
+  }
+  const charLimit = OBSERVATION_TOKEN_LIMIT * 4;
+  const truncated = text.slice(0, charLimit) + `\n[TRUNCATED — original output was ${originalTokens} tokens]`;
+  return { content: truncated, truncated: true, original_token_count: originalTokens };
+}
+
 export class GrokParser {
   // TODO: Grok agentic CLI not yet available. Wire up when headless CLI is built.
   parseEvent(_jsonEvent) {