groove-dev 0.27.179 → 0.27.180

package/moe-training/client/envelope-builder.js

@@ -1,7 +1,12 @@
 // FSL-1.1-Apache-2.0 — see LICENSE
 
 import { randomUUID } from 'node:crypto';
-import { CHUNK_SIZE } from '../shared/constants.js';
+import { CHUNK_SIZE, MAX_STEP_CONTENT_CHARS, MAX_TOKEN_COUNT } from '../shared/constants.js';
+
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
 
 export class EnvelopeBuilder {
   constructor(sessionId, contributorId, metadata) {
@@ -13,11 +18,23 @@ export class EnvelopeBuilder {
   }
 
   addStep(step) {
-    if (step.content && typeof step.content === 'string' && step.content.length > 10_000) {
-      step.content = step.content.slice(0, 10_000);
+    // Last-resort trim so the envelope passes ingest validation (an oversized
+    // step.content is rejected, which drops the whole session). Parsers already
+    // truncate observations to OBSERVATION_TOKEN_LIMIT and flag it; this only
+    // fires for content that slipped past them. When it does fire it MUST record
+    // the loss — a silent trim looks like complete data to downstream training.
+    if (step.content && typeof step.content === 'string' && step.content.length > MAX_STEP_CONTENT_CHARS) {
+      const originalTokens = estimateTokens(step.content);
+      step.content = step.content.slice(0, MAX_STEP_CONTENT_CHARS);
+      step.truncated = true;
+      // Preserve the parser's count if it already trimmed — that one reflects
+      // the true original size, ours only sees what survived the first pass.
+      if (typeof step.original_token_count !== 'number') {
+        step.original_token_count = Math.min(originalTokens, MAX_TOKEN_COUNT);
+      }
     }
-    if (typeof step.token_count === 'number' && step.token_count > 100_000) {
-      step.token_count = 100_000;
+    if (typeof step.token_count === 'number' && step.token_count > MAX_TOKEN_COUNT) {
+      step.token_count = MAX_TOKEN_COUNT;
     }
     this._buffer.push(step);
     if (this._buffer.length >= CHUNK_SIZE) {

package/moe-training/shared/constants.js

@@ -28,7 +28,17 @@ export const QUALITY_MULTIPLIERS = {
   highQuality: 1.5,
 };
 
-export const OBSERVATION_TOKEN_LIMIT = 4096;
+// Hard ceiling on step.content, shared by the client trimmer (envelope-builder)
+// and the ingest validator (envelope-schema). These MUST stay equal: the
+// validator rejects the whole envelope when exceeded, which drops the entire
+// session — so the client must trim to exactly this before building.
+export const MAX_STEP_CONTENT_CHARS = 100_000;
+export const MAX_TOKEN_COUNT = 100_000;
+
+// Per-observation budget, in estimated tokens (~4 chars/token). Kept safely
+// under MAX_STEP_CONTENT_CHARS so a parser-truncated observation plus its
+// "[TRUNCATED …]" suffix still fits without a second trim downstream.
+export const OBSERVATION_TOKEN_LIMIT = 24_000;
 export const TIER_A_MIN_QUALITY = 70;
 export const TIER_B_MIN_QUALITY = 50;
 export const TRAINING_MIN_STEPS = 5;

package/moe-training/shared/envelope-schema.js

@@ -1,6 +1,12 @@
 // FSL-1.1-Apache-2.0 — see LICENSE
 
-import { SUPPORTED_PROVIDERS, MODEL_TIERS, TRAINING_EXCLUSION_REASONS } from './constants.js';
+import {
+  SUPPORTED_PROVIDERS,
+  MODEL_TIERS,
+  TRAINING_EXCLUSION_REASONS,
+  MAX_STEP_CONTENT_CHARS,
+  MAX_TOKEN_COUNT,
+} from './constants.js';
 
 export const STEP_TYPES = ['thought', 'action', 'observation', 'correction', 'resolution', 'error', 'coordination', 'edit', 'instruction', 'clarification', 'approval', 'delegate', 'yield'];
 const VALID_QUALITY_TIERS = ['TIER_A', 'TIER_B', 'TIER_C'];
@@ -10,8 +16,10 @@ const VALID_MODEL_ENGINES = Object.keys(MODEL_TIERS);
 const VALID_COMPLEXITIES = ['light', 'medium', 'heavy'];
 const VALID_OUTCOME_STATUSES = ['SUCCESS', 'CRASH', 'KILLED'];
 const MAX_STEPS_PER_ENVELOPE = 500;
-const MAX_STEP_CONTENT_LENGTH = 10_000;
-const MAX_TOKEN_COUNT = 100_000;
+// Sourced from shared constants — the client trims to the same value before
+// building envelopes. Keeping one definition prevents the drift that silently
+// truncated observations at 10k while the validator allowed more.
+const MAX_STEP_CONTENT_LENGTH = MAX_STEP_CONTENT_CHARS;
 const MAX_STEP_NUMBER = 50_000;
 const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
 const ONE_HOUR_MS = 60 * 60 * 1000;

package/moe-training/test/client/envelope-builder.test.js

@@ -3,7 +3,7 @@
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
 import { EnvelopeBuilder } from '../../client/envelope-builder.js';
-import { CHUNK_SIZE } from '../../shared/constants.js';
+import { CHUNK_SIZE, MAX_STEP_CONTENT_CHARS } from '../../shared/constants.js';
 
 const metadata = {
   model_engine: 'claude-opus-4-6',
@@ -74,12 +74,48 @@ describe('EnvelopeBuilder', () => {
     assert.deepEqual(close.outcome, outcome);
   });
 
-  it('truncates step content at 10000 characters', () => {
+  it('leaves content under the ingest ceiling untouched', () => {
     const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
-    const longContent = 'x'.repeat(15_000);
-    builder.addStep({ step: 1, type: 'thought', timestamp: 123, content: longContent });
-    const envelope = builder.flush();
-    assert.equal(envelope.trajectory_log[0].content.length, 10_000);
+    builder.addStep({ step: 1, type: 'thought', timestamp: 123, content: 'x'.repeat(15_000) });
+    const step = builder.flush().trajectory_log[0];
+    assert.equal(step.content.length, 15_000);
+    assert.ok(!step.truncated);
+  });
+
+  it('truncates step content at MAX_STEP_CONTENT_CHARS', () => {
+    const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
+    builder.addStep({ step: 1, type: 'thought', timestamp: 123, content: 'x'.repeat(150_000) });
+    const step = builder.flush().trajectory_log[0];
+    assert.equal(step.content.length, MAX_STEP_CONTENT_CHARS);
+  });
+
+  // Regression: a trim that doesn't flag itself looks like complete data to
+  // downstream training and silently corrupts the corpus.
+  it('never truncates silently — always sets truncated + original_token_count', () => {
+    const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
+    builder.addStep({ step: 1, type: 'observation', timestamp: 123, content: 'x'.repeat(150_000) });
+    const step = builder.flush().trajectory_log[0];
+    assert.equal(step.truncated, true);
+    assert.equal(step.original_token_count, 37_500); // 150k chars / 4
+  });
+
+  it('preserves a parser-supplied original_token_count when trimming again', () => {
+    const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
+    builder.addStep({
+      step: 1, type: 'observation', timestamp: 123,
+      content: 'x'.repeat(150_000), truncated: true, original_token_count: 99_000,
+    });
+    const step = builder.flush().trajectory_log[0];
+    // The parser saw the true original; our count only sees what survived it.
+    assert.equal(step.original_token_count, 99_000);
+  });
+
+  it('output always satisfies the ingest validator', () => {
+    const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
+    builder.addStep({ step: 1, type: 'observation', timestamp: 123, content: 'x'.repeat(500_000) });
+    const step = builder.flush().trajectory_log[0];
+    assert.ok(step.content.length <= MAX_STEP_CONTENT_CHARS,
+      'client must trim to the validator ceiling or the whole session is rejected');
   });
 
   it('caps token_count at 100000', () => {

package/moe-training/test/shared/envelope-schema.test.js

@@ -3,7 +3,7 @@
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
 import { validateEnvelope, STEP_TYPES } from '../../shared/envelope-schema.js';
-import { TRAINING_EXCLUSION_REASONS } from '../../shared/constants.js';
+import { TRAINING_EXCLUSION_REASONS, MAX_STEP_CONTENT_CHARS } from '../../shared/constants.js';
 
 const VALID_HMAC = 'a'.repeat(64);
 const VALID_APP_HASH = 'b'.repeat(64);
@@ -113,12 +113,19 @@ describe('envelope-schema', () => {
     assert.ok(result.errors.some(e => e.includes('500')));
   });
 
-  it('rejects step with content > 10KB', () => {
+  it('accepts step content up to MAX_STEP_CONTENT_CHARS', () => {
     const env = validEnvelope();
-    env.trajectory_log[0].content = 'x'.repeat(10_001);
+    env.trajectory_log[0].content = 'x'.repeat(MAX_STEP_CONTENT_CHARS);
+    const result = validateEnvelope(env);
+    assert.ok(!result.errors.some(e => e.includes('step.content exceeds')));
+  });
+
+  it('rejects step content over MAX_STEP_CONTENT_CHARS', () => {
+    const env = validEnvelope();
+    env.trajectory_log[0].content = 'x'.repeat(MAX_STEP_CONTENT_CHARS + 1);
     const result = validateEnvelope(env);
     assert.equal(result.valid, false);
-    assert.ok(result.errors.some(e => e.includes('10000')));
+    assert.ok(result.errors.some(e => e.includes(String(MAX_STEP_CONTENT_CHARS))));
   });
 
   it('rejects step with token_count > 100,000', () => {

package/node_modules/@groove-dev/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/cli",
-  "version": "0.27.179",
+  "version": "0.27.180",
   "description": "GROOVE CLI — manage AI coding agents from your terminal",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/node_modules/@groove-dev/daemon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/daemon",
-  "version": "0.27.179",
+  "version": "0.27.180",
   "description": "GROOVE daemon — agent orchestration engine",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/node_modules/@groove-dev/gui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/gui",
-  "version": "0.27.179",
+  "version": "0.27.180",
   "description": "GROOVE GUI — visual agent control plane",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/node_modules/moe-training/client/envelope-builder.js

@@ -1,7 +1,12 @@
 // FSL-1.1-Apache-2.0 — see LICENSE
 
 import { randomUUID } from 'node:crypto';
-import { CHUNK_SIZE } from '../shared/constants.js';
+import { CHUNK_SIZE, MAX_STEP_CONTENT_CHARS, MAX_TOKEN_COUNT } from '../shared/constants.js';
+
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
 
 export class EnvelopeBuilder {
   constructor(sessionId, contributorId, metadata) {
@@ -13,11 +18,23 @@ export class EnvelopeBuilder {
   }
 
   addStep(step) {
-    if (step.content && typeof step.content === 'string' && step.content.length > 10_000) {
-      step.content = step.content.slice(0, 10_000);
+    // Last-resort trim so the envelope passes ingest validation (an oversized
+    // step.content is rejected, which drops the whole session). Parsers already
+    // truncate observations to OBSERVATION_TOKEN_LIMIT and flag it; this only
+    // fires for content that slipped past them. When it does fire it MUST record
+    // the loss — a silent trim looks like complete data to downstream training.
+    if (step.content && typeof step.content === 'string' && step.content.length > MAX_STEP_CONTENT_CHARS) {
+      const originalTokens = estimateTokens(step.content);
+      step.content = step.content.slice(0, MAX_STEP_CONTENT_CHARS);
+      step.truncated = true;
+      // Preserve the parser's count if it already trimmed — that one reflects
+      // the true original size, ours only sees what survived the first pass.
+      if (typeof step.original_token_count !== 'number') {
+        step.original_token_count = Math.min(originalTokens, MAX_TOKEN_COUNT);
+      }
     }
-    if (typeof step.token_count === 'number' && step.token_count > 100_000) {
-      step.token_count = 100_000;
+    if (typeof step.token_count === 'number' && step.token_count > MAX_TOKEN_COUNT) {
+      step.token_count = MAX_TOKEN_COUNT;
     }
     this._buffer.push(step);
     if (this._buffer.length >= CHUNK_SIZE) {

package/node_modules/moe-training/shared/constants.js

@@ -28,7 +28,17 @@ export const QUALITY_MULTIPLIERS = {
   highQuality: 1.5,
 };
 
-export const OBSERVATION_TOKEN_LIMIT = 4096;
+// Hard ceiling on step.content, shared by the client trimmer (envelope-builder)
+// and the ingest validator (envelope-schema). These MUST stay equal: the
+// validator rejects the whole envelope when exceeded, which drops the entire
+// session — so the client must trim to exactly this before building.
+export const MAX_STEP_CONTENT_CHARS = 100_000;
+export const MAX_TOKEN_COUNT = 100_000;
+
+// Per-observation budget, in estimated tokens (~4 chars/token). Kept safely
+// under MAX_STEP_CONTENT_CHARS so a parser-truncated observation plus its
+// "[TRUNCATED …]" suffix still fits without a second trim downstream.
+export const OBSERVATION_TOKEN_LIMIT = 24_000;
 export const TIER_A_MIN_QUALITY = 70;
 export const TIER_B_MIN_QUALITY = 50;
 export const TRAINING_MIN_STEPS = 5;

package/node_modules/moe-training/shared/envelope-schema.js

@@ -1,6 +1,12 @@
 // FSL-1.1-Apache-2.0 — see LICENSE
 
-import { SUPPORTED_PROVIDERS, MODEL_TIERS, TRAINING_EXCLUSION_REASONS } from './constants.js';
+import {
+  SUPPORTED_PROVIDERS,
+  MODEL_TIERS,
+  TRAINING_EXCLUSION_REASONS,
+  MAX_STEP_CONTENT_CHARS,
+  MAX_TOKEN_COUNT,
+} from './constants.js';
 
 export const STEP_TYPES = ['thought', 'action', 'observation', 'correction', 'resolution', 'error', 'coordination', 'edit', 'instruction', 'clarification', 'approval', 'delegate', 'yield'];
 const VALID_QUALITY_TIERS = ['TIER_A', 'TIER_B', 'TIER_C'];
@@ -10,8 +16,10 @@ const VALID_MODEL_ENGINES = Object.keys(MODEL_TIERS);
 const VALID_COMPLEXITIES = ['light', 'medium', 'heavy'];
 const VALID_OUTCOME_STATUSES = ['SUCCESS', 'CRASH', 'KILLED'];
 const MAX_STEPS_PER_ENVELOPE = 500;
-const MAX_STEP_CONTENT_LENGTH = 10_000;
-const MAX_TOKEN_COUNT = 100_000;
+// Sourced from shared constants — the client trims to the same value before
+// building envelopes. Keeping one definition prevents the drift that silently
+// truncated observations at 10k while the validator allowed more.
+const MAX_STEP_CONTENT_LENGTH = MAX_STEP_CONTENT_CHARS;
 const MAX_STEP_NUMBER = 50_000;
 const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
 const ONE_HOUR_MS = 60 * 60 * 1000;

package/node_modules/moe-training/test/client/envelope-builder.test.js

@@ -3,7 +3,7 @@
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
 import { EnvelopeBuilder } from '../../client/envelope-builder.js';
-import { CHUNK_SIZE } from '../../shared/constants.js';
+import { CHUNK_SIZE, MAX_STEP_CONTENT_CHARS } from '../../shared/constants.js';
 
 const metadata = {
   model_engine: 'claude-opus-4-6',
@@ -74,12 +74,48 @@ describe('EnvelopeBuilder', () => {
     assert.deepEqual(close.outcome, outcome);
   });
 
-  it('truncates step content at 10000 characters', () => {
+  it('leaves content under the ingest ceiling untouched', () => {
     const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
-    const longContent = 'x'.repeat(15_000);
-    builder.addStep({ step: 1, type: 'thought', timestamp: 123, content: longContent });
-    const envelope = builder.flush();
-    assert.equal(envelope.trajectory_log[0].content.length, 10_000);
+    builder.addStep({ step: 1, type: 'thought', timestamp: 123, content: 'x'.repeat(15_000) });
+    const step = builder.flush().trajectory_log[0];
+    assert.equal(step.content.length, 15_000);
+    assert.ok(!step.truncated);
+  });
+
+  it('truncates step content at MAX_STEP_CONTENT_CHARS', () => {
+    const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
+    builder.addStep({ step: 1, type: 'thought', timestamp: 123, content: 'x'.repeat(150_000) });
+    const step = builder.flush().trajectory_log[0];
+    assert.equal(step.content.length, MAX_STEP_CONTENT_CHARS);
+  });
+
+  // Regression: a trim that doesn't flag itself looks like complete data to
+  // downstream training and silently corrupts the corpus.
+  it('never truncates silently — always sets truncated + original_token_count', () => {
+    const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
+    builder.addStep({ step: 1, type: 'observation', timestamp: 123, content: 'x'.repeat(150_000) });
+    const step = builder.flush().trajectory_log[0];
+    assert.equal(step.truncated, true);
+    assert.equal(step.original_token_count, 37_500); // 150k chars / 4
+  });
+
+  it('preserves a parser-supplied original_token_count when trimming again', () => {
+    const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
+    builder.addStep({
+      step: 1, type: 'observation', timestamp: 123,
+      content: 'x'.repeat(150_000), truncated: true, original_token_count: 99_000,
+    });
+    const step = builder.flush().trajectory_log[0];
+    // The parser saw the true original; our count only sees what survived it.
+    assert.equal(step.original_token_count, 99_000);
+  });
+
+  it('output always satisfies the ingest validator', () => {
+    const builder = new EnvelopeBuilder('sess_1', 'user_1', metadata);
+    builder.addStep({ step: 1, type: 'observation', timestamp: 123, content: 'x'.repeat(500_000) });
+    const step = builder.flush().trajectory_log[0];
+    assert.ok(step.content.length <= MAX_STEP_CONTENT_CHARS,
+      'client must trim to the validator ceiling or the whole session is rejected');
   });
 
   it('caps token_count at 100000', () => {

package/node_modules/moe-training/test/shared/envelope-schema.test.js

@@ -3,7 +3,7 @@
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
 import { validateEnvelope, STEP_TYPES } from '../../shared/envelope-schema.js';
-import { TRAINING_EXCLUSION_REASONS } from '../../shared/constants.js';
+import { TRAINING_EXCLUSION_REASONS, MAX_STEP_CONTENT_CHARS } from '../../shared/constants.js';
 
 const VALID_HMAC = 'a'.repeat(64);
 const VALID_APP_HASH = 'b'.repeat(64);
@@ -113,12 +113,19 @@ describe('envelope-schema', () => {
     assert.ok(result.errors.some(e => e.includes('500')));
   });
 
-  it('rejects step with content > 10KB', () => {
+  it('accepts step content up to MAX_STEP_CONTENT_CHARS', () => {
     const env = validEnvelope();
-    env.trajectory_log[0].content = 'x'.repeat(10_001);
+    env.trajectory_log[0].content = 'x'.repeat(MAX_STEP_CONTENT_CHARS);
+    const result = validateEnvelope(env);
+    assert.ok(!result.errors.some(e => e.includes('step.content exceeds')));
+  });
+
+  it('rejects step content over MAX_STEP_CONTENT_CHARS', () => {
+    const env = validEnvelope();
+    env.trajectory_log[0].content = 'x'.repeat(MAX_STEP_CONTENT_CHARS + 1);
     const result = validateEnvelope(env);
     assert.equal(result.valid, false);
-    assert.ok(result.errors.some(e => e.includes('10000')));
+    assert.ok(result.errors.some(e => e.includes(String(MAX_STEP_CONTENT_CHARS))));
   });
 
   it('rejects step with token_count > 100,000', () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "groove-dev",
-  "version": "0.27.179",
+  "version": "0.27.180",
   "description": "Open-source agent orchestration layer — the AI company OS. Local model agent engine (GGUF/Ollama/llama-server), HuggingFace model browser, MCP integrations (Slack, Gmail, Stripe, 15+), agent scheduling (cron), business roles (CMO, CFO, EA). GUI dashboard, multi-agent coordination, zero cold-start, infinite sessions. Works with Claude Code, Codex, Gemini CLI, Ollama, any local model.",
   "license": "FSL-1.1-Apache-2.0",
   "author": "Groove Dev <hello@groovedev.ai> (https://groovedev.ai)",

package/packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/cli",
-  "version": "0.27.179",
+  "version": "0.27.180",
   "description": "GROOVE CLI — manage AI coding agents from your terminal",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/packages/daemon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/daemon",
-  "version": "0.27.179",
+  "version": "0.27.180",
   "description": "GROOVE daemon — agent orchestration engine",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",

package/packages/gui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@groove-dev/gui",
-  "version": "0.27.179",
+  "version": "0.27.180",
   "description": "GROOVE GUI — visual agent control plane",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",