groove-dev 0.27.77 → 0.27.78

package/moe-training/server/stats.js

@@ -0,0 +1,129 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+
+import { readFileSync, statSync } from 'node:fs';
+
+const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+const MAX_LINES_PER_FILE = 100_000;
+
+export class CentralStats {
+  constructor(storage, ledger, sessionRegistry) {
+    this.storage = storage;
+    this.ledger = ledger;
+    this.sessionRegistry = sessionRegistry;
+  }
+
+  summary() {
+    const storageStats = this.storage.getTotalStats();
+    const activeSessions = this.sessionRegistry.getActiveSessions().length;
+    const totalPointsAwarded = this.ledger.getTotalPointsAwarded();
+
+    return {
+      totalEnvelopes: storageStats.totalEnvelopes,
+      totalSteps: storageStats.totalSteps,
+      totalSessions: storageStats.uniqueSessions,
+      activeSessions,
+      uniqueContributors: storageStats.uniqueContributors,
+      storageSizeMb: storageStats.storageSizeMb,
+      totalPointsAwarded,
+    };
+  }
+
+  dailyGrowth(days = 7) {
+    days = Math.max(1, Math.min(365, Number(days) || 7));
+    const storageDaily = this.storage.getDailyStats(days);
+    const creditDaily = this.ledger.getDailyCredits(days);
+
+    const creditMap = {};
+    for (const c of creditDaily) {
+      creditMap[c.date] = { points: c.totalPoints, sessions: c.totalSessions };
+    }
+
+    return storageDaily.map(d => ({
+      date: d.date,
+      envelopes: d.envelopeCount,
+      steps: d.stepCount,
+      sessions: creditMap[d.date]?.sessions || 0,
+      points: creditMap[d.date]?.points || 0,
+    }));
+  }
+
+  modelBreakdown() {
+    const files = this.storage._listFiles();
+    const models = {};
+
+    for (const file of files) {
+      try {
+        const stat = statSync(file);
+        if (stat.size > MAX_FILE_SIZE) continue;
+      } catch { continue; }
+
+      try {
+        const lines = readFileSync(file, 'utf-8').split('\n').filter(Boolean);
+        const capped = lines.slice(0, MAX_LINES_PER_FILE);
+        for (const line of capped) {
+          try {
+            const env = JSON.parse(line);
+            const model = env.metadata?.model_engine || 'unknown';
+            if (!models[model]) models[model] = { sessions: new Set(), steps: 0, points: 0 };
+            models[model].sessions.add(env.session_id);
+            models[model].steps += env.trajectory_log?.length || 0;
+          } catch { /* skip */ }
+        }
+      } catch { /* skip file */ }
+    }
+
+    const total = Object.values(models).reduce((sum, m) => sum + m.steps, 0) || 1;
+    const result = {};
+    for (const [model, data] of Object.entries(models)) {
+      result[model] = {
+        sessions: data.sessions.size,
+        steps: data.steps,
+        points: data.points,
+        percentage: +((data.steps / total) * 100).toFixed(1),
+      };
+    }
+    return result;
+  }
+
+  providerBreakdown() {
+    const files = this.storage._listFiles();
+    const providers = {};
+
+    for (const file of files) {
+      try {
+        const stat = statSync(file);
+        if (stat.size > MAX_FILE_SIZE) continue;
+      } catch { continue; }
+
+      try {
+        const lines = readFileSync(file, 'utf-8').split('\n').filter(Boolean);
+        const capped = lines.slice(0, MAX_LINES_PER_FILE);
+        for (const line of capped) {
+          try {
+            const env = JSON.parse(line);
+            const provider = env.metadata?.provider || 'unknown';
+            if (!providers[provider]) providers[provider] = { sessions: new Set(), steps: 0, points: 0 };
+            providers[provider].sessions.add(env.session_id);
+            providers[provider].steps += env.trajectory_log?.length || 0;
+          } catch { /* skip */ }
+        }
+      } catch { /* skip file */ }
+    }
+
+    const result = {};
+    for (const [provider, data] of Object.entries(providers)) {
+      result[provider] = { sessions: data.sessions.size, steps: data.steps, points: data.points };
+    }
+    return result;
+  }
+
+  topContributors(limit = 10) {
+    limit = Math.max(1, Math.min(1000, Number(limit) || 10));
+    const leaders = this.ledger.getLeaderboard(limit);
+    return leaders.map(l => ({
+      contributor_id: l.contributor_id.slice(0, 8) + '...',
+      total_points: l.total_points,
+      total_sessions: l.total_sessions,
+    }));
+  }
+}

package/moe-training/server/stitcher.js

@@ -0,0 +1,69 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+
+export class TrajectoryStitcher {
+  constructor(storage) {
+    this.storage = storage;
+  }
+
+  stitch(sessionId) {
+    const envelopes = this.storage.getSessionEnvelopes(sessionId);
+    if (!envelopes.length) return null;
+
+    const chunks = envelopes
+      .filter(e => e.type !== 'SESSION_CLOSE')
+      .sort((a, b) => (a.chunk_sequence ?? 0) - (b.chunk_sequence ?? 0));
+
+    const closeEnvelope = envelopes.find(e => e.type === 'SESSION_CLOSE');
+    const firstChunk = chunks[0];
+    if (!firstChunk) return null;
+
+    const allSteps = [];
+    for (const chunk of chunks) {
+      if (Array.isArray(chunk.trajectory_log)) {
+        allSteps.push(...chunk.trajectory_log);
+      }
+    }
+
+    allSteps.sort((a, b) => a.step - b.step);
+
+    const toolsUsed = new Set();
+    const stepTypeCounts = {};
+    let totalTokens = 0;
+
+    for (const step of allSteps) {
+      if (step.tool) toolsUsed.add(step.tool);
+      stepTypeCounts[step.type] = (stepTypeCounts[step.type] || 0) + 1;
+      totalTokens += step.token_count || 0;
+    }
+
+    return {
+      session_id: sessionId,
+      contributor_id: firstChunk.contributor_id,
+      metadata: firstChunk.metadata || {},
+      trajectory_log: allSteps,
+      outcome: closeEnvelope?.outcome || null,
+      total_steps: allSteps.length,
+      total_tokens: totalTokens,
+      unique_tools_used: [...toolsUsed],
+      step_type_distribution: stepTypeCounts,
+      total_chunks: chunks.length,
+    };
+  }
+
+  linkCoordination(trajectory) {
+    if (!trajectory || !trajectory.trajectory_log) return trajectory;
+
+    const coordSteps = trajectory.trajectory_log.filter(s => s.type === 'coordination' && s.coordination_id);
+
+    for (const step of coordSteps) {
+      step.coordination_partner = {
+        coordination_id: step.coordination_id,
+        direction: step.direction,
+        target_agent: step.target_agent || null,
+        linked: true,
+      };
+    }
+
+    return trajectory;
+  }
+}

package/moe-training/server/storage.js

@@ -0,0 +1,147 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+
+import { mkdirSync, appendFileSync, readFileSync, readdirSync, statSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+const QUOTA_BYTES = 50 * 1024 * 1024 * 1024; // 50GB
+const WARN_BYTES = 40 * 1024 * 1024 * 1024;  // 40GB
+
+export class EnvelopeStorage {
+  constructor(basePath = './data/envelopes') {
+    this.basePath = basePath;
+    if (!existsSync(basePath)) mkdirSync(basePath, { recursive: true, mode: 0o700 });
+  }
+
+  checkQuota() {
+    const files = this._listFiles();
+    let total = 0;
+    for (const file of files) {
+      try { total += statSync(file).size; } catch { /* skip */ }
+    }
+    if (total > WARN_BYTES && total <= QUOTA_BYTES) {
+      console.warn(`[storage] WARNING: storage at ${(total / (1024 * 1024 * 1024)).toFixed(1)}GB, approaching 50GB quota`);
+    }
+    return total < QUOTA_BYTES;
+  }
+
+  store(envelope) {
+    if (!this.checkQuota()) {
+      throw new Error('STORAGE_QUOTA_EXCEEDED');
+    }
+
+    const date = new Date().toISOString().slice(0, 10);
+    const filePath = join(this.basePath, `${date}.jsonl`);
+
+    if (!existsSync(this.basePath)) {
+      mkdirSync(this.basePath, { recursive: true, mode: 0o700 });
+    }
+
+    const line = JSON.stringify(envelope) + '\n';
+    appendFileSync(filePath, line, { mode: 0o600 });
+  }
+
+  getSessionEnvelopes(sessionId) {
+    const envelopes = [];
+    const files = this._listFiles();
+
+    for (const file of files) {
+      const lines = readFileSync(file, 'utf-8').split('\n').filter(Boolean);
+      for (const line of lines) {
+        try {
+          const env = JSON.parse(line);
+          if (env.session_id === sessionId) envelopes.push(env);
+        } catch { /* skip malformed lines */ }
+      }
+    }
+
+    return envelopes.sort((a, b) => (a.chunk_sequence ?? 0) - (b.chunk_sequence ?? 0));
+  }
+
+  getDateRange(startDate, endDate) {
+    const envelopes = [];
+    const files = this._listFiles();
+
+    for (const file of files) {
+      const fileName = file.split('/').pop().replace('.jsonl', '');
+      if (fileName < startDate || fileName > endDate) continue;
+
+      const lines = readFileSync(file, 'utf-8').split('\n').filter(Boolean);
+      for (const line of lines) {
+        try {
+          envelopes.push(JSON.parse(line));
+        } catch { /* skip */ }
+      }
+    }
+
+    return envelopes;
+  }
+
+  getDailyStats(days = 7) {
+    const stats = [];
+    const files = this._listFiles();
+
+    const cutoff = new Date();
+    cutoff.setDate(cutoff.getDate() - days);
+    const cutoffStr = cutoff.toISOString().slice(0, 10);
+
+    for (const file of files) {
+      const date = file.split('/').pop().replace('.jsonl', '');
+      if (date < cutoffStr) continue;
+
+      const lines = readFileSync(file, 'utf-8').split('\n').filter(Boolean);
+      let stepCount = 0;
+      for (const line of lines) {
+        try {
+          const env = JSON.parse(line);
+          stepCount += env.trajectory_log?.length ?? 0;
+        } catch { /* skip */ }
+      }
+
+      stats.push({ date, envelopeCount: lines.length, stepCount });
+    }
+
+    return stats.sort((a, b) => a.date.localeCompare(b.date));
+  }
+
+  getTotalStats() {
+    const files = this._listFiles();
+    let totalEnvelopes = 0;
+    let totalSteps = 0;
+    let storageSizeBytes = 0;
+    const contributors = new Set();
+    const sessions = new Set();
+
+    for (const file of files) {
+      const stat = statSync(file);
+      storageSizeBytes += stat.size;
+
+      const lines = readFileSync(file, 'utf-8').split('\n').filter(Boolean);
+      totalEnvelopes += lines.length;
+
+      for (const line of lines) {
+        try {
+          const env = JSON.parse(line);
+          totalSteps += env.trajectory_log?.length ?? 0;
+          if (env.contributor_id) contributors.add(env.contributor_id);
+          if (env.session_id) sessions.add(env.session_id);
+        } catch { /* skip */ }
+      }
+    }
+
+    return {
+      totalEnvelopes,
+      totalSteps,
+      storageSizeMb: +(storageSizeBytes / 1_048_576).toFixed(2),
+      uniqueContributors: contributors.size,
+      uniqueSessions: sessions.size,
+    };
+  }
+
+  _listFiles() {
+    if (!existsSync(this.basePath)) return [];
+    return readdirSync(this.basePath)
+      .filter(f => f.endsWith('.jsonl'))
+      .map(f => join(this.basePath, f))
+      .sort();
+  }
+}

package/moe-training/server/verifier.js

@@ -0,0 +1,102 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+
+import { verifyEnvelope } from '../shared/crypto.js';
+
+let validateEnvelope;
+try {
+  const schema = await import('../shared/envelope-schema.js');
+  validateEnvelope = schema.validateEnvelope;
+} catch {
+  validateEnvelope = () => ({ valid: true });
+}
+
+export class EnvelopeVerifier {
+  constructor(sessionRegistry) {
+    this.sessionRegistry = sessionRegistry;
+  }
+
+  verify(envelope) {
+    const sessionId = envelope.session_id;
+    if (!sessionId) return { valid: false, reason: 'missing session_id' };
+
+    const session = this.sessionRegistry.getSession(sessionId);
+    if (!session) return { valid: false, reason: 'unknown session_id' };
+    if (session.status !== 'active') return { valid: false, reason: 'session is closed' };
+
+    const attestation = envelope.attestation;
+    if (!attestation) return { valid: false, reason: 'missing attestation' };
+
+    if (!attestation.session_hmac || typeof attestation.session_hmac !== 'string' || attestation.session_hmac.length === 0) {
+      return { valid: false, reason: 'empty or missing HMAC' };
+    }
+
+    // HMAC covers the entire envelope EXCEPT the attestation block
+    const envelopeForHmac = { ...envelope };
+    delete envelopeForHmac.attestation;
+    const envelopeBytes = JSON.stringify(envelopeForHmac);
+
+    const hmacValid = verifyEnvelope(
+      session.shared_secret,
+      envelopeBytes,
+      attestation.sequence,
+      attestation.session_hmac
+    );
+    if (!hmacValid) return { valid: false, reason: 'HMAC verification failed' };
+
+    // Atomic sequence check + increment (prevents race conditions)
+    const seqResult = this.sessionRegistry.checkAndIncrementSequence(sessionId, attestation.sequence);
+    if (!seqResult.valid) {
+      return { valid: false, reason: seqResult.reason };
+    }
+
+    const schemaResult = validateEnvelope(envelope);
+    if (!schemaResult.valid) {
+      return { valid: false, reason: `schema validation failed: ${schemaResult.reason || schemaResult.errors?.join(', ')}` };
+    }
+
+    return { valid: true };
+  }
+
+  verifyClose(envelope) {
+    const sessionId = envelope.session_id;
+    if (!sessionId) return { valid: false, reason: 'missing session_id' };
+
+    const session = this.sessionRegistry.getSession(sessionId);
+    if (!session) return { valid: false, reason: 'unknown session_id' };
+    if (session.status !== 'active') return { valid: false, reason: 'session already closed' };
+
+    const attestation = envelope.attestation;
+    if (!attestation) return { valid: false, reason: 'missing attestation' };
+
+    if (!attestation.session_hmac || typeof attestation.session_hmac !== 'string' || attestation.session_hmac.length === 0) {
+      return { valid: false, reason: 'empty or missing HMAC' };
+    }
+
+    // HMAC covers the entire envelope EXCEPT the attestation block
+    const envelopeForHmac = { ...envelope };
+    delete envelopeForHmac.attestation;
+    const envelopeBytes = JSON.stringify(envelopeForHmac);
+
+    const hmacValid = verifyEnvelope(
+      session.shared_secret,
+      envelopeBytes,
+      attestation.sequence,
+      attestation.session_hmac
+    );
+    if (!hmacValid) return { valid: false, reason: 'HMAC verification failed' };
+
+    // Sequence check for close envelopes too
+    const seqResult = this.sessionRegistry.checkAndIncrementSequence(sessionId, attestation.sequence);
+    if (!seqResult.valid) {
+      return { valid: false, reason: seqResult.reason };
+    }
+
+    const schemaResult = validateEnvelope(envelope);
+    if (!schemaResult.valid) {
+      return { valid: false, reason: `schema validation failed: ${schemaResult.reason || schemaResult.errors?.join(', ')}` };
+    }
+
+    this.sessionRegistry.closeSession(sessionId);
+    return { valid: true };
+  }
+}

package/moe-training/shared/constants.js

@@ -0,0 +1,30 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+
+export const CURRENT_CONSENT_VERSION = '1.0';
+export const CHUNK_SIZE = 200;
+export const CHUNK_TIMEOUT_MS = 300_000;
+export const MAX_QUEUE_SIZE = 10_000;
+export const OBSERVATION_TRUNCATE_HEAD = 50;
+export const OBSERVATION_TRUNCATE_TAIL = 20;
+export const SUPPORTED_PROVIDERS = ['claude-code', 'codex', 'gemini'];
+
+export const MODEL_TIERS = {
+  'claude-opus-4-6': 5,
+  'claude-opus-4-7': 5,
+  'claude-sonnet-4-6': 3,
+  'gpt-4.5': 5,
+  'o3': 5,
+  'o4-mini': 2,
+  'gemini-2.5-pro': 3,
+  'gemini-2.5-flash': 1.5,
+};
+
+export const QUALITY_MULTIPLIERS = {
+  correction: 10,
+  coordination: 5,
+  errorRecovery: 3,
+  heavyTask: 2,
+  highQuality: 1.5,
+};
+
+export const CENTRAL_COMMAND_URL = process.env.GROOVE_CENTRAL_URL || 'https://api.groovedev.ai';

package/moe-training/shared/crypto.js

@@ -0,0 +1,45 @@
+// FSL-1.1-Apache-2.0 — see LICENSE
+
+import { createECDH, createHmac, createHash, randomBytes } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+
+export function generateECDHKeypair() {
+  const ecdh = createECDH('prime256v1');
+  ecdh.generateKeys();
+  return {
+    publicKey: ecdh.getPublicKey('base64'),
+    privateKey: ecdh.getPrivateKey('base64'),
+  };
+}
+
+export function deriveSharedSecret(privateKeyB64, otherPublicKeyB64) {
+  const ecdh = createECDH('prime256v1');
+  ecdh.setPrivateKey(Buffer.from(privateKeyB64, 'base64'));
+  const secret = ecdh.computeSecret(Buffer.from(otherPublicKeyB64, 'base64'));
+  return secret.toString('base64');
+}
+
+// envelopeBytes MUST contain ALL envelope fields EXCEPT attestation (which holds the HMAC itself).
+// Callers must strip attestation before JSON.stringify to produce envelopeBytes.
+export function signEnvelope(sharedSecretB64, envelopeBytes, sequenceNumber) {
+  const key = Buffer.from(sharedSecretB64, 'base64');
+  const seqBuf = Buffer.alloc(4);
+  seqBuf.writeUInt32BE(sequenceNumber);
+  const payload = Buffer.concat([seqBuf, Buffer.from(envelopeBytes)]);
+  return createHmac('sha256', key).update(payload).digest('hex');
+}
+
+export function verifyEnvelope(sharedSecretB64, envelopeBytes, sequenceNumber, hmac) {
+  const expected = signEnvelope(sharedSecretB64, envelopeBytes, sequenceNumber);
+  if (expected.length !== hmac.length) return false;
+  let diff = 0;
+  for (let i = 0; i < expected.length; i++) {
+    diff |= expected.charCodeAt(i) ^ hmac.charCodeAt(i);
+  }
+  return diff === 0;
+}
+
+export function computeAppHash(filePath) {
+  const content = readFileSync(filePath);
+  return createHash('sha256').update(content).digest('hex');
+}