@cccarv82/freya 3.5.2 → 3.6.1

package/templates/base/scripts/retroactive-ingest.js

@@ -0,0 +1,386 @@
+#!/usr/bin/env node
+/**
+ * retroactive-ingest.js
+ *
+ * Reads ALL existing daily logs, sends each through the Copilot CLI planner
+ * to extract tasks/blockers, and applies them to SQLite.
+ * Also generates embeddings for all daily logs.
+ *
+ * Usage:
+ *   node scripts/retroactive-ingest.js [--dry-run] [--days N] [--embeddings-only]
+ *
+ * Options:
+ *   --dry-run          Show what would be extracted without writing to SQLite
+ *   --days N           Only process the last N days (default: all)
+ *   --embeddings-only  Skip planner, only generate embeddings for existing logs
+ *   --skip-embeddings  Skip embedding generation (only extract tasks/blockers)
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { spawn } = require('child_process');
+const os = require('os');
+
+// Parse args
+const args = process.argv.slice(2);
+const DRY_RUN = args.includes('--dry-run');
+const EMBEDDINGS_ONLY = args.includes('--embeddings-only');
+const SKIP_EMBEDDINGS = args.includes('--skip-embeddings');
+const daysIdx = args.indexOf('--days');
+const MAX_DAYS = daysIdx >= 0 ? parseInt(args[daysIdx + 1], 10) : 0;
+
+// Resolve workspace directory
+const workspaceDir = process.env.FREYA_WORKSPACE_DIR
+  ? path.resolve(process.env.FREYA_WORKSPACE_DIR)
+  : path.join(__dirname, '..');
+
+const { defaultInstance: dl, ready } = require('./lib/DataLayer');
+const DataManager = require('./lib/DataManager');
+
+function sha1(text) {
+  return crypto.createHash('sha1').update(text).digest('hex');
+}
+
+function normalizeWhitespace(t) {
+  return String(t || '').replace(/\s+/g, ' ').trim();
+}
+
+function normalizeTextForKey(t) {
+  return normalizeWhitespace(t).toLowerCase();
+}
+
+function run(cmd, args, cwd, extraEnv) {
+  return new Promise((resolve) => {
+    let child;
+    const env = extraEnv ? { ...process.env, ...extraEnv } : process.env;
+    try {
+      if (process.platform === 'win32') {
+        const comspec = process.env.ComSpec || 'cmd.exe';
+        child = spawn(comspec, ['/d', '/s', '/c', cmd, ...args], { cwd, shell: false, env });
+      } else {
+        child = spawn(cmd, args, { cwd, shell: false, env });
+      }
+    } catch (e) {
+      return resolve({ code: 1, stdout: '', stderr: e.message || String(e) });
+    }
+    let stdout = '';
+    let stderr = '';
+    child.stdout && child.stdout.on('data', (d) => { stdout += d.toString(); });
+    child.stderr && child.stderr.on('data', (d) => { stderr += d.toString(); });
+    child.on('error', (e) => { stderr += `\n${e.message}`; resolve({ code: 1, stdout, stderr }); });
+    child.on('close', (code) => resolve({ code: code ?? 0, stdout, stderr }));
+  });
+}
+
+function extractFirstJsonObject(text) {
+  if (!text) return null;
+  const start = text.indexOf('{');
+  if (start === -1) return null;
+  let depth = 0;
+  for (let i = start; i < text.length; i++) {
+    if (text[i] === '{') depth++;
+    else if (text[i] === '}') { depth--; if (depth === 0) return text.slice(start, i + 1); }
+  }
+  return null;
+}
+
+function escapeJsonControlChars(jsonText) {
+  return jsonText.replace(/[\x00-\x1F\x7F]/g, (ch) => {
+    if (ch === '\n' || ch === '\r' || ch === '\t') return ch;
+    return '\\u' + ch.charCodeAt(0).toString(16).padStart(4, '0');
+  });
+}
+
+function readProjectSlugMap(wsDir) {
+  const p = path.join(wsDir, 'data', 'settings', 'project-slug-map.json');
+  try {
+    return JSON.parse(fs.readFileSync(p, 'utf8'));
+  } catch { return {}; }
+}
+
+function inferProjectSlug(text, map) {
+  if (!text || !map || typeof map !== 'object') return '';
+  const lower = text.toLowerCase();
+  let bestSlug = '';
+  let bestLen = 0;
+  for (const [keyword, slug] of Object.entries(map)) {
+    if (lower.includes(keyword.toLowerCase()) && keyword.length > bestLen) {
+      bestSlug = slug;
+      bestLen = keyword.length;
+    }
+  }
+  return bestSlug;
+}
+
+async function main() {
+  await ready;
+
+  console.log('╔══════════════════════════════════════════════════════╗');
+  console.log('║  FREYA — Retroactive Ingestion                     ║');
+  console.log('╚══════════════════════════════════════════════════════╝');
+  console.log(`Workspace: ${workspaceDir}`);
+  console.log(`Mode: ${DRY_RUN ? 'DRY RUN' : EMBEDDINGS_ONLY ? 'EMBEDDINGS ONLY' : 'FULL INGESTION'}`);
+  console.log('');
+
+  const logsDir = path.join(workspaceDir, 'logs', 'daily');
+  if (!fs.existsSync(logsDir)) {
+    console.log('❌ No daily logs directory found at:', logsDir);
+    process.exit(1);
+  }
+
+  let files = fs.readdirSync(logsDir)
+    .filter(f => /^\d{4}-\d{2}-\d{2}\.md$/.test(f))
+    .sort();
+
+  if (MAX_DAYS > 0) {
+    files = files.slice(-MAX_DAYS);
+  }
+
+  console.log(`📁 Found ${files.length} daily log files to process`);
+  console.log('');
+
+  // Step 1: Sync all daily logs to SQLite
+  console.log('── Step 1: Syncing daily logs to SQLite ──');
+  const upsert = dl.db.prepare(`
+    INSERT INTO daily_logs (date, raw_markdown) VALUES (?, ?)
+    ON CONFLICT(date) DO UPDATE SET raw_markdown = excluded.raw_markdown
+  `);
+  const syncTx = dl.db.transaction((fileList) => {
+    for (const file of fileList) {
+      const date = file.replace('.md', '');
+      const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
+      upsert.run(date, content);
+    }
+  });
+  syncTx(files);
+  console.log(`✅ ${files.length} daily logs synced to SQLite`);
+  console.log('');
+
+  // Step 2: Generate embeddings for all daily logs
+  if (!SKIP_EMBEDDINGS) {
+    console.log('── Step 2: Generating embeddings ──');
+    const dm = new DataManager(workspaceDir, logsDir);
+    let totalChunks = 0;
+    for (let i = 0; i < files.length; i++) {
+      const date = files[i].replace('.md', '');
+      const content = fs.readFileSync(path.join(logsDir, files[i]), 'utf8');
+      try {
+        const count = await dm.generateEmbeddings('daily_log', date, content);
+        totalChunks += count;
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ${count} chunks`);
+      } catch (err) {
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ ${err.message}`);
+      }
+    }
+    console.log(`\n✅ Generated ${totalChunks} embedding chunks total`);
+    console.log('');
+  }
+
+  if (EMBEDDINGS_ONLY) {
+    console.log('── Embeddings-only mode. Skipping task/blocker extraction. ──');
+    dl.db.save();
+    console.log('\n✅ Done!');
+    return;
+  }
+
+  // Step 3: Extract tasks/blockers from each daily log via planner
+  console.log('── Step 3: Extracting tasks & blockers via planner ──');
+
+  const cmd = process.env.COPILOT_CMD || 'copilot';
+  const agentEnv = { FREYA_WORKSPACE_DIR: workspaceDir };
+  const slugMap = readProjectSlugMap(workspaceDir);
+  const validTaskCats = new Set(['DO_NOW', 'SCHEDULE', 'DELEGATE', 'IGNORE']);
+
+  const schema = {
+    actions: [
+      { type: 'create_task', description: '<string>', priority: 'HIGH|MEDIUM|LOW', category: 'DO_NOW|SCHEDULE|DELEGATE|IGNORE', projectSlug: '<string optional>' },
+      { type: 'create_blocker', title: '<string>', severity: 'CRITICAL|HIGH|MEDIUM|LOW', notes: '<string>', projectSlug: '<string optional>' }
+    ]
+  };
+
+  const sysInstructions = `Você é o planner do sistema F.R.E.Y.A.
+
+Analise o daily log abaixo e extraia TODAS as tarefas e blockers mencionados.
+Procure por: ações mencionadas, pendências, problemas, impedimentos, decisões que geram trabalho.
+Se NÃO houver tarefas ou blockers claros, retorne: {"actions":[]}
+Retorne APENAS JSON válido no formato: ${JSON.stringify(schema)}
+NÃO use code fences. NÃO inclua texto extra.
+IMPORTANTE: Extraia APENAS informações explícitas do log. NÃO invente dados.`;
+
+  let totalTasks = 0;
+  let totalBlockers = 0;
+  let totalSkipped = 0;
+  let totalErrors = 0;
+
+  const insertTask = dl.db.prepare(`INSERT OR IGNORE INTO tasks (id, project_slug, description, category, status, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+  const insertBlocker = dl.db.prepare(`INSERT OR IGNORE INTO blockers (id, project_slug, title, severity, status, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+
+  // Build existing keys for dedup
+  const existingTaskDescs = new Set(
+    dl.db.prepare("SELECT description FROM tasks").all().map(t => sha1(normalizeTextForKey(t.description)))
+  );
+  const existingBlockerTitles = new Set(
+    dl.db.prepare("SELECT title FROM blockers").all().map(b => sha1(normalizeTextForKey(b.title)))
+  );
+
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    const date = file.replace('.md', '');
+    const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
+
+    // Skip very small logs (< 50 chars) — likely empty or just a header
+    if (content.trim().length < 50) {
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — skipped (too small)`);
+      continue;
+    }
+
+    const fullPrompt = `${sysInstructions}\n\nDAILY LOG (${date}):\n${content}\n`;
+    const SAFE_ARG_LEN = 24000;
+
+    try {
+      let r;
+      if (fullPrompt.length > SAFE_ARG_LEN) {
+        const tmpFile = path.join(os.tmpdir(), `freya-retro-${Date.now()}.txt`);
+        fs.writeFileSync(tmpFile, fullPrompt, 'utf8');
+        const filePrompt = `Leia o arquivo abaixo e extraia tasks/blockers conforme as instruções contidas nele.\nARQUIVO: ${tmpFile}`;
+        r = await run(cmd, ['-s', '--no-color', '--stream', 'off', '--add-dir', os.tmpdir(), '--allow-all-tools', '-p', filePrompt], workspaceDir, agentEnv);
+        try { fs.unlinkSync(tmpFile); } catch { }
+      } else {
+        r = await run(cmd, ['-s', '--no-color', '--stream', 'off', '-p', fullPrompt], workspaceDir, agentEnv);
+      }
+
+      const out = (r.stdout + r.stderr).trim();
+      if (r.code !== 0 || !out) {
+        totalErrors++;
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ planner error     `);
+        continue;
+      }
+
+      // Parse plan
+      const jsonText = extractFirstJsonObject(out) || out;
+      let plan;
+      try {
+        plan = JSON.parse(jsonText);
+      } catch {
+        try { plan = JSON.parse(escapeJsonControlChars(jsonText)); } catch {
+          totalErrors++;
+          process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ invalid JSON     `);
+          continue;
+        }
+      }
+
+      const actions = Array.isArray(plan.actions) ? plan.actions : [];
+      let fileTasks = 0;
+      let fileBlockers = 0;
+      let fileSkipped = 0;
+
+      if (!DRY_RUN) {
+        const applyTx = dl.db.transaction(() => {
+          for (const a of actions) {
+            if (!a || typeof a !== 'object') continue;
+
+            if (a.type === 'create_task' && a.description) {
+              const desc = normalizeWhitespace(a.description);
+              if (!desc) continue;
+              const projectSlug = String(a.projectSlug || '').trim() || inferProjectSlug(desc, slugMap);
+              const key = sha1(normalizeTextForKey((projectSlug ? projectSlug + ' ' : '') + desc));
+              if (existingTaskDescs.has(key)) { fileSkipped++; continue; }
+
+              const id = `t-retro-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+              const category = validTaskCats.has(String(a.category || '').trim()) ? String(a.category).trim() : 'DO_NOW';
+              const metadata = JSON.stringify({ priority: a.priority || 'medium', source: 'retroactive', sourceDate: date });
+              // Use the log date as created_at for chronological accuracy
+              insertTask.run(id, projectSlug || null, desc, category, 'PENDING', `${date}T12:00:00.000Z`, metadata);
+              existingTaskDescs.add(key);
+              fileTasks++;
+            }
+
+            if (a.type === 'create_blocker' && a.title) {
+              const title = normalizeWhitespace(a.title);
+              if (!title) continue;
+              const notes = normalizeWhitespace(a.notes);
+              const projectSlug = String(a.projectSlug || '').trim() || inferProjectSlug(title + ' ' + notes, slugMap);
+              const key = sha1(normalizeTextForKey((projectSlug ? projectSlug + ' ' : '') + title));
+              if (existingBlockerTitles.has(key)) { fileSkipped++; continue; }
+
+              const id = `b-retro-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+              const severity = String(a.severity || 'MEDIUM').toUpperCase();
+              const metadata = JSON.stringify({ description: notes || title, source: 'retroactive', sourceDate: date });
+              insertBlocker.run(id, projectSlug || null, title, severity, 'OPEN', `${date}T12:00:00.000Z`, metadata);
+              existingBlockerTitles.add(key);
+              fileBlockers++;
+            }
+          }
+        });
+        applyTx();
+      } else {
+        // Dry run — just count
+        for (const a of actions) {
+          if (a && a.type === 'create_task' && a.description) fileTasks++;
+          if (a && a.type === 'create_blocker' && a.title) fileBlockers++;
+        }
+      }
+
+      totalTasks += fileTasks;
+      totalBlockers += fileBlockers;
+      totalSkipped += fileSkipped;
+
+      const status = fileTasks || fileBlockers
+        ? `${fileTasks}T ${fileBlockers}B${fileSkipped ? ` (${fileSkipped} dup)` : ''}`
+        : 'no actions';
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ${status}              `);
+
+      // Small delay to avoid rate limiting
+      if (i < files.length - 1) {
+        await new Promise(r => setTimeout(r, 500));
+      }
+
+    } catch (err) {
+      totalErrors++;
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ ${err.message}     `);
+    }
+  }
+
+  // Ensure data is persisted
+  dl.db.save();
+
+  console.log('\n');
+  console.log('══════════════════════════════════════════════════════');
+  console.log(`  📊 Results ${DRY_RUN ? '(DRY RUN)' : ''}`);
+  console.log(`     Tasks created:    ${totalTasks}`);
+  console.log(`     Blockers created: ${totalBlockers}`);
+  console.log(`     Duplicates skipped: ${totalSkipped}`);
+  console.log(`     Errors:           ${totalErrors}`);
+  console.log('══════════════════════════════════════════════════════');
+
+  // Step 4: Generate embeddings for newly created tasks/blockers
+  if (!DRY_RUN && !SKIP_EMBEDDINGS && (totalTasks > 0 || totalBlockers > 0)) {
+    console.log('\n── Step 4: Generating embeddings for new tasks/blockers ──');
+    const dm = new DataManager(workspaceDir, logsDir);
+    const newTasks = dl.db.prepare("SELECT id, description FROM tasks WHERE json_extract(metadata, '$.source') = 'retroactive'").all();
+    const newBlockers = dl.db.prepare("SELECT id, title, json_extract(metadata, '$.description') as notes FROM blockers WHERE json_extract(metadata, '$.source') = 'retroactive'").all();
+
+    let embCount = 0;
+    for (const t of newTasks) {
+      try {
+        embCount += await dm.generateEmbeddings('task', t.id, t.description);
+      } catch { }
+    }
+    for (const b of newBlockers) {
+      try {
+        embCount += await dm.generateEmbeddings('blocker', b.id, b.title + ' ' + (b.notes || ''));
+      } catch { }
+    }
+    console.log(`✅ Generated ${embCount} embedding chunks for new entities`);
+  }
+
+  console.log('\n✅ Retroactive ingestion complete!');
+}
+
+main().catch(err => {
+  console.error('\n❌ Fatal error:', err.message || err);
+  process.exit(1);
+});