@cccarv82/freya 3.7.7 → 3.8.0

package/bin/freya.js

@@ -8,11 +8,17 @@ const { cmdWeb } = require('../cli/web');
 const DEFAULT_PORT = 3872;
 
 function usage() {
-  return `FREYA web
+  return `FREYA — Personal AI Assistant
 
 Usage:
   freya [--port <n>] [--dir <path>] [--no-open] [--dev]
+  freya retroactive-ingest [--dry-run] [--days N] [--embeddings-only] [--skip-embeddings]
   freya --version | -v
+
+Commands:
+  (default)            Start the web server
+  retroactive-ingest   Process all historical daily logs (sync, embeddings, task extraction)
+
 Options:
   --port <n>    Port to bind (default: ${DEFAULT_PORT})
   --dir <path>  Workspace directory (default: ./freya)
@@ -22,8 +28,8 @@ Options:
 Examples:
   freya
   freya --port 4000
-  freya --dir ./freya
-  freya --no-open
+  freya retroactive-ingest
+  freya retroactive-ingest --days 7
 `;
 }
 
@@ -56,7 +62,6 @@ function parseArgs(argv) {
 
 async function run(argv) {
   const { args, flags, kv } = parseArgs(argv);
-  const positionals = args.filter((a) => a !== 'web');
 
   if (flags.has('--help') || flags.has('-h')) {
     process.stdout.write(usage());
@@ -69,8 +74,43 @@ async function run(argv) {
     return;
   }
 
+  // Check for subcommands
+  const subcommand = args[0] || '';
+
+  if (subcommand === 'retroactive-ingest') {
+    // Run retroactive ingestion directly from the global package
+    const dir = kv['--dir'] ? path.resolve(process.cwd(), kv['--dir']) : path.resolve(process.cwd(), 'freya');
+    // Ensure workspace exists first
+    const { initWorkspace } = require('../cli/init');
+    const { autoUpdate } = require('../cli/auto-update');
+    try {
+      const fs = require('fs');
+      if (!fs.existsSync(dir)) {
+        await initWorkspace({ targetDir: dir, force: false, forceData: false, forceLogs: false });
+        console.log('[FREYA] Workspace initialized at', dir);
+      }
+      await autoUpdate(dir);
+    } catch (e) {
+      console.error('[FREYA] Warning:', e.message || String(e));
+    }
+    // Forward remaining flags
+    const fwdArgs = [];
+    if (flags.has('--dry-run')) fwdArgs.push('--dry-run');
+    if (flags.has('--embeddings-only')) fwdArgs.push('--embeddings-only');
+    if (flags.has('--skip-embeddings')) fwdArgs.push('--skip-embeddings');
+    if (kv['--days']) { fwdArgs.push('--days'); fwdArgs.push(kv['--days']); }
+    // Set workspace dir and run directly
+    process.env.FREYA_WORKSPACE_DIR = dir;
+    // Inject args for the script to parse
+    process.argv = [process.argv[0], 'retroactive-ingest', ...fwdArgs];
+    require('../cli/retroactive-ingest')(dir, fwdArgs);
+    return;
+  }
+
+  // Filter out 'web' positional for backwards compat
+  const positionals = args.filter((a) => a !== 'web');
   if (positionals.length > 0) {
-    process.stderr.write(`Unknown arguments: ${positionals.join(' ')}\n`);
+    process.stderr.write(`Unknown command: ${positionals.join(' ')}\n`);
     process.stdout.write(usage());
     process.exitCode = 1;
     return;

package/cli/init.js

@@ -89,7 +89,7 @@ function ensurePackageJson(targetDir, force, summary) {
     daily: 'node scripts/generate-daily-summary.js',
     status: 'node scripts/generate-executive-report.js',
     blockers: 'node scripts/generate-blockers-report.js',
-    'retroactive-ingest': 'node scripts/retroactive-ingest.js'
+    'retroactive-ingest': 'freya retroactive-ingest'
   };
 
   const depsToEnsure = {

package/cli/retroactive-ingest.js

@@ -0,0 +1,430 @@
+/**
+ * cli/retroactive-ingest.js
+ *
+ * Retroactive ingestion — runs from the GLOBAL package (not workspace copy).
+ * Reuses the same DataLayer, DataManager, run(), getCopilotCmd() as web.js.
+ *
+ * Called from: freya retroactive-ingest [flags]
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { spawn, execSync } = require('child_process');
+const os = require('os');
+const { defaultInstance: dl, ready, configure: configureDataLayer } = require('../scripts/lib/DataLayer');
+let DataManager;
+try { DataManager = require('../scripts/lib/DataManager'); } catch { DataManager = null; }
+
+function sha1(text) {
+  return crypto.createHash('sha1').update(text).digest('hex');
+}
+function normalizeWhitespace(t) {
+  return String(t || '').replace(/\s+/g, ' ').trim();
+}
+function normalizeTextForKey(t) {
+  return normalizeWhitespace(t).toLowerCase();
+}
+
+// ---------- run() — exact copy from web.js ----------
+function run(cmd, args, cwd, extraEnv) {
+  return new Promise((resolve) => {
+    let child;
+    const env = extraEnv ? { ...process.env, ...extraEnv } : process.env;
+    try {
+      if (process.platform === 'win32') {
+        const escapedArgs = args.map(a => {
+          const escaped = String(a).replace(/'/g, "''");
+          return `'${escaped}'`;
+        });
+        const psCommand = `& '${cmd}' ${escapedArgs.join(' ')}`;
+        child = spawn('powershell.exe', [
+          '-NoLogo', '-Command', psCommand
+        ], { cwd, env, windowsHide: true });
+      } else {
+        child = spawn(cmd, args, { cwd, shell: true, env });
+      }
+    } catch (e) {
+      return resolve({ code: 1, stdout: '', stderr: e.message || String(e) });
+    }
+    let stdout = '';
+    let stderr = '';
+    child.stdout && child.stdout.on('data', (d) => { stdout += d.toString(); });
+    child.stderr && child.stderr.on('data', (d) => { stderr += d.toString(); });
+    child.on('error', (e) => { stderr += `\n${e.message}`; resolve({ code: 1, stdout, stderr }); });
+    child.on('close', (code) => resolve({ code: code ?? 0, stdout, stderr }));
+  });
+}
+
+// ---------- getCopilotCmd() — exact copy from web.js ----------
+let _copilotPathCache = null;
+function getCopilotCmd() {
+  if (_copilotPathCache !== null) return _copilotPathCache;
+
+  if (process.env.COPILOT_CMD) {
+    _copilotPathCache = process.env.COPILOT_CMD;
+    return _copilotPathCache;
+  }
+
+  if (process.platform === 'win32') {
+    try {
+      const p = execSync('where copilot 2>nul', { encoding: 'utf8', timeout: 5000 }).trim().split(/\r?\n/)[0];
+      if (p && fs.existsSync(p)) { _copilotPathCache = p; return p; }
+    } catch { }
+    try {
+      const p = execSync(
+        'powershell.exe -NoProfile -Command "(Get-Command copilot -ErrorAction SilentlyContinue).Source"',
+        { encoding: 'utf8', timeout: 10000 }
+      ).trim();
+      if (p && fs.existsSync(p)) { _copilotPathCache = p; return p; }
+    } catch { }
+    try {
+      const p = execSync(
+        'powershell.exe -NoProfile -Command "(Get-Command gh -ErrorAction SilentlyContinue).Source"',
+        { encoding: 'utf8', timeout: 10000 }
+      ).trim();
+      if (p && fs.existsSync(p)) { _copilotPathCache = `gh-copilot:${p}`; return _copilotPathCache; }
+    } catch { }
+  } else {
+    try {
+      const p = execSync('which copilot 2>/dev/null', { encoding: 'utf8', timeout: 5000 }).trim();
+      if (p) { _copilotPathCache = p; return p; }
+    } catch { }
+    try {
+      const p = execSync('which gh 2>/dev/null', { encoding: 'utf8', timeout: 5000 }).trim();
+      if (p) { _copilotPathCache = `gh-copilot:${p}`; return _copilotPathCache; }
+    } catch { }
+  }
+
+  _copilotPathCache = 'copilot';
+  return _copilotPathCache;
+}
+
+function copilotSpawnArgs(copilotCmd, extraArgs) {
+  if (copilotCmd.startsWith('gh-copilot:')) {
+    const ghPath = copilotCmd.slice('gh-copilot:'.length);
+    return { cmd: ghPath, args: ['copilot', ...extraArgs] };
+  }
+  return { cmd: copilotCmd, args: extraArgs };
+}
+
+// ---------- Helpers ----------
+function extractFirstJsonObject(text) {
+  if (!text) return null;
+  const start = text.indexOf('{');
+  if (start === -1) return null;
+  let depth = 0;
+  for (let i = start; i < text.length; i++) {
+    if (text[i] === '{') depth++;
+    else if (text[i] === '}') { depth--; if (depth === 0) return text.slice(start, i + 1); }
+  }
+  return null;
+}
+
+function escapeJsonControlChars(jsonText) {
+  return jsonText.replace(/[\x00-\x1F\x7F]/g, (ch) => {
+    if (ch === '\n' || ch === '\r' || ch === '\t') return ch;
+    return '\\u' + ch.charCodeAt(0).toString(16).padStart(4, '0');
+  });
+}
+
+function readProjectSlugMap(wsDir) {
+  const p = path.join(wsDir, 'data', 'settings', 'project-slug-map.json');
+  try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return {}; }
+}
+
+function inferProjectSlug(text, map) {
+  if (!text || !map || typeof map !== 'object') return '';
+  const lower = text.toLowerCase();
+  let bestSlug = '', bestLen = 0;
+  for (const [keyword, slug] of Object.entries(map)) {
+    if (lower.includes(keyword.toLowerCase()) && keyword.length > bestLen) {
+      bestSlug = slug; bestLen = keyword.length;
+    }
+  }
+  return bestSlug;
+}
+
+// ---------- Main ----------
+async function retroactiveIngest(workspaceDir, fwdArgs) {
+  const DRY_RUN = fwdArgs.includes('--dry-run');
+  const EMBEDDINGS_ONLY = fwdArgs.includes('--embeddings-only');
+  const SKIP_EMBEDDINGS = fwdArgs.includes('--skip-embeddings');
+  const daysIdx = fwdArgs.indexOf('--days');
+  const MAX_DAYS = daysIdx >= 0 ? parseInt(fwdArgs[daysIdx + 1], 10) : 0;
+
+  // Configure DataLayer to use workspace DB
+  await configureDataLayer(workspaceDir);
+  await ready;
+
+  console.log('╔══════════════════════════════════════════════════════╗');
+  console.log('║  FREYA — Retroactive Ingestion                     ║');
+  console.log('╚══════════════════════════════════════════════════════╝');
+  console.log(`Workspace: ${workspaceDir}`);
+  console.log(`Mode: ${DRY_RUN ? 'DRY RUN' : EMBEDDINGS_ONLY ? 'EMBEDDINGS ONLY' : 'FULL INGESTION'}`);
+  console.log('');
+
+  const logsDir = path.join(workspaceDir, 'logs', 'daily');
+  if (!fs.existsSync(logsDir)) {
+    console.log('❌ No daily logs directory found at:', logsDir);
+    process.exit(1);
+  }
+
+  let files = fs.readdirSync(logsDir)
+    .filter(f => /^\d{4}-\d{2}-\d{2}\.md$/.test(f))
+    .sort();
+
+  if (MAX_DAYS > 0) files = files.slice(-MAX_DAYS);
+
+  console.log(`📁 Found ${files.length} daily log files to process`);
+  console.log('');
+
+  // Step 1: Sync all daily logs to SQLite
+  console.log('── Step 1: Syncing daily logs to SQLite ──');
+  const upsert = dl.db.prepare(`
+    INSERT INTO daily_logs (date, raw_markdown) VALUES (?, ?)
+    ON CONFLICT(date) DO UPDATE SET raw_markdown = excluded.raw_markdown
+  `);
+  const syncTx = dl.db.transaction((fileList) => {
+    for (const file of fileList) {
+      const date = file.replace('.md', '');
+      const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
+      upsert.run(date, content);
+    }
+  });
+  syncTx(files);
+  console.log(`✅ ${files.length} daily logs synced to SQLite`);
+  console.log('');
+
+  // Step 2: Generate embeddings
+  if (!SKIP_EMBEDDINGS) {
+    console.log('── Step 2: Generating embeddings ──');
+    const dm = new DataManager(workspaceDir, logsDir);
+    let totalChunks = 0;
+    for (let i = 0; i < files.length; i++) {
+      const date = files[i].replace('.md', '');
+      const content = fs.readFileSync(path.join(logsDir, files[i]), 'utf8');
+      try {
+        const count = await dm.generateEmbeddings('daily_log', date, content);
+        totalChunks += count;
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ${count} chunks`);
+      } catch (err) {
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ ${err.message}`);
+      }
+    }
+    console.log(`\n✅ Generated ${totalChunks} embedding chunks total`);
+    console.log('');
+  }
+
+  if (EMBEDDINGS_ONLY) {
+    console.log('── Embeddings-only mode. Skipping task/blocker extraction. ──');
+    dl.db.save();
+    console.log('\n✅ Done!');
+    return;
+  }
+
+  // Step 3: Extract tasks/blockers via copilot planner
+  console.log('── Step 3: Extracting tasks & blockers via planner ──');
+
+  const copilotResolved = getCopilotCmd();
+  console.log(`  ℹ Copilot resolved to: ${copilotResolved}`);
+
+  // Verify it works
+  const { cmd: testCmd, args: testArgsArr } = copilotSpawnArgs(copilotResolved, ['--version']);
+  const testResult = await run(testCmd, testArgsArr, workspaceDir);
+  if (testResult.code === 0) {
+    console.log(`  ✓ Found: ${(testResult.stdout || '').trim().split(/\r?\n/)[0]}`);
+  } else {
+    console.log('  ❌ Could not find copilot CLI.');
+    console.log(`     stderr: ${(testResult.stderr || '').slice(0, 200)}`);
+    console.log('     Skipping task/blocker extraction.');
+    dl.db.save();
+    console.log('\n✅ Retroactive ingestion complete (embeddings only)!');
+    return;
+  }
+
+  const agentEnv = { FREYA_WORKSPACE_DIR: workspaceDir };
+  const slugMap = readProjectSlugMap(workspaceDir);
+  const validTaskCats = new Set(['DO_NOW', 'SCHEDULE', 'DELEGATE', 'IGNORE']);
+
+  const schema = {
+    actions: [
+      { type: 'create_task', description: '<string>', priority: 'HIGH|MEDIUM|LOW', category: 'DO_NOW|SCHEDULE|DELEGATE|IGNORE', projectSlug: '<string optional>' },
+      { type: 'create_blocker', title: '<string>', severity: 'CRITICAL|HIGH|MEDIUM|LOW', notes: '<string>', projectSlug: '<string optional>' }
+    ]
+  };
+
+  const sysInstructions = `Você é o planner do sistema F.R.E.Y.A.
+
+Analise o daily log abaixo e extraia TODAS as tarefas e blockers mencionados.
+Procure por: ações mencionadas, pendências, problemas, impedimentos, decisões que geram trabalho.
+Se NÃO houver tarefas ou blockers claros, retorne: {"actions":[]}
+Retorne APENAS JSON válido no formato: ${JSON.stringify(schema)}
+NÃO use code fences. NÃO inclua texto extra.
+IMPORTANTE: Extraia APENAS informações explícitas do log. NÃO invente dados.`;
+
+  let totalTasks = 0, totalBlockers = 0, totalSkipped = 0, totalErrors = 0;
+
+  const insertTask = dl.db.prepare(`INSERT OR IGNORE INTO tasks (id, project_slug, description, category, status, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+  const insertBlocker = dl.db.prepare(`INSERT OR IGNORE INTO blockers (id, project_slug, title, severity, status, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+
+  const existingTaskDescs = new Set(
+    dl.db.prepare("SELECT description FROM tasks").all().map(t => sha1(normalizeTextForKey(t.description)))
+  );
+  const existingBlockerTitles = new Set(
+    dl.db.prepare("SELECT title FROM blockers").all().map(b => sha1(normalizeTextForKey(b.title)))
+  );
+
+  const SAFE_ARG_LEN = 24000;
+
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    const date = file.replace('.md', '');
+    const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
+
+    if (content.trim().length < 50) {
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — skipped (too small)`);
+      continue;
+    }
+
+    const fullPrompt = `${sysInstructions}\n\nDAILY LOG (${date}):\n${content}\n`;
+
+    try {
+      let r;
+      const copilotExtra = ['-s', '--no-color', '--stream', 'off'];
+
+      if (fullPrompt.length > SAFE_ARG_LEN) {
+        const tmpFile = path.join(os.tmpdir(), `freya-retro-${Date.now()}.txt`);
+        fs.writeFileSync(tmpFile, fullPrompt, 'utf8');
+        const filePrompt = `Leia o arquivo abaixo e extraia tasks/blockers conforme as instruções contidas nele.\nARQUIVO: ${tmpFile}`;
+        const { cmd: sc, args: sa } = copilotSpawnArgs(copilotResolved, [...copilotExtra, '--add-dir', os.tmpdir(), '--allow-all-tools', '-p', filePrompt]);
+        r = await run(sc, sa, workspaceDir, agentEnv);
+        try { fs.unlinkSync(tmpFile); } catch { }
+      } else {
+        const { cmd: sc, args: sa } = copilotSpawnArgs(copilotResolved, [...copilotExtra, '-p', fullPrompt]);
+        r = await run(sc, sa, workspaceDir, agentEnv);
+      }
+
+      const out = (r.stdout + r.stderr).trim();
+      if (r.code !== 0 || !out) {
+        totalErrors++;
+        if (totalErrors === 1) {
+          console.log(`\n  ⚠ Planner diagnostic for ${date}:`);
+          console.log(`    Copilot: ${copilotResolved}`);
+          console.log(`    Exit code: ${r.code}`);
+          console.log(`    stdout: ${(r.stdout || '').slice(0, 300)}`);
+          console.log(`    stderr: ${(r.stderr || '').slice(0, 300)}`);
+        }
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ planner error (code=${r.code})     `);
+        continue;
+      }
+
+      // Parse plan
+      const jsonText = extractFirstJsonObject(out) || out;
+      let plan;
+      try {
+        plan = JSON.parse(jsonText);
+      } catch {
+        try { plan = JSON.parse(escapeJsonControlChars(jsonText)); } catch {
+          totalErrors++;
+          process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ invalid JSON     `);
+          continue;
+        }
+      }
+
+      const actions = Array.isArray(plan.actions) ? plan.actions : [];
+      let fileTasks = 0, fileBlockers = 0, fileSkipped = 0;
+
+      if (!DRY_RUN) {
+        const applyTx = dl.db.transaction(() => {
+          for (const a of actions) {
+            if (!a || typeof a !== 'object') continue;
+
+            if (a.type === 'create_task' && a.description) {
+              const desc = normalizeWhitespace(a.description);
+              if (!desc) continue;
+              const projectSlug = String(a.projectSlug || '').trim() || inferProjectSlug(desc, slugMap);
+              const key = sha1(normalizeTextForKey((projectSlug ? projectSlug + ' ' : '') + desc));
+              if (existingTaskDescs.has(key)) { fileSkipped++; continue; }
+              const id = `t-retro-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+              const category = validTaskCats.has(String(a.category || '').trim()) ? String(a.category).trim() : 'DO_NOW';
+              const metadata = JSON.stringify({ priority: a.priority || 'medium', source: 'retroactive', sourceDate: date });
+              insertTask.run(id, projectSlug || null, desc, category, 'PENDING', `${date}T12:00:00.000Z`, metadata);
+              existingTaskDescs.add(key);
+              fileTasks++;
+            }
+
+            if (a.type === 'create_blocker' && a.title) {
+              const title = normalizeWhitespace(a.title);
+              if (!title) continue;
+              const notes = normalizeWhitespace(a.notes);
+              const projectSlug = String(a.projectSlug || '').trim() || inferProjectSlug(title + ' ' + notes, slugMap);
+              const key = sha1(normalizeTextForKey((projectSlug ? projectSlug + ' ' : '') + title));
+              if (existingBlockerTitles.has(key)) { fileSkipped++; continue; }
+              const id = `b-retro-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+              const severity = String(a.severity || 'MEDIUM').toUpperCase();
+              const metadata = JSON.stringify({ description: notes || title, source: 'retroactive', sourceDate: date });
+              insertBlocker.run(id, projectSlug || null, title, severity, 'OPEN', `${date}T12:00:00.000Z`, metadata);
+              existingBlockerTitles.add(key);
+              fileBlockers++;
+            }
+          }
+        });
+        applyTx();
+      } else {
+        for (const a of actions) {
+          if (a && a.type === 'create_task' && a.description) fileTasks++;
+          if (a && a.type === 'create_blocker' && a.title) fileBlockers++;
+        }
+      }
+
+      totalTasks += fileTasks;
+      totalBlockers += fileBlockers;
+      totalSkipped += fileSkipped;
+
+      const status = fileTasks || fileBlockers
+        ? `${fileTasks}T ${fileBlockers}B${fileSkipped ? ` (${fileSkipped} dup)` : ''}`
+        : 'no actions';
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ${status}              `);
+
+      if (i < files.length - 1) {
+        await new Promise(r => setTimeout(r, 500));
+      }
+
+    } catch (err) {
+      totalErrors++;
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ ${err.message}     `);
+    }
+  }
+
+  dl.db.save();
+
+  console.log('\n');
+  console.log('══════════════════════════════════════════════════════');
+  console.log(`  📊 Results ${DRY_RUN ? '(DRY RUN)' : ''}`);
+  console.log(`     Tasks created:    ${totalTasks}`);
+  console.log(`     Blockers created: ${totalBlockers}`);
+  console.log(`     Duplicates skipped: ${totalSkipped}`);
+  console.log(`     Errors:           ${totalErrors}`);
+  console.log('══════════════════════════════════════════════════════');
+
+  if (!DRY_RUN && !SKIP_EMBEDDINGS && (totalTasks > 0 || totalBlockers > 0)) {
+    console.log('\n── Step 4: Generating embeddings for new tasks/blockers ──');
+    const dm = new DataManager(workspaceDir, logsDir);
+    const newTasks = dl.db.prepare("SELECT id, description FROM tasks WHERE json_extract(metadata, '$.source') = 'retroactive'").all();
+    const newBlockers = dl.db.prepare("SELECT id, title, json_extract(metadata, '$.description') as notes FROM blockers WHERE json_extract(metadata, '$.source') = 'retroactive'").all();
+    let embCount = 0;
+    for (const t of newTasks) {
+      try { embCount += await dm.generateEmbeddings('task', t.id, t.description); } catch { }
+    }
+    for (const b of newBlockers) {
+      try { embCount += await dm.generateEmbeddings('blocker', b.id, b.title + ' ' + (b.notes || '')); } catch { }
+    }
+    console.log(`✅ Generated ${embCount} embedding chunks for new entities`);
+  }
+
+  console.log('\n✅ Retroactive ingestion complete!');
+}
+
+module.exports = retroactiveIngest;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cccarv82/freya",
-  "version": "3.7.7",
+  "version": "3.8.0",
   "description": "Personal AI Assistant with local-first persistence",
   "scripts": {
     "health": "node scripts/validate-data.js && node scripts/validate-structure.js",