@cccarv82/freya 3.6.0 → 3.6.1

package/cli/init.js

@@ -88,7 +88,8 @@ function ensurePackageJson(targetDir, force, summary) {
 'sm-weekly': 'node scripts/generate-sm-weekly-report.js',
     daily: 'node scripts/generate-daily-summary.js',
     status: 'node scripts/generate-executive-report.js',
-    blockers: 'node scripts/generate-blockers-report.js'
+    blockers: 'node scripts/generate-blockers-report.js',
+    'retroactive-ingest': 'node scripts/retroactive-ingest.js'
   };
 
   const depsToEnsure = {

package/cli/web.js

@@ -235,31 +235,27 @@ async function buildSmartContext(workspaceDir, query) {
     console.error('[context] RAG search failed:', ragErr.message);
   }
 
-  // 2. Fallback: if RAG not available or returned few results, include recent daily logs
+  // 2. Fallback: if RAG not available or returned few results, include recent daily logs from SQLite
   if (!ragUsed || usedBudget < TOKEN_BUDGET / 3) {
     try {
-      const logsDir = path.join(workspaceDir, 'logs', 'daily');
-      if (exists(logsDir)) {
-        const maxDays = ragUsed ? 3 : 5; // fewer if RAG already provided some context
-        const files = fs.readdirSync(logsDir)
-          .filter(f => /^\d{4}-\d{2}-\d{2}\.md$/.test(f))
-          .sort()
-          .slice(-maxDays);
-        if (files.length) {
-          parts.push('\n[DAILY LOGS — ÚLTIMOS ' + files.length + ' DIAS]');
-          for (const file of files) {
-            const date = file.replace('.md', '');
-            const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
-            const maxPerLog = Math.floor((TOKEN_BUDGET - usedBudget) / files.length);
-            const trimmed = content.length > maxPerLog ? content.slice(0, maxPerLog) + '\n...(truncado)' : content;
-            parts.push(`\n--- LOG ${date} ---\n${trimmed}`);
-            usedBudget += trimmed.length;
-            if (usedBudget >= TOKEN_BUDGET) break;
-          }
+      const maxDays = ragUsed ? 3 : 5;
+      const recentLogs = dl.db.prepare(
+        `SELECT date, raw_markdown FROM daily_logs ORDER BY date DESC LIMIT ?`
+      ).all(maxDays);
+      if (recentLogs.length) {
+        parts.push('\n[DAILY LOGS — ÚLTIMOS ' + recentLogs.length + ' DIAS]');
+        // Reverse to show chronologically (oldest first)
+        for (const log of recentLogs.reverse()) {
+          const maxPerLog = Math.floor((TOKEN_BUDGET - usedBudget) / recentLogs.length);
+          const content = log.raw_markdown || '';
+          const trimmed = content.length > maxPerLog ? content.slice(0, maxPerLog) + '\n...(truncado)' : content;
+          parts.push(`\n--- LOG ${log.date} ---\n${trimmed}`);
+          usedBudget += trimmed.length;
+          if (usedBudget >= TOKEN_BUDGET) break;
         }
       }
     } catch (e) {
-      console.error('[context] Failed to read daily logs:', e.message);
+      console.error('[context] Failed to read daily logs from SQLite:', e.message);
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cccarv82/freya",
-  "version": "3.6.0",
+  "version": "3.6.1",
   "description": "Personal AI Assistant with local-first persistence",
   "scripts": {
     "health": "node scripts/validate-data.js && node scripts/validate-structure.js",
@@ -10,6 +10,7 @@
     "status": "node scripts/generate-executive-report.js",
     "blockers": "node scripts/generate-blockers-report.js",
     "export-obsidian": "node scripts/export-obsidian.js",
+    "retroactive-ingest": "node scripts/retroactive-ingest.js",
     "build-index": "node scripts/index/build-index.js",
     "update-index": "node scripts/index/update-index.js",
     "test": "node tests/unit/test-package-config.js && node tests/unit/test-cli-init.js && node tests/unit/test-cli-web-help.js && node tests/unit/test-web-static-assets.js && node tests/unit/test-fs-utils.js && node tests/unit/test-search-utils.js && node tests/unit/test-index-utils.js && node tests/unit/test-daily-generation.js && node tests/unit/test-report-generation.js && node tests/unit/test-executive-report-logs.js && node tests/unit/test-oracle-retrieval.js && node tests/unit/test-task-completion.js && node tests/unit/test-migrate-data.js && node tests/unit/test-blockers-report.js && node tests/unit/test-sm-weekly-report.js && node tests/integration/test-ingestor-task.js && node tests/unit/test-structure-validation.js"

package/scripts/retroactive-ingest.js

@@ -0,0 +1,386 @@
+#!/usr/bin/env node
+/**
+ * retroactive-ingest.js
+ *
+ * Reads ALL existing daily logs, sends each through the Copilot CLI planner
+ * to extract tasks/blockers, and applies them to SQLite.
+ * Also generates embeddings for all daily logs.
+ *
+ * Usage:
+ *   node scripts/retroactive-ingest.js [--dry-run] [--days N] [--embeddings-only]
+ *
+ * Options:
+ *   --dry-run          Show what would be extracted without writing to SQLite
+ *   --days N           Only process the last N days (default: all)
+ *   --embeddings-only  Skip planner, only generate embeddings for existing logs
+ *   --skip-embeddings  Skip embedding generation (only extract tasks/blockers)
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { spawn } = require('child_process');
+const os = require('os');
+
+// Parse args
+const args = process.argv.slice(2);
+const DRY_RUN = args.includes('--dry-run');
+const EMBEDDINGS_ONLY = args.includes('--embeddings-only');
+const SKIP_EMBEDDINGS = args.includes('--skip-embeddings');
+const daysIdx = args.indexOf('--days');
+const MAX_DAYS = daysIdx >= 0 ? parseInt(args[daysIdx + 1], 10) : 0;
+
+// Resolve workspace directory
+const workspaceDir = process.env.FREYA_WORKSPACE_DIR
+  ? path.resolve(process.env.FREYA_WORKSPACE_DIR)
+  : path.join(__dirname, '..');
+
+const { defaultInstance: dl, ready } = require('./lib/DataLayer');
+const DataManager = require('./lib/DataManager');
+
+function sha1(text) {
+  return crypto.createHash('sha1').update(text).digest('hex');
+}
+
+function normalizeWhitespace(t) {
+  return String(t || '').replace(/\s+/g, ' ').trim();
+}
+
+function normalizeTextForKey(t) {
+  return normalizeWhitespace(t).toLowerCase();
+}
+
+function run(cmd, args, cwd, extraEnv) {
+  return new Promise((resolve) => {
+    let child;
+    const env = extraEnv ? { ...process.env, ...extraEnv } : process.env;
+    try {
+      if (process.platform === 'win32') {
+        const comspec = process.env.ComSpec || 'cmd.exe';
+        child = spawn(comspec, ['/d', '/s', '/c', cmd, ...args], { cwd, shell: false, env });
+      } else {
+        child = spawn(cmd, args, { cwd, shell: false, env });
+      }
+    } catch (e) {
+      return resolve({ code: 1, stdout: '', stderr: e.message || String(e) });
+    }
+    let stdout = '';
+    let stderr = '';
+    child.stdout && child.stdout.on('data', (d) => { stdout += d.toString(); });
+    child.stderr && child.stderr.on('data', (d) => { stderr += d.toString(); });
+    child.on('error', (e) => { stderr += `\n${e.message}`; resolve({ code: 1, stdout, stderr }); });
+    child.on('close', (code) => resolve({ code: code ?? 0, stdout, stderr }));
+  });
+}
+
+function extractFirstJsonObject(text) {
+  if (!text) return null;
+  const start = text.indexOf('{');
+  if (start === -1) return null;
+  let depth = 0;
+  for (let i = start; i < text.length; i++) {
+    if (text[i] === '{') depth++;
+    else if (text[i] === '}') { depth--; if (depth === 0) return text.slice(start, i + 1); }
+  }
+  return null;
+}
+
+function escapeJsonControlChars(jsonText) {
+  return jsonText.replace(/[\x00-\x1F\x7F]/g, (ch) => {
+    if (ch === '\n' || ch === '\r' || ch === '\t') return ch;
+    return '\\u' + ch.charCodeAt(0).toString(16).padStart(4, '0');
+  });
+}
+
+function readProjectSlugMap(wsDir) {
+  const p = path.join(wsDir, 'data', 'settings', 'project-slug-map.json');
+  try {
+    return JSON.parse(fs.readFileSync(p, 'utf8'));
+  } catch { return {}; }
+}
+
+function inferProjectSlug(text, map) {
+  if (!text || !map || typeof map !== 'object') return '';
+  const lower = text.toLowerCase();
+  let bestSlug = '';
+  let bestLen = 0;
+  for (const [keyword, slug] of Object.entries(map)) {
+    if (lower.includes(keyword.toLowerCase()) && keyword.length > bestLen) {
+      bestSlug = slug;
+      bestLen = keyword.length;
+    }
+  }
+  return bestSlug;
+}
+
+async function main() {
+  await ready;
+
+  console.log('╔══════════════════════════════════════════════════════╗');
+  console.log('║  FREYA — Retroactive Ingestion                     ║');
+  console.log('╚══════════════════════════════════════════════════════╝');
+  console.log(`Workspace: ${workspaceDir}`);
+  console.log(`Mode: ${DRY_RUN ? 'DRY RUN' : EMBEDDINGS_ONLY ? 'EMBEDDINGS ONLY' : 'FULL INGESTION'}`);
+  console.log('');
+
+  const logsDir = path.join(workspaceDir, 'logs', 'daily');
+  if (!fs.existsSync(logsDir)) {
+    console.log('❌ No daily logs directory found at:', logsDir);
+    process.exit(1);
+  }
+
+  let files = fs.readdirSync(logsDir)
+    .filter(f => /^\d{4}-\d{2}-\d{2}\.md$/.test(f))
+    .sort();
+
+  if (MAX_DAYS > 0) {
+    files = files.slice(-MAX_DAYS);
+  }
+
+  console.log(`📁 Found ${files.length} daily log files to process`);
+  console.log('');
+
+  // Step 1: Sync all daily logs to SQLite
+  console.log('── Step 1: Syncing daily logs to SQLite ──');
+  const upsert = dl.db.prepare(`
+    INSERT INTO daily_logs (date, raw_markdown) VALUES (?, ?)
+    ON CONFLICT(date) DO UPDATE SET raw_markdown = excluded.raw_markdown
+  `);
+  const syncTx = dl.db.transaction((fileList) => {
+    for (const file of fileList) {
+      const date = file.replace('.md', '');
+      const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
+      upsert.run(date, content);
+    }
+  });
+  syncTx(files);
+  console.log(`✅ ${files.length} daily logs synced to SQLite`);
+  console.log('');
+
+  // Step 2: Generate embeddings for all daily logs
+  if (!SKIP_EMBEDDINGS) {
+    console.log('── Step 2: Generating embeddings ──');
+    const dm = new DataManager(workspaceDir, logsDir);
+    let totalChunks = 0;
+    for (let i = 0; i < files.length; i++) {
+      const date = files[i].replace('.md', '');
+      const content = fs.readFileSync(path.join(logsDir, files[i]), 'utf8');
+      try {
+        const count = await dm.generateEmbeddings('daily_log', date, content);
+        totalChunks += count;
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ${count} chunks`);
+      } catch (err) {
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ ${err.message}`);
+      }
+    }
+    console.log(`\n✅ Generated ${totalChunks} embedding chunks total`);
+    console.log('');
+  }
+
+  if (EMBEDDINGS_ONLY) {
+    console.log('── Embeddings-only mode. Skipping task/blocker extraction. ──');
+    dl.db.save();
+    console.log('\n✅ Done!');
+    return;
+  }
+
+  // Step 3: Extract tasks/blockers from each daily log via planner
+  console.log('── Step 3: Extracting tasks & blockers via planner ──');
+
+  const cmd = process.env.COPILOT_CMD || 'copilot';
+  const agentEnv = { FREYA_WORKSPACE_DIR: workspaceDir };
+  const slugMap = readProjectSlugMap(workspaceDir);
+  const validTaskCats = new Set(['DO_NOW', 'SCHEDULE', 'DELEGATE', 'IGNORE']);
+
+  const schema = {
+    actions: [
+      { type: 'create_task', description: '<string>', priority: 'HIGH|MEDIUM|LOW', category: 'DO_NOW|SCHEDULE|DELEGATE|IGNORE', projectSlug: '<string optional>' },
+      { type: 'create_blocker', title: '<string>', severity: 'CRITICAL|HIGH|MEDIUM|LOW', notes: '<string>', projectSlug: '<string optional>' }
+    ]
+  };
+
+  const sysInstructions = `Você é o planner do sistema F.R.E.Y.A.
+
+Analise o daily log abaixo e extraia TODAS as tarefas e blockers mencionados.
+Procure por: ações mencionadas, pendências, problemas, impedimentos, decisões que geram trabalho.
+Se NÃO houver tarefas ou blockers claros, retorne: {"actions":[]}
+Retorne APENAS JSON válido no formato: ${JSON.stringify(schema)}
+NÃO use code fences. NÃO inclua texto extra.
+IMPORTANTE: Extraia APENAS informações explícitas do log. NÃO invente dados.`;
+
+  let totalTasks = 0;
+  let totalBlockers = 0;
+  let totalSkipped = 0;
+  let totalErrors = 0;
+
+  const insertTask = dl.db.prepare(`INSERT OR IGNORE INTO tasks (id, project_slug, description, category, status, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+  const insertBlocker = dl.db.prepare(`INSERT OR IGNORE INTO blockers (id, project_slug, title, severity, status, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+
+  // Build existing keys for dedup
+  const existingTaskDescs = new Set(
+    dl.db.prepare("SELECT description FROM tasks").all().map(t => sha1(normalizeTextForKey(t.description)))
+  );
+  const existingBlockerTitles = new Set(
+    dl.db.prepare("SELECT title FROM blockers").all().map(b => sha1(normalizeTextForKey(b.title)))
+  );
+
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    const date = file.replace('.md', '');
+    const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
+
+    // Skip very small logs (< 50 chars) — likely empty or just a header
+    if (content.trim().length < 50) {
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — skipped (too small)`);
+      continue;
+    }
+
+    const fullPrompt = `${sysInstructions}\n\nDAILY LOG (${date}):\n${content}\n`;
+    const SAFE_ARG_LEN = 24000;
+
+    try {
+      let r;
+      if (fullPrompt.length > SAFE_ARG_LEN) {
+        const tmpFile = path.join(os.tmpdir(), `freya-retro-${Date.now()}.txt`);
+        fs.writeFileSync(tmpFile, fullPrompt, 'utf8');
+        const filePrompt = `Leia o arquivo abaixo e extraia tasks/blockers conforme as instruções contidas nele.\nARQUIVO: ${tmpFile}`;
+        r = await run(cmd, ['-s', '--no-color', '--stream', 'off', '--add-dir', os.tmpdir(), '--allow-all-tools', '-p', filePrompt], workspaceDir, agentEnv);
+        try { fs.unlinkSync(tmpFile); } catch { }
+      } else {
+        r = await run(cmd, ['-s', '--no-color', '--stream', 'off', '-p', fullPrompt], workspaceDir, agentEnv);
+      }
+
+      const out = (r.stdout + r.stderr).trim();
+      if (r.code !== 0 || !out) {
+        totalErrors++;
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ planner error     `);
+        continue;
+      }
+
+      // Parse plan
+      const jsonText = extractFirstJsonObject(out) || out;
+      let plan;
+      try {
+        plan = JSON.parse(jsonText);
+      } catch {
+        try { plan = JSON.parse(escapeJsonControlChars(jsonText)); } catch {
+          totalErrors++;
+          process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ invalid JSON     `);
+          continue;
+        }
+      }
+
+      const actions = Array.isArray(plan.actions) ? plan.actions : [];
+      let fileTasks = 0;
+      let fileBlockers = 0;
+      let fileSkipped = 0;
+
+      if (!DRY_RUN) {
+        const applyTx = dl.db.transaction(() => {
+          for (const a of actions) {
+            if (!a || typeof a !== 'object') continue;
+
+            if (a.type === 'create_task' && a.description) {
+              const desc = normalizeWhitespace(a.description);
+              if (!desc) continue;
+              const projectSlug = String(a.projectSlug || '').trim() || inferProjectSlug(desc, slugMap);
+              const key = sha1(normalizeTextForKey((projectSlug ? projectSlug + ' ' : '') + desc));
+              if (existingTaskDescs.has(key)) { fileSkipped++; continue; }
+
+              const id = `t-retro-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+              const category = validTaskCats.has(String(a.category || '').trim()) ? String(a.category).trim() : 'DO_NOW';
+              const metadata = JSON.stringify({ priority: a.priority || 'medium', source: 'retroactive', sourceDate: date });
+              // Use the log date as created_at for chronological accuracy
+              insertTask.run(id, projectSlug || null, desc, category, 'PENDING', `${date}T12:00:00.000Z`, metadata);
+              existingTaskDescs.add(key);
+              fileTasks++;
+            }
+
+            if (a.type === 'create_blocker' && a.title) {
+              const title = normalizeWhitespace(a.title);
+              if (!title) continue;
+              const notes = normalizeWhitespace(a.notes);
+              const projectSlug = String(a.projectSlug || '').trim() || inferProjectSlug(title + ' ' + notes, slugMap);
+              const key = sha1(normalizeTextForKey((projectSlug ? projectSlug + ' ' : '') + title));
+              if (existingBlockerTitles.has(key)) { fileSkipped++; continue; }
+
+              const id = `b-retro-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+              const severity = String(a.severity || 'MEDIUM').toUpperCase();
+              const metadata = JSON.stringify({ description: notes || title, source: 'retroactive', sourceDate: date });
+              insertBlocker.run(id, projectSlug || null, title, severity, 'OPEN', `${date}T12:00:00.000Z`, metadata);
+              existingBlockerTitles.add(key);
+              fileBlockers++;
+            }
+          }
+        });
+        applyTx();
+      } else {
+        // Dry run — just count
+        for (const a of actions) {
+          if (a && a.type === 'create_task' && a.description) fileTasks++;
+          if (a && a.type === 'create_blocker' && a.title) fileBlockers++;
+        }
+      }
+
+      totalTasks += fileTasks;
+      totalBlockers += fileBlockers;
+      totalSkipped += fileSkipped;
+
+      const status = fileTasks || fileBlockers
+        ? `${fileTasks}T ${fileBlockers}B${fileSkipped ? ` (${fileSkipped} dup)` : ''}`
+        : 'no actions';
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ${status}              `);
+
+      // Small delay to avoid rate limiting
+      if (i < files.length - 1) {
+        await new Promise(r => setTimeout(r, 500));
+      }
+
+    } catch (err) {
+      totalErrors++;
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ ${err.message}     `);
+    }
+  }
+
+  // Ensure data is persisted
+  dl.db.save();
+
+  console.log('\n');
+  console.log('══════════════════════════════════════════════════════');
+  console.log(`  📊 Results ${DRY_RUN ? '(DRY RUN)' : ''}`);
+  console.log(`     Tasks created:    ${totalTasks}`);
+  console.log(`     Blockers created: ${totalBlockers}`);
+  console.log(`     Duplicates skipped: ${totalSkipped}`);
+  console.log(`     Errors:           ${totalErrors}`);
+  console.log('══════════════════════════════════════════════════════');
+
+  // Step 4: Generate embeddings for newly created tasks/blockers
+  if (!DRY_RUN && !SKIP_EMBEDDINGS && (totalTasks > 0 || totalBlockers > 0)) {
+    console.log('\n── Step 4: Generating embeddings for new tasks/blockers ──');
+    const dm = new DataManager(workspaceDir, logsDir);
+    const newTasks = dl.db.prepare("SELECT id, description FROM tasks WHERE json_extract(metadata, '$.source') = 'retroactive'").all();
+    const newBlockers = dl.db.prepare("SELECT id, title, json_extract(metadata, '$.description') as notes FROM blockers WHERE json_extract(metadata, '$.source') = 'retroactive'").all();
+
+    let embCount = 0;
+    for (const t of newTasks) {
+      try {
+        embCount += await dm.generateEmbeddings('task', t.id, t.description);
+      } catch { }
+    }
+    for (const b of newBlockers) {
+      try {
+        embCount += await dm.generateEmbeddings('blocker', b.id, b.title + ' ' + (b.notes || ''));
+      } catch { }
+    }
+    console.log(`✅ Generated ${embCount} embedding chunks for new entities`);
+  }
+
+  console.log('\n✅ Retroactive ingestion complete!');
+}
+
+main().catch(err => {
+  console.error('\n❌ Fatal error:', err.message || err);
+  process.exit(1);
+});

package/templates/base/scripts/lib/DataLayer.js

@@ -325,6 +325,12 @@ class DataLayer {
         embedding BLOB NOT NULL, /* Stored as Buffer of Float32Array */
         created_at DATETIME DEFAULT CURRENT_TIMESTAMP
       );
+
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_doc_emb_ref
+        ON document_embeddings(reference_type, reference_id, chunk_index);
+
+      CREATE INDEX IF NOT EXISTS idx_doc_emb_type
+        ON document_embeddings(reference_type);
     `);
 
     // --- Migrations for existing databases ---

package/templates/base/scripts/lib/DataManager.js

@@ -229,6 +229,95 @@ class DataManager {
         return NaN;
     }
 
+    // --- Embedding Generation ---
+
+    /**
+     * Split text into chunks suitable for embedding (~400-600 chars each).
+     * Splits on markdown headings, then paragraphs, then sentences.
+     */
+    chunkText(text, maxChunkSize = 500) {
+        if (!text || text.length <= maxChunkSize) return [text].filter(Boolean);
+
+        const chunks = [];
+        // First split on markdown ## headings
+        const sections = text.split(/(?=^## )/m).filter(s => s.trim());
+
+        for (const section of sections) {
+            if (section.length <= maxChunkSize) {
+                chunks.push(section.trim());
+                continue;
+            }
+            // Split long sections on double newlines (paragraphs)
+            const paragraphs = section.split(/\n\n+/).filter(p => p.trim());
+            let buffer = '';
+            for (const para of paragraphs) {
+                if (buffer.length + para.length + 2 > maxChunkSize && buffer) {
+                    chunks.push(buffer.trim());
+                    buffer = '';
+                }
+                buffer += (buffer ? '\n\n' : '') + para;
+            }
+            if (buffer.trim()) chunks.push(buffer.trim());
+        }
+
+        return chunks.filter(c => c.length > 10); // skip tiny fragments
+    }
+
+    /**
+     * Generate embeddings for a piece of content and store in document_embeddings.
+     * Deletes existing embeddings for (referenceType, referenceId) first to avoid stale data.
+     * @param {string} referenceType - 'daily_log', 'task', or 'blocker'
+     * @param {string} referenceId - unique ID (date for logs, task/blocker id)
+     * @param {string} text - content to embed
+     */
+    async generateEmbeddings(referenceType, referenceId, text) {
+        if (!text || !text.trim()) return 0;
+
+        const chunks = this.chunkText(text);
+        if (!chunks.length) return 0;
+
+        // Delete existing embeddings for this reference
+        dl.db.prepare('DELETE FROM document_embeddings WHERE reference_type = ? AND reference_id = ?')
+            .run(referenceType, referenceId);
+
+        const insert = dl.db.prepare(`
+            INSERT INTO document_embeddings (reference_type, reference_id, chunk_index, text_chunk, embedding)
+            VALUES (?, ?, ?, ?, ?)
+        `);
+
+        let count = 0;
+        for (let i = 0; i < chunks.length; i++) {
+            try {
+                const vector = await defaultEmbedder.embedText(chunks[i]);
+                const buffer = defaultEmbedder.vectorToBuffer(vector);
+                insert.run(referenceType, referenceId, i, chunks[i], buffer);
+                count++;
+            } catch (err) {
+                console.error(`[embeddings] Failed to embed chunk ${i} of ${referenceType}/${referenceId}:`, err.message);
+            }
+        }
+        return count;
+    }
+
+    /**
+     * Check if embeddings exist and are up-to-date for a reference.
+     * @returns {boolean} true if embeddings exist
+     */
+    hasEmbeddings(referenceType, referenceId) {
+        const row = dl.db.prepare(
+            'SELECT COUNT(*) as c FROM document_embeddings WHERE reference_type = ? AND reference_id = ?'
+        ).get(referenceType, referenceId);
+        return row && row.c > 0;
+    }
+
+    /**
+     * Get total embedding count (for checking if RAG is available).
+     */
+    getEmbeddingCount() {
+        const row = dl.db.prepare('SELECT COUNT(*) as c FROM document_embeddings').get();
+        return row ? row.c : 0;
+    }
+
     // --- RAG (Vector Search) ---
     async semanticSearch(query, topK = 10) {
         const queryVector = await defaultEmbedder.embedText(query);

package/templates/base/scripts/retroactive-ingest.js

@@ -0,0 +1,386 @@
+#!/usr/bin/env node
+/**
+ * retroactive-ingest.js
+ *
+ * Reads ALL existing daily logs, sends each through the Copilot CLI planner
+ * to extract tasks/blockers, and applies them to SQLite.
+ * Also generates embeddings for all daily logs.
+ *
+ * Usage:
+ *   node scripts/retroactive-ingest.js [--dry-run] [--days N] [--embeddings-only]
+ *
+ * Options:
+ *   --dry-run          Show what would be extracted without writing to SQLite
+ *   --days N           Only process the last N days (default: all)
+ *   --embeddings-only  Skip planner, only generate embeddings for existing logs
+ *   --skip-embeddings  Skip embedding generation (only extract tasks/blockers)
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { spawn } = require('child_process');
+const os = require('os');
+
+// Parse args
+const args = process.argv.slice(2);
+const DRY_RUN = args.includes('--dry-run');
+const EMBEDDINGS_ONLY = args.includes('--embeddings-only');
+const SKIP_EMBEDDINGS = args.includes('--skip-embeddings');
+const daysIdx = args.indexOf('--days');
+const MAX_DAYS = daysIdx >= 0 ? parseInt(args[daysIdx + 1], 10) : 0;
+
+// Resolve workspace directory
+const workspaceDir = process.env.FREYA_WORKSPACE_DIR
+  ? path.resolve(process.env.FREYA_WORKSPACE_DIR)
+  : path.join(__dirname, '..');
+
+const { defaultInstance: dl, ready } = require('./lib/DataLayer');
+const DataManager = require('./lib/DataManager');
+
+function sha1(text) {
+  return crypto.createHash('sha1').update(text).digest('hex');
+}
+
+function normalizeWhitespace(t) {
+  return String(t || '').replace(/\s+/g, ' ').trim();
+}
+
+function normalizeTextForKey(t) {
+  return normalizeWhitespace(t).toLowerCase();
+}
+
+function run(cmd, args, cwd, extraEnv) {
+  return new Promise((resolve) => {
+    let child;
+    const env = extraEnv ? { ...process.env, ...extraEnv } : process.env;
+    try {
+      if (process.platform === 'win32') {
+        const comspec = process.env.ComSpec || 'cmd.exe';
+        child = spawn(comspec, ['/d', '/s', '/c', cmd, ...args], { cwd, shell: false, env });
+      } else {
+        child = spawn(cmd, args, { cwd, shell: false, env });
+      }
+    } catch (e) {
+      return resolve({ code: 1, stdout: '', stderr: e.message || String(e) });
+    }
+    let stdout = '';
+    let stderr = '';
+    child.stdout && child.stdout.on('data', (d) => { stdout += d.toString(); });
+    child.stderr && child.stderr.on('data', (d) => { stderr += d.toString(); });
+    child.on('error', (e) => { stderr += `\n${e.message}`; resolve({ code: 1, stdout, stderr }); });
+    child.on('close', (code) => resolve({ code: code ?? 0, stdout, stderr }));
+  });
+}
+
+function extractFirstJsonObject(text) {
+  if (!text) return null;
+  const start = text.indexOf('{');
+  if (start === -1) return null;
+  let depth = 0;
+  for (let i = start; i < text.length; i++) {
+    if (text[i] === '{') depth++;
+    else if (text[i] === '}') { depth--; if (depth === 0) return text.slice(start, i + 1); }
+  }
+  return null;
+}
+
+function escapeJsonControlChars(jsonText) {
+  return jsonText.replace(/[\x00-\x1F\x7F]/g, (ch) => {
+    if (ch === '\n' || ch === '\r' || ch === '\t') return ch;
+    return '\\u' + ch.charCodeAt(0).toString(16).padStart(4, '0');
+  });
+}
+
+function readProjectSlugMap(wsDir) {
+  const p = path.join(wsDir, 'data', 'settings', 'project-slug-map.json');
+  try {
+    return JSON.parse(fs.readFileSync(p, 'utf8'));
+  } catch { return {}; }
+}
+
+function inferProjectSlug(text, map) {
+  if (!text || !map || typeof map !== 'object') return '';
+  const lower = text.toLowerCase();
+  let bestSlug = '';
+  let bestLen = 0;
+  for (const [keyword, slug] of Object.entries(map)) {
+    if (lower.includes(keyword.toLowerCase()) && keyword.length > bestLen) {
+      bestSlug = slug;
+      bestLen = keyword.length;
+    }
+  }
+  return bestSlug;
+}
+
+async function main() {
+  await ready;
+
+  console.log('╔══════════════════════════════════════════════════════╗');
+  console.log('║  FREYA — Retroactive Ingestion                     ║');
+  console.log('╚══════════════════════════════════════════════════════╝');
+  console.log(`Workspace: ${workspaceDir}`);
+  console.log(`Mode: ${DRY_RUN ? 'DRY RUN' : EMBEDDINGS_ONLY ? 'EMBEDDINGS ONLY' : 'FULL INGESTION'}`);
+  console.log('');
+
+  const logsDir = path.join(workspaceDir, 'logs', 'daily');
+  if (!fs.existsSync(logsDir)) {
+    console.log('❌ No daily logs directory found at:', logsDir);
+    process.exit(1);
+  }
+
+  let files = fs.readdirSync(logsDir)
+    .filter(f => /^\d{4}-\d{2}-\d{2}\.md$/.test(f))
+    .sort();
+
+  if (MAX_DAYS > 0) {
+    files = files.slice(-MAX_DAYS);
+  }
+
+  console.log(`📁 Found ${files.length} daily log files to process`);
+  console.log('');
+
+  // Step 1: Sync all daily logs to SQLite
+  console.log('── Step 1: Syncing daily logs to SQLite ──');
+  const upsert = dl.db.prepare(`
+    INSERT INTO daily_logs (date, raw_markdown) VALUES (?, ?)
+    ON CONFLICT(date) DO UPDATE SET raw_markdown = excluded.raw_markdown
+  `);
+  const syncTx = dl.db.transaction((fileList) => {
+    for (const file of fileList) {
+      const date = file.replace('.md', '');
+      const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
+      upsert.run(date, content);
+    }
+  });
+  syncTx(files);
+  console.log(`✅ ${files.length} daily logs synced to SQLite`);
+  console.log('');
+
+  // Step 2: Generate embeddings for all daily logs
+  if (!SKIP_EMBEDDINGS) {
+    console.log('── Step 2: Generating embeddings ──');
+    const dm = new DataManager(workspaceDir, logsDir);
+    let totalChunks = 0;
+    for (let i = 0; i < files.length; i++) {
+      const date = files[i].replace('.md', '');
+      const content = fs.readFileSync(path.join(logsDir, files[i]), 'utf8');
+      try {
+        const count = await dm.generateEmbeddings('daily_log', date, content);
+        totalChunks += count;
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ${count} chunks`);
+      } catch (err) {
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ ${err.message}`);
+      }
+    }
+    console.log(`\n✅ Generated ${totalChunks} embedding chunks total`);
+    console.log('');
+  }
+
+  if (EMBEDDINGS_ONLY) {
+    console.log('── Embeddings-only mode. Skipping task/blocker extraction. ──');
+    dl.db.save();
+    console.log('\n✅ Done!');
+    return;
+  }
+
+  // Step 3: Extract tasks/blockers from each daily log via planner
+  console.log('── Step 3: Extracting tasks & blockers via planner ──');
+
+  const cmd = process.env.COPILOT_CMD || 'copilot';
+  const agentEnv = { FREYA_WORKSPACE_DIR: workspaceDir };
+  const slugMap = readProjectSlugMap(workspaceDir);
+  const validTaskCats = new Set(['DO_NOW', 'SCHEDULE', 'DELEGATE', 'IGNORE']);
+
+  const schema = {
+    actions: [
+      { type: 'create_task', description: '<string>', priority: 'HIGH|MEDIUM|LOW', category: 'DO_NOW|SCHEDULE|DELEGATE|IGNORE', projectSlug: '<string optional>' },
+      { type: 'create_blocker', title: '<string>', severity: 'CRITICAL|HIGH|MEDIUM|LOW', notes: '<string>', projectSlug: '<string optional>' }
+    ]
+  };
+
+  const sysInstructions = `Você é o planner do sistema F.R.E.Y.A.
+
+Analise o daily log abaixo e extraia TODAS as tarefas e blockers mencionados.
+Procure por: ações mencionadas, pendências, problemas, impedimentos, decisões que geram trabalho.
+Se NÃO houver tarefas ou blockers claros, retorne: {"actions":[]}
+Retorne APENAS JSON válido no formato: ${JSON.stringify(schema)}
+NÃO use code fences. NÃO inclua texto extra.
+IMPORTANTE: Extraia APENAS informações explícitas do log. NÃO invente dados.`;
+
+  let totalTasks = 0;
+  let totalBlockers = 0;
+  let totalSkipped = 0;
+  let totalErrors = 0;
+
+  const insertTask = dl.db.prepare(`INSERT OR IGNORE INTO tasks (id, project_slug, description, category, status, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+  const insertBlocker = dl.db.prepare(`INSERT OR IGNORE INTO blockers (id, project_slug, title, severity, status, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+
+  // Build existing keys for dedup
+  const existingTaskDescs = new Set(
+    dl.db.prepare("SELECT description FROM tasks").all().map(t => sha1(normalizeTextForKey(t.description)))
+  );
+  const existingBlockerTitles = new Set(
+    dl.db.prepare("SELECT title FROM blockers").all().map(b => sha1(normalizeTextForKey(b.title)))
+  );
+
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    const date = file.replace('.md', '');
+    const content = fs.readFileSync(path.join(logsDir, file), 'utf8');
+
+    // Skip very small logs (< 50 chars) — likely empty or just a header
+    if (content.trim().length < 50) {
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — skipped (too small)`);
+      continue;
+    }
+
+    const fullPrompt = `${sysInstructions}\n\nDAILY LOG (${date}):\n${content}\n`;
+    const SAFE_ARG_LEN = 24000;
+
+    try {
+      let r;
+      if (fullPrompt.length > SAFE_ARG_LEN) {
+        const tmpFile = path.join(os.tmpdir(), `freya-retro-${Date.now()}.txt`);
+        fs.writeFileSync(tmpFile, fullPrompt, 'utf8');
+        const filePrompt = `Leia o arquivo abaixo e extraia tasks/blockers conforme as instruções contidas nele.\nARQUIVO: ${tmpFile}`;
+        r = await run(cmd, ['-s', '--no-color', '--stream', 'off', '--add-dir', os.tmpdir(), '--allow-all-tools', '-p', filePrompt], workspaceDir, agentEnv);
+        try { fs.unlinkSync(tmpFile); } catch { }
+      } else {
+        r = await run(cmd, ['-s', '--no-color', '--stream', 'off', '-p', fullPrompt], workspaceDir, agentEnv);
+      }
+
+      const out = (r.stdout + r.stderr).trim();
+      if (r.code !== 0 || !out) {
+        totalErrors++;
+        process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ planner error     `);
+        continue;
+      }
+
+      // Parse plan
+      const jsonText = extractFirstJsonObject(out) || out;
+      let plan;
+      try {
+        plan = JSON.parse(jsonText);
+      } catch {
+        try { plan = JSON.parse(escapeJsonControlChars(jsonText)); } catch {
+          totalErrors++;
+          process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ invalid JSON     `);
+          continue;
+        }
+      }
+
+      const actions = Array.isArray(plan.actions) ? plan.actions : [];
+      let fileTasks = 0;
+      let fileBlockers = 0;
+      let fileSkipped = 0;
+
+      if (!DRY_RUN) {
+        const applyTx = dl.db.transaction(() => {
+          for (const a of actions) {
+            if (!a || typeof a !== 'object') continue;
+
+            if (a.type === 'create_task' && a.description) {
+              const desc = normalizeWhitespace(a.description);
+              if (!desc) continue;
+              const projectSlug = String(a.projectSlug || '').trim() || inferProjectSlug(desc, slugMap);
+              const key = sha1(normalizeTextForKey((projectSlug ? projectSlug + ' ' : '') + desc));
+              if (existingTaskDescs.has(key)) { fileSkipped++; continue; }
+
+              const id = `t-retro-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+              const category = validTaskCats.has(String(a.category || '').trim()) ? String(a.category).trim() : 'DO_NOW';
+              const metadata = JSON.stringify({ priority: a.priority || 'medium', source: 'retroactive', sourceDate: date });
+              // Use the log date as created_at for chronological accuracy
+              insertTask.run(id, projectSlug || null, desc, category, 'PENDING', `${date}T12:00:00.000Z`, metadata);
+              existingTaskDescs.add(key);
+              fileTasks++;
+            }
+
+            if (a.type === 'create_blocker' && a.title) {
+              const title = normalizeWhitespace(a.title);
+              if (!title) continue;
+              const notes = normalizeWhitespace(a.notes);
+              const projectSlug = String(a.projectSlug || '').trim() || inferProjectSlug(title + ' ' + notes, slugMap);
+              const key = sha1(normalizeTextForKey((projectSlug ? projectSlug + ' ' : '') + title));
+              if (existingBlockerTitles.has(key)) { fileSkipped++; continue; }
+
+              const id = `b-retro-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`;
+              const severity = String(a.severity || 'MEDIUM').toUpperCase();
+              const metadata = JSON.stringify({ description: notes || title, source: 'retroactive', sourceDate: date });
+              insertBlocker.run(id, projectSlug || null, title, severity, 'OPEN', `${date}T12:00:00.000Z`, metadata);
+              existingBlockerTitles.add(key);
+              fileBlockers++;
+            }
+          }
+        });
+        applyTx();
+      } else {
+        // Dry run — just count
+        for (const a of actions) {
+          if (a && a.type === 'create_task' && a.description) fileTasks++;
+          if (a && a.type === 'create_blocker' && a.title) fileBlockers++;
+        }
+      }
+
+      totalTasks += fileTasks;
+      totalBlockers += fileBlockers;
+      totalSkipped += fileSkipped;
+
+      const status = fileTasks || fileBlockers
+        ? `${fileTasks}T ${fileBlockers}B${fileSkipped ? ` (${fileSkipped} dup)` : ''}`
+        : 'no actions';
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ${status}              `);
+
+      // Small delay to avoid rate limiting
+      if (i < files.length - 1) {
+        await new Promise(r => setTimeout(r, 500));
+      }
+
+    } catch (err) {
+      totalErrors++;
+      process.stdout.write(`\r  [${i + 1}/${files.length}] ${date} — ❌ ${err.message}     `);
+    }
+  }
+
+  // Ensure data is persisted
+  dl.db.save();
+
+  console.log('\n');
+  console.log('══════════════════════════════════════════════════════');
+  console.log(`  📊 Results ${DRY_RUN ? '(DRY RUN)' : ''}`);
+  console.log(`     Tasks created:    ${totalTasks}`);
+  console.log(`     Blockers created: ${totalBlockers}`);
+  console.log(`     Duplicates skipped: ${totalSkipped}`);
+  console.log(`     Errors:           ${totalErrors}`);
+  console.log('══════════════════════════════════════════════════════');
+
+  // Step 4: Generate embeddings for newly created tasks/blockers
+  if (!DRY_RUN && !SKIP_EMBEDDINGS && (totalTasks > 0 || totalBlockers > 0)) {
+    console.log('\n── Step 4: Generating embeddings for new tasks/blockers ──');
+    const dm = new DataManager(workspaceDir, logsDir);
+    const newTasks = dl.db.prepare("SELECT id, description FROM tasks WHERE json_extract(metadata, '$.source') = 'retroactive'").all();
+    const newBlockers = dl.db.prepare("SELECT id, title, json_extract(metadata, '$.description') as notes FROM blockers WHERE json_extract(metadata, '$.source') = 'retroactive'").all();
+
+    let embCount = 0;
+    for (const t of newTasks) {
+      try {
+        embCount += await dm.generateEmbeddings('task', t.id, t.description);
+      } catch { }
+    }
+    for (const b of newBlockers) {
+      try {
+        embCount += await dm.generateEmbeddings('blocker', b.id, b.title + ' ' + (b.notes || ''));
+      } catch { }
+    }
+    console.log(`✅ Generated ${embCount} embedding chunks for new entities`);
+  }
+
+  console.log('\n✅ Retroactive ingestion complete!');
+}
+
+main().catch(err => {
+  console.error('\n❌ Fatal error:', err.message || err);
+  process.exit(1);
+});