@cccarv82/freya 2.7.0 → 2.8.2

package/templates/base/scripts/generate-weekly-report.js

@@ -1,23 +1,34 @@
+/**
+ * generate-weekly-report.js
+ * Generates a weekly Markdown report aggregating Tasks, Blockers, Career entries,
+ * and Project Updates from the SQLite database.
+ *
+ * Usage: node scripts/generate-weekly-report.js
+ */
+
 const fs = require('fs');
 const path = require('path');
 
-const { toIsoDate, isWithinRange } = require('./lib/date-utils');
+const { toIsoDate, safeParseToMs } = require('./lib/date-utils');
+const DataManager = require('./lib/DataManager');
+const { ready } = require('./lib/DataLayer');
 
-const DATA_DIR = path.join(__dirname, '../data');
-const REPORT_DIR = path.join(__dirname, '../docs/reports');
+// --- Configuration (BUG-30: use FREYA_WORKSPACE_DIR instead of __dirname) ---
+const WORKSPACE_DIR = process.env.FREYA_WORKSPACE_DIR
+  ? path.resolve(process.env.FREYA_WORKSPACE_DIR)
+  : path.join(__dirname, '..'); // fallback: scripts/ is one level below workspace root
 
-// Ensure output dir exists
-if (!fs.existsSync(REPORT_DIR)) {
-    fs.mkdirSync(REPORT_DIR, { recursive: true });
-}
+const REPORT_DIR = path.join(WORKSPACE_DIR, 'docs', 'reports');
 
 // --- Date Logic ---
 const now = new Date();
 const oneDay = 24 * 60 * 60 * 1000;
 
 function isWithinWeek(dateStr) {
-    const sevenDaysAgo = new Date(now.getTime() - (7 * oneDay));
-    return isWithinRange(dateStr, sevenDaysAgo, now);
+    const ms = safeParseToMs(dateStr);
+    if (!Number.isFinite(ms)) return false;
+    const sevenDaysAgo = now.getTime() - (7 * oneDay);
+    return ms >= sevenDaysAgo && ms <= now.getTime();
 }
 
 function getFormattedDate() {
@@ -31,112 +42,87 @@ function getFormattedTime() {
     return `${hh}${mm}${ss}`;
 }
 
-// --- File Walking ---
-function walk(dir, fileList = []) {
-    const files = fs.readdirSync(dir);
-    files.forEach(file => {
-        const filePath = path.join(dir, file);
-        const stat = fs.statSync(filePath);
-        if (stat.isDirectory()) {
-            walk(filePath, fileList);
-        } else {
-            if (path.extname(file) === '.json') {
-                fileList.push(filePath);
-            }
-        }
-    });
-    return fileList;
-}
+// --- Report Generation ---
+async function generateWeeklyReport() {
+    await ready;
+
+    const start = new Date(now.getTime() - 7 * oneDay);
+    const end = now;
+
+    const dm = new DataManager();
+
+    // Fetch data from SQLite
+    const { completed: completedTasks } = dm.getTasks(start, end);
+    const { open: openBlockers, resolvedRecent } = dm.getBlockers(start, end);
+    const projectUpdates = dm.getProjectUpdates(start, end);
+    const careerEntries = dm.getCareerEntries ? dm.getCareerEntries(start, end) : [];
+
+    // Ensure output dir exists
+    if (!fs.existsSync(REPORT_DIR)) {
+        fs.mkdirSync(REPORT_DIR, { recursive: true });
+    }
 
-// --- Aggregation ---
-function generateWeeklyReport() {
-    const files = walk(DATA_DIR);
-    
-    const projects = [];
-    let taskLog = { schemaVersion: 1, tasks: [] };
-    let careerLog = { entries: [] };
-
-    // 1. Collect Data
-    files.forEach(file => {
-        try {
-            const content = fs.readFileSync(file, 'utf8');
-            const json = JSON.parse(content);
-
-            if (file.endsWith('task-log.json')) {
-                taskLog = json;
-            } else if (file.endsWith('career-log.json')) {
-                careerLog = json;
-            } else if (file.endsWith('status.json')) {
-                projects.push(json);
-            }
-        } catch (err) {
-            console.error(`Error reading ${file}: ${err.message}`);
-        }
-    });
-
-    // 2. Generate Content
     const reportDate = getFormattedDate();
     const reportTime = getFormattedTime();
     let report = `# Weekly Report - ${reportDate}\n\n`;
 
     // Projects
-    report += "## 🚀 Project Updates\n";
-    let hasProjectUpdates = false;
-    projects.forEach(p => {
-        if (p.history && Array.isArray(p.history)) {
-            const recentUpdates = p.history.filter(h => isWithinWeek(h.date));
-            if (recentUpdates.length > 0) {
-                hasProjectUpdates = true;
-                report += `### ${p.client} - ${p.project}\n`;
-                recentUpdates.forEach(u => {
-                    const dateStr = u.date ? u.date.split('T')[0] : 'Unknown Date';
-                    report += `- **${dateStr}**: ${u.content}\n`;
-                });
-                report += "\n";
-            }
-        }
-    });
-    if (!hasProjectUpdates) report += "No project updates recorded this week.\n\n";
-
-    // Tasks
-    report += "## ✅ Completed Tasks\n";
-    if (taskLog.tasks && Array.isArray(taskLog.tasks)) {
-        const recentTasks = taskLog.tasks.filter(t => t.status === "COMPLETED" && isWithinWeek(t.completedAt));
-        if (recentTasks.length > 0) {
-            recentTasks.forEach(t => {
-                report += `- ${t.description}\n`;
+    report += '## Project Updates\n';
+    if (projectUpdates.length > 0) {
+        projectUpdates.forEach(p => {
+            report += `### ${p.client || 'Unknown'} - ${p.project || p.slug || 'Unknown'}\n`;
+            const events = Array.isArray(p.events) ? p.events : [];
+            events.forEach(e => {
+                const dateStr = e.date ? String(e.date).slice(0, 10) : 'Unknown Date';
+                report += `- **${dateStr}**: ${e.content || ''}\n`;
             });
-        } else {
-            report += "No tasks completed this week.\n";
-        }
+            report += '\n';
+        });
     } else {
-        report += "No task log found.\n";
+        report += 'No project updates recorded this week.\n\n';
     }
-    report += "\n";
-
-    // Career
-    report += "## 🌟 Career Highlights\n";
-    if (careerLog.entries && Array.isArray(careerLog.entries)) {
-        const recentCareer = careerLog.entries.filter(e => isWithinWeek(e.date));
-        if (recentCareer.length > 0) {
-            recentCareer.forEach(e => {
-                report += `- **[${e.type}]**: ${e.description}\n`;
-            });
-        } else {
-            report += "No career updates this week.\n";
-        }
+
+    // Completed Tasks
+    report += '## Completed Tasks\n';
+    if (completedTasks.length > 0) {
+        completedTasks.forEach(t => {
+            const projectTag = t.projectSlug || t.project_slug ? `[${t.projectSlug || t.project_slug}] ` : '';
+            report += `- ${projectTag}${t.description}\n`;
+        });
     } else {
-        report += "No career log found.\n";
+        report += 'No tasks completed this week.\n';
+    }
+    report += '\n';
+
+    // Open Blockers
+    report += '## Open Blockers\n';
+    if (openBlockers.length > 0) {
+        openBlockers.forEach(b => {
+            const sev = b.severity ? `[${b.severity}] ` : '';
+            report += `- ${sev}${b.title}\n`;
+        });
+    } else {
+        report += 'No open blockers.\n';
+    }
+    report += '\n';
+
+    // Career entries (if DataManager supports it)
+    if (Array.isArray(careerEntries) && careerEntries.length > 0) {
+        report += '## Career Highlights\n';
+        careerEntries.forEach(e => {
+            report += `- **[${e.type || 'Note'}]**: ${e.description || e.content || ''}\n`;
+        });
+        report += '\n';
     }
 
     // 3. Save and Output
     const outputPath = path.join(REPORT_DIR, `weekly-${reportDate}-${reportTime}.md`);
     fs.writeFileSync(outputPath, report);
-    
-    console.log(`✅ Report generated at: ${outputPath}`);
-    console.log("---------------------------------------------------");
+
+    console.log(`Report generated at: ${outputPath}`);
+    console.log('---------------------------------------------------');
     console.log(report);
-    console.log("---------------------------------------------------");
+    console.log('---------------------------------------------------');
 }
 
-generateWeeklyReport();
+generateWeeklyReport().catch(err => { console.error(err); process.exit(1); });

package/templates/base/scripts/lib/DataLayer.js

@@ -1,10 +1,10 @@
 /**
  * DataLayer.js (V2.1 - sql.js powered, no native compilation needed)
- * 
+ *
  * Drop-in replacement for better-sqlite3 using sql.js (WebAssembly SQLite).
  * Exposes the same API surface: .prepare().all(), .prepare().get(), .prepare().run(),
  * .exec(), .pragma(), .transaction(), .close()
- * 
+ *
  * Auto-persists to disk after every write operation.
  */
 
@@ -72,7 +72,10 @@ class PreparedStatement {
     const db = this._wrapper._db;
     db.run(this._sql, this._flattenParams(params));
     const changes = db.getRowsModified();
-    this._wrapper._save();
+    // Only save to disk if NOT inside a transaction (save happens at COMMIT)
+    if (!this._wrapper._inTransaction) {
+      this._wrapper._save();
+    }
     return { changes };
   }
 
@@ -92,7 +95,7 @@ class SqlJsDatabase {
   constructor(sqlJsDb, filePath) {
     this._db = sqlJsDb;
     this._filePath = filePath;
-    this._saveScheduled = false;
+    this._inTransaction = false;
   }
 
   /**
@@ -126,7 +129,9 @@ class SqlJsDatabase {
    */
   exec(sql) {
     this._db.run(sql);
-    this._save();
+    if (!this._inTransaction) {
+      this._save();
+    }
   }
 
   /**
@@ -144,19 +149,31 @@ class SqlJsDatabase {
   transaction(fn) {
     const self = this;
     return function (...args) {
+      self._inTransaction = true;
       self._db.run('BEGIN TRANSACTION');
       try {
         const result = fn(...args);
         self._db.run('COMMIT');
+        self._inTransaction = false;
         self._save();
         return result;
       } catch (e) {
+        self._inTransaction = false;
         try { self._db.run('ROLLBACK'); } catch { /* already rolled back */ }
         throw e;
       }
     };
   }
 
+  /**
+   * Public save — explicitly flush the in-memory database to disk.
+   * Use this when you need to ensure data is persisted before spawning a
+   * child process that will read the same SQLite file (e.g. in tests).
+   */
+  save() {
+    this._save();
+  }
+
   /**
    * Close the database.
    */
@@ -173,7 +190,13 @@ class SqlJsDatabase {
 class DataLayer {
   constructor(dbPath = null) {
     if (!dbPath) {
-      const dataDir = path.join(__dirname, '..', '..', 'data');
+      // Prefer FREYA_WORKSPACE_DIR env var so that the web server and report
+      // child processes always share the same SQLite file regardless of where
+      // each DataLayer.js lives on disk.
+      const workspaceDir = process.env.FREYA_WORKSPACE_DIR
+        ? path.resolve(process.env.FREYA_WORKSPACE_DIR)
+        : path.join(__dirname, '..', '..');
+      const dataDir = path.join(workspaceDir, 'data');
       if (!fs.existsSync(dataDir)) {
         fs.mkdirSync(dataDir, { recursive: true });
       }
@@ -192,6 +215,40 @@ class DataLayer {
     const initSqlJs = require('sql.js');
     const SQL = await initSqlJs();
 
+    // --- WAL consolidation (migration from better-sqlite3) ---
+    // better-sqlite3 used WAL mode, which may leave a -wal file with uncommitted data.
+    // sql.js cannot read WAL files directly, so we need to checkpoint first.
+    const walPath = this._dbPath + '-wal';
+    const shmPath = this._dbPath + '-shm';
+    if (fs.existsSync(this._dbPath) && fs.existsSync(walPath)) {
+      try {
+        console.log('[DataLayer] Consolidating WAL file from previous better-sqlite3 usage...');
+        // Open the database including the WAL data
+        const fileBuffer = fs.readFileSync(this._dbPath);
+        const tempDb = new SQL.Database(fileBuffer);
+
+        // Try to checkpoint the WAL (merge pending writes into main file)
+        try { tempDb.run('PRAGMA wal_checkpoint(TRUNCATE)'); } catch { /* may not work in WASM */ }
+
+        // Switch to DELETE journal mode (fully compatible with sql.js)
+        try { tempDb.run('PRAGMA journal_mode = DELETE'); } catch { /* ignore */ }
+
+        // Export the consolidated database and save it
+        const consolidated = tempDb.export();
+        fs.writeFileSync(this._dbPath, Buffer.from(consolidated));
+        tempDb.close();
+
+        // Clean up WAL/SHM files
+        try { fs.unlinkSync(walPath); } catch { /* ignore */ }
+        try { fs.unlinkSync(shmPath); } catch { /* ignore */ }
+
+        console.log('[DataLayer] WAL consolidation complete.');
+      } catch (err) {
+        console.error('[DataLayer] Warning: WAL consolidation failed:', err.message);
+        // Continue anyway — the main .sqlite file may still be usable
+      }
+    }
+
     // Load existing database from disk, or create new one
     let sqlJsDb;
     if (fs.existsSync(this._dbPath)) {
@@ -206,8 +263,8 @@ class DataLayer {
   }
 
   _initSchema() {
-    // WAL mode may not work in WASM — we try, but it's silently ignored if unsupported
-    this.db.pragma('journal_mode = WAL');
+    // Use DELETE journal mode (compatible with sql.js WASM)
+    this.db.pragma('journal_mode = DELETE');
 
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS projects (
@@ -262,7 +319,7 @@ class DataLayer {
       CREATE TABLE IF NOT EXISTS document_embeddings (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
         reference_type TEXT NOT NULL, /* 'daily_log', 'task', 'blocker' */
-        reference_id TEXT NOT NULL, 
+        reference_id TEXT NOT NULL,
         chunk_index INTEGER DEFAULT 0,
         text_chunk TEXT NOT NULL,
         embedding BLOB NOT NULL, /* Stored as Buffer of Float32Array */
@@ -290,4 +347,35 @@ const ready = defaultInstance.init().catch(err => {
   process.exit(1);
 });
 
-module.exports = { defaultInstance, DataLayer, ready };
+/**
+ * Configure the singleton to use a specific workspace directory.
+ * Call this before performing any DB operations against the target workspace.
+ * Closes the current in-memory DB (if open) and reinitializes from the new path.
+ *
+ * This is used by the web server to ensure it uses the same SQLite file as
+ * report child processes (both pointed at {workspaceDir}/data/freya.sqlite).
+ *
+ * @param {string} workspaceDir  Absolute path to the Freya workspace directory.
+ * @returns {Promise<void>}      Resolves when the new DB is ready.
+ */
+async function configure(workspaceDir) {
+  const newDbPath = path.join(path.resolve(workspaceDir), 'data', 'freya.sqlite');
+  if (defaultInstance._dbPath === newDbPath && defaultInstance.db) {
+    // Already pointing at the right file.
+    return;
+  }
+  // Close existing connection if open.
+  if (defaultInstance.db) {
+    try { defaultInstance.db.close(); } catch { /* ignore */ }
+    defaultInstance.db = null;
+  }
+  // Set the new path and reinitialize.
+  defaultInstance._dbPath = newDbPath;
+  const dataDir = path.dirname(newDbPath);
+  if (!fs.existsSync(dataDir)) {
+    fs.mkdirSync(dataDir, { recursive: true });
+  }
+  await defaultInstance.init();
+}
+
+module.exports = { defaultInstance, DataLayer, ready, configure };

package/templates/base/scripts/lib/DataManager.js

@@ -25,7 +25,13 @@ class DataManager {
     // --- Tasks ---
     getTasksRaw() {
         return dl.db.prepare('SELECT * FROM tasks').all().map(t => {
-            const meta = t.metadata ? JSON.parse(t.metadata) : {};
+            // BUG-17: JSON.parse can throw on corrupted metadata; default to {}
+            let meta = {};
+            try {
+                meta = t.metadata ? JSON.parse(t.metadata) : {};
+            } catch {
+                meta = {};
+            }
             return {
                 id: t.id,
                 projectSlug: t.project_slug,
@@ -48,13 +54,13 @@ class DataManager {
 
         const tasks = this.getTasksRaw();
         const completed = dl.db.prepare(`
-            SELECT * FROM tasks 
-            WHERE status = 'COMPLETED' 
+            SELECT * FROM tasks
+            WHERE status = 'COMPLETED'
             AND completed_at >= ? AND completed_at <= ?
         `).all(startIso, endIso).map(t => ({ ...t, completedAt: t.completed_at, createdAt: t.created_at }));
 
         const pending = dl.db.prepare(`
-            SELECT * FROM tasks 
+            SELECT * FROM tasks
             WHERE status = 'PENDING' AND category = 'DO_NOW'
         `).all().map(t => ({ ...t, createdAt: t.created_at }));
 
@@ -64,7 +70,13 @@ class DataManager {
     // --- Blockers ---
     getBlockersRaw() {
         return dl.db.prepare('SELECT * FROM blockers').all().map(b => {
-            const meta = b.metadata ? JSON.parse(b.metadata) : {};
+            // BUG-17: JSON.parse can throw on corrupted metadata; default to {}
+            let meta = {};
+            try {
+                meta = b.metadata ? JSON.parse(b.metadata) : {};
+            } catch {
+                meta = {};
+            }
             return {
                 id: b.id,
                 projectSlug: b.project_slug,
@@ -88,27 +100,27 @@ class DataManager {
         const blockers = this.getBlockersRaw();
 
         const open = dl.db.prepare(`
-            SELECT * FROM blockers 
+            SELECT * FROM blockers
             WHERE status NOT IN ('RESOLVED', 'CLOSED', 'DONE', 'FIXED')
             AND resolved_at IS NULL
-            ORDER BY 
-              CASE severity 
-                WHEN 'CRITICAL' THEN 0 
-                WHEN 'HIGH' THEN 1 
-                WHEN 'MEDIUM' THEN 2 
-                WHEN 'LOW' THEN 3 
-                ELSE 99 
+            ORDER BY
+              CASE severity
+                WHEN 'CRITICAL' THEN 0
+                WHEN 'HIGH' THEN 1
+                WHEN 'MEDIUM' THEN 2
+                WHEN 'LOW' THEN 3
+                ELSE 99
               END ASC,
               created_at ASC
         `).all().map(b => ({ ...b, projectSlug: b.project_slug, createdAt: b.created_at }));
 
         const openedRecent = dl.db.prepare(`
-            SELECT * FROM blockers 
+            SELECT * FROM blockers
             WHERE created_at >= ? AND created_at <= ?
         `).all(startIso, endIso).map(b => ({ ...b, projectSlug: b.project_slug, createdAt: b.created_at }));
 
         const resolvedRecent = dl.db.prepare(`
-            SELECT * FROM blockers 
+            SELECT * FROM blockers
             WHERE resolved_at >= ? AND resolved_at <= ?
         `).all(startIso, endIso).map(b => ({ ...b, projectSlug: b.project_slug, resolvedAt: b.resolved_at }));
 
@@ -160,9 +172,17 @@ class DataManager {
 
     // --- Daily Logs ---
     getDailyLogs(start, end) {
-        function toIso(d) { return (d instanceof Date ? d : new Date(d)).toISOString().slice(0, 10); }
-        const startIso = formatDate ? formatDate(start) : toIso(start);
-        const endIso = formatDate ? formatDate(end) : toIso(end);
+        // BUG-41: use local date parts instead of toISOString().slice(0,10) to avoid
+        // timezone drift (UTC midnight may differ from local midnight)
+        function toLocalDate(d) {
+            const dt = d instanceof Date ? d : new Date(d);
+            const yyyy = dt.getFullYear();
+            const mm = String(dt.getMonth() + 1).padStart(2, '0');
+            const dd = String(dt.getDate()).padStart(2, '0');
+            return `${yyyy}-${mm}-${dd}`;
+        }
+        const startIso = formatDate ? formatDate(start) : toLocalDate(start);
+        const endIso = formatDate ? formatDate(end) : toLocalDate(end);
 
         return dl.db.prepare(`
             SELECT date, raw_markdown as content FROM daily_logs
@@ -171,13 +191,50 @@ class DataManager {
         `).all(startIso, endIso);
     }
 
+    // --- Timestamp helpers ---
+    // These accept a record (task or blocker) and return milliseconds since epoch, or NaN.
+
+    getCreatedAt(record) {
+        const { safeParseToMs } = require('./date-utils');
+        const candidates = [
+            record.createdAt,
+            record.created_at,
+            record.openedAt,
+            record.opened_at,
+            record.date,
+            record.loggedAt,
+        ];
+        for (const value of candidates) {
+            const ms = safeParseToMs(value);
+            if (Number.isFinite(ms)) return ms;
+        }
+        return NaN;
+    }
+
+    getResolvedAt(record) {
+        const { safeParseToMs } = require('./date-utils');
+        const candidates = [
+            record.resolvedAt,
+            record.resolved_at,
+            record.closedAt,
+            record.closed_at,
+            record.completedAt,
+            record.completed_at,
+        ];
+        for (const value of candidates) {
+            const ms = safeParseToMs(value);
+            if (Number.isFinite(ms)) return ms;
+        }
+        return NaN;
+    }
+
     // --- RAG (Vector Search) ---
     async semanticSearch(query, topK = 10) {
         const queryVector = await defaultEmbedder.embedText(query);
 
         // Fetch all stored embeddings. For a local personal tool with < 100k chunks, in-memory cosine sim is perfectly fast.
         const rows = dl.db.prepare(`
-            SELECT reference_type, reference_id, chunk_index, text_chunk, embedding 
+            SELECT reference_type, reference_id, chunk_index, text_chunk, embedding
             FROM document_embeddings
         `).all();
 

package/templates/base/scripts/lib/index-utils.js

@@ -8,7 +8,8 @@ const ID_PATTERNS = [
 ];
 
 const TEXT_EXTS = new Set(['.md', '.txt', '.log', '.json', '.yaml', '.yml']);
-const TOKEN_RE = /[A-Za-z0-9_-]{3,}/g;
+// BUG-46: TOKEN_RE was a module-level global with /g flag — its lastIndex would
+// persist between calls causing random skipped tokens. Moved inside the function.
 
 const DEFAULT_MAX_SIZE = 2 * 1024 * 1024;
 const DEFAULT_TOKEN_LIMIT = 500;
@@ -142,6 +143,8 @@ function extractIdMatches(text) {
 }
 
 function extractKeywordIndexMap(textLower, tokenLimit) {
+  // BUG-46: create a fresh regex each call — module-level /g regex retains lastIndex
+  const TOKEN_RE = /[A-Za-z0-9_-]{3,}/g;
   const map = new Map();
   let m;
   while ((m = TOKEN_RE.exec(textLower)) !== null) {
@@ -151,7 +154,6 @@ function extractKeywordIndexMap(textLower, tokenLimit) {
       if (map.size >= tokenLimit) break;
     }
   }
-  TOKEN_RE.lastIndex = 0;
   return map;
 }
 

package/templates/base/scripts/migrate-data.js

@@ -3,9 +3,16 @@ const path = require('path');
 
 const { safeReadJson, quarantineCorruptedFile } = require('./lib/fs-utils');
 
+// BUG-43: support FREYA_WORKSPACE_DIR (used by web server) in addition to DATA_DIR
+const WORKSPACE_DIR = process.env.FREYA_WORKSPACE_DIR
+  ? path.resolve(process.env.FREYA_WORKSPACE_DIR)
+  : null;
+
 const DATA_DIR = process.env.DATA_DIR
   ? path.resolve(process.env.DATA_DIR)
-  : path.join(__dirname, '../data');
+  : WORKSPACE_DIR
+    ? path.join(WORKSPACE_DIR, 'data')
+    : path.join(__dirname, '../data');
 
 const KNOWN_FILES = [
   { relPath: path.join('tasks', 'task-log.json'), label: 'tasks/task-log.json' },