@cccarv82/freya 2.3.13 → 2.4.1

package/scripts/generate-sm-weekly-report.js

@@ -1,61 +1,39 @@
 const fs = require('fs');
 const path = require('path');
 
-const { toIsoDate, safeParseToMs, isWithinRange } = require('./lib/date-utils');
-const { safeReadJson, quarantineCorruptedFile } = require('./lib/fs-utils');
+const { toIsoDate, safeParseToMs } = require('./lib/date-utils');
+const DataManager = require('./lib/DataManager');
 
 const DATA_DIR = path.join(__dirname, '../data');
+const LOGS_DIR = path.join(__dirname, '../logs/daily');
 const REPORT_DIR = path.join(__dirname, '../docs/reports');
 
-const TASKS_FILE = path.join(DATA_DIR, 'tasks/task-log.json');
-const BLOCKERS_FILE = path.join(DATA_DIR, 'blockers/blocker-log.json');
-const CLIENTS_DIR = path.join(DATA_DIR, 'Clients');
-
-const SEV_ORDER = { CRITICAL: 0, HIGH: 1, MEDIUM: 2, LOW: 3 };
+const dm = new DataManager(DATA_DIR, LOGS_DIR);
 
 function ensureDir(dir) {
   if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
 }
 
-function readJsonOrQuarantine(filePath, fallback) {
-  if (!fs.existsSync(filePath)) return fallback;
-  const res = safeReadJson(filePath);
-  if (!res.ok) {
-    if (res.error.type === 'parse') {
-      quarantineCorruptedFile(filePath, res.error.message);
-      console.warn(`⚠️ JSON parse failed; quarantined: ${filePath}`);
-    } else {
-      console.warn(`⚠️ JSON read failed: ${filePath}: ${res.error.message}`);
-    }
-    return fallback;
-  }
-  return res.json;
-}
-
 function daysBetweenMs(aMs, bMs) {
   const diff = Math.max(0, bMs - aMs);
   return Math.floor(diff / (24 * 60 * 60 * 1000));
 }
 
-function scanProjectStatusFiles() {
-  const results = [];
-  if (!fs.existsSync(CLIENTS_DIR)) return results;
-
-  const clients = fs.readdirSync(CLIENTS_DIR);
-  for (const clientSlug of clients) {
-    const clientPath = path.join(CLIENTS_DIR, clientSlug);
-    if (!fs.statSync(clientPath).isDirectory()) continue;
-
-    const projects = fs.readdirSync(clientPath);
-    for (const projectSlug of projects) {
-      const projectPath = path.join(clientPath, projectSlug);
-      if (!fs.statSync(projectPath).isDirectory()) continue;
+function normalizeStatus(blocker) {
+  const raw = blocker.status || blocker.state || blocker.currentStatus;
+  if (!raw) return 'UNKNOWN';
+  return String(raw).trim().toUpperCase();
+}
 
-      const statusPath = path.join(projectPath, 'status.json');
-      if (fs.existsSync(statusPath)) results.push(statusPath);
-    }
-  }
-  return results;
+function normalizeSeverity(blocker) {
+  const raw = blocker.severity || blocker.priority || blocker.level;
+  if (!raw) return 'UNSPECIFIED';
+  const value = String(raw).trim().toUpperCase();
+  if (value.includes('CRIT')) return 'CRITICAL';
+  if (value.includes('HIGH')) return 'HIGH';
+  if (value.includes('MED')) return 'MEDIUM';
+  if (value.includes('LOW')) return 'LOW';
+  return value;
 }
 
 function generate() {
@@ -67,40 +45,15 @@ function generate() {
   start.setDate(now.getDate() - 7);
 
   const reportDate = toIsoDate(now);
-const reportTime = (() => { const d = new Date(); const hh = String(d.getHours()).padStart(2,'0'); const mm = String(d.getMinutes()).padStart(2,'0'); const ss = String(d.getSeconds()).padStart(2,'0'); return `${hh}${mm}${ss}`; })();
-
-  const taskLog = readJsonOrQuarantine(TASKS_FILE, { schemaVersion: 1, tasks: [] });
-  const blockersLog = readJsonOrQuarantine(BLOCKERS_FILE, { schemaVersion: 1, blockers: [] });
+  const reportTime = (() => { const d = new Date(); const hh = String(d.getHours()).padStart(2, '0'); const mm = String(d.getMinutes()).padStart(2, '0'); const ss = String(d.getSeconds()).padStart(2, '0'); return `${hh}${mm}${ss}`; })();
 
-  const tasks = taskLog.tasks || [];
-  const blockers = blockersLog.blockers || [];
-
-  const completedTasks = tasks.filter(t => {
-    if (t.status !== 'COMPLETED') return false;
-    return isWithinRange(t.completedAt || t.completed_at, start, end);
-  });
-
-  const pendingDoNow = tasks.filter(t => t.status === 'PENDING' && t.category === 'DO_NOW');
-
-  const activeBlockers = blockers.filter(b => {
-    const st = String(b.status || '').toUpperCase();
-    return st === 'OPEN' || st === 'MITIGATING';
-  });
-
-  activeBlockers.sort((a, b) => {
-    const sa = SEV_ORDER[String(a.severity || '').toUpperCase()] ?? 99;
-    const sb = SEV_ORDER[String(b.severity || '').toUpperCase()] ?? 99;
-    if (sa !== sb) return sa - sb;
-    const ta = safeParseToMs(a.createdAt) || 0;
-    const tb = safeParseToMs(b.createdAt) || 0;
-    return ta - tb; // older first
-  });
+  const { completed: completedTasks, pending: pendingDoNow } = dm.getTasks(start, end);
+  const { open: activeBlockers, blockers } = dm.getBlockers(start, end);
 
   const upcomingDueBlockers = blockers.filter(b => {
-    const st = String(b.status || '').toUpperCase();
+    const st = normalizeStatus(b);
     if (st !== 'OPEN') return false;
     if (!b.dueDate) return false;
-    // dueDate is an ISO date; treat as UTC midnight
     const dueMs = safeParseToMs(String(b.dueDate));
     if (!Number.isFinite(dueMs)) return false;
     const due = new Date(dueMs);
@@ -109,25 +62,7 @@ const reportTime = (() => { const d = new Date(); const hh = String(d.getHours()
     return due >= now && due <= soonEnd;
   });
 
-  // Projects
-  const statusFiles = scanProjectStatusFiles();
-  const projectsWithUpdates = [];
-
-  for (const statusPath of statusFiles) {
-    const proj = readJsonOrQuarantine(statusPath, null);
-    if (!proj) continue;
-
-    const history = Array.isArray(proj.history) ? proj.history : [];
-    const recent = history.filter(h => isWithinRange(h.date || h.timestamp, start, end));
-    if (recent.length === 0) continue;
-
-    projectsWithUpdates.push({
-      client: proj.client || path.basename(path.dirname(path.dirname(statusPath))),
-      project: proj.project || path.basename(path.dirname(statusPath)),
-      currentStatus: proj.currentStatus,
-      recent
-    });
-  }
+  const projectsWithUpdates = dm.getProjectUpdates(start, end);
 
   // Markdown
   let md = `# Scrum Master Weekly Report — ${reportDate}\n`;
@@ -153,13 +88,13 @@ const reportTime = (() => { const d = new Date(); const hh = String(d.getHours()
     md += `None.\n\n`;
   } else {
     activeBlockers.forEach(b => {
-      const createdMs = safeParseToMs(b.createdAt) || Date.now();
+      const createdMs = safeParseToMs(b.created_at || b.createdAt) || Date.now();
       const agingDays = daysBetweenMs(createdMs, Date.now());
-      const sev = String(b.severity || '').toUpperCase();
-      const st = String(b.status || '').toUpperCase();
+      const sev = normalizeSeverity(b);
+      const st = normalizeStatus(b);
       const owner = b.owner ? `; owner: ${b.owner}` : '';
       const proj = b.projectSlug ? `; project: ${b.projectSlug}` : '';
-      const next = b.nextAction ? `; next: ${b.nextAction}` : '';
+      const next = b.next_action || b.nextAction ? `; next: ${b.next_action || b.nextAction}` : '';
       md += `- [${sev}/${st}] ${b.title} (aging: ${agingDays}d${owner}${proj}${next})\n`;
     });
     md += `\n`;

package/scripts/lib/DataLayer.js

@@ -0,0 +1,92 @@
+const Database = require('better-sqlite3');
+const path = require('path');
+const fs = require('fs');
+
+class DataLayer {
+  constructor(dbPath = null) {
+    if (!dbPath) {
+      const dataDir = path.join(__dirname, '..', '..', 'data');
+      if (!fs.existsSync(dataDir)) {
+        fs.mkdirSync(dataDir, { recursive: true });
+      }
+      dbPath = path.join(dataDir, 'freya.sqlite');
+    }
+    this.db = new Database(dbPath, { verbose: console.log });
+    this.initSchema();
+  }
+
+  initSchema() {
+    // Enable Write-Ahead Logging for better concurrent performance
+    this.db.pragma('journal_mode = WAL');
+
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS projects (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        slug TEXT UNIQUE NOT NULL,
+        client TEXT,
+        name TEXT,
+        is_active BOOLEAN DEFAULT 1,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      );
+
+      CREATE TABLE IF NOT EXISTS project_status_history (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        project_id INTEGER NOT NULL,
+        status_text TEXT NOT NULL,
+        date DATETIME DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE
+      );
+
+      CREATE TABLE IF NOT EXISTS tasks (
+        id TEXT PRIMARY KEY, /* UUID */
+        project_slug TEXT, /* Can be null, or match projects.slug */
+        description TEXT NOT NULL,
+        category TEXT NOT NULL, /* DO_NOW, SCHEDULE, DELEGATE, IGNORE */
+        status TEXT NOT NULL, /* PENDING, COMPLETED, ARCHIVED */
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        completed_at DATETIME,
+        metadata TEXT /* JSON string for extra fields like priority, streamSlug */
+      );
+
+      CREATE TABLE IF NOT EXISTS blockers (
+        id TEXT PRIMARY KEY, /* UUID */
+        project_slug TEXT,
+        title TEXT NOT NULL,
+        severity TEXT NOT NULL, /* CRITICAL, HIGH, MEDIUM, LOW */
+        status TEXT NOT NULL, /* OPEN, MITIGATING, RESOLVED, ARCHIVED */
+        owner TEXT,
+        next_action TEXT,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        resolved_at DATETIME,
+        metadata TEXT /* JSON string */
+      );
+
+      CREATE TABLE IF NOT EXISTS daily_logs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        date TEXT UNIQUE NOT NULL, /* YYYY-MM-DD */
+        raw_markdown TEXT NOT NULL,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      );
+
+      CREATE TABLE IF NOT EXISTS document_embeddings (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        reference_type TEXT NOT NULL, /* 'daily_log', 'task', 'blocker' */
+        reference_id TEXT NOT NULL, 
+        chunk_index INTEGER DEFAULT 0,
+        text_chunk TEXT NOT NULL,
+        embedding BLOB NOT NULL, /* Stored as Buffer of Float32Array */
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      );
+    `);
+  }
+
+  // Helper close method
+  close() {
+    this.db.close();
+  }
+}
+
+// Export a singleton instance by default, or the class for testing
+const defaultInstance = new DataLayer();
+module.exports = { defaultInstance, DataLayer };

package/scripts/lib/DataManager.js

@@ -0,0 +1,198 @@
+/**
+ * DataManager.js (V2 - SQLite Powered)
+ * Centralized data access layer for F.R.E.Y.A. reports and scripts.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { isWithinRange, formatDate } = require('./date-utils');
+const SCHEMA = require('./schema');
+const { defaultInstance: dl } = require('./DataLayer');
+const { defaultEmbedder } = require('./Embedder');
+
+class DataManager {
+    constructor(dataDir, logsDir) {
+        // Keeping args for backwards compatibility, though SQLite handles the storage now
+        this.dataDir = dataDir;
+        this.logsDir = logsDir;
+    }
+
+    // --- Helpers (Deprecated but kept for backwards compat if needed elsewhere) ---
+    readJsonOrQuarantine(filePath, fallback) {
+        return fallback;
+    }
+
+    // --- Tasks ---
+    getTasksRaw() {
+        return dl.db.prepare('SELECT * FROM tasks').all().map(t => {
+            const meta = t.metadata ? JSON.parse(t.metadata) : {};
+            return {
+                id: t.id,
+                projectSlug: t.project_slug,
+                description: t.description,
+                category: t.category,
+                status: t.status,
+                createdAt: t.created_at,
+                completedAt: t.completed_at,
+                priority: meta.priority,
+                streamSlug: meta.streamSlug
+            };
+        });
+    }
+
+    getTasks(start, end) {
+        // SQLite date func: use ISO format
+        const toIso = (d) => d ? (d instanceof Date ? d.toISOString() : new Date(d).toISOString()) : null;
+        const startIso = toIso(start);
+        const endIso = toIso(end);
+
+        const tasks = this.getTasksRaw();
+        const completed = dl.db.prepare(`
+            SELECT * FROM tasks 
+            WHERE status = 'COMPLETED' 
+            AND completed_at >= ? AND completed_at <= ?
+        `).all(startIso, endIso).map(t => ({ ...t, completedAt: t.completed_at, createdAt: t.created_at }));
+
+        const pending = dl.db.prepare(`
+            SELECT * FROM tasks 
+            WHERE status = 'PENDING' AND category = 'DO_NOW'
+        `).all().map(t => ({ ...t, createdAt: t.created_at }));
+
+        return { tasks, completed, pending };
+    }
+
+    // --- Blockers ---
+    getBlockersRaw() {
+        return dl.db.prepare('SELECT * FROM blockers').all().map(b => {
+            const meta = b.metadata ? JSON.parse(b.metadata) : {};
+            return {
+                id: b.id,
+                projectSlug: b.project_slug,
+                title: b.title,
+                severity: b.severity,
+                status: b.status,
+                owner: b.owner,
+                nextAction: b.next_action,
+                createdAt: b.created_at,
+                resolvedAt: b.resolved_at,
+                streamSlug: meta.streamSlug
+            };
+        });
+    }
+
+    getBlockers(start, end) {
+        const toIso = (d) => d ? (d instanceof Date ? d.toISOString() : new Date(d).toISOString()) : null;
+        const startIso = toIso(start);
+        const endIso = toIso(end);
+
+        const blockers = this.getBlockersRaw();
+
+        const open = dl.db.prepare(`
+            SELECT * FROM blockers 
+            WHERE status NOT IN ('RESOLVED', 'CLOSED', 'DONE', 'FIXED')
+            AND resolved_at IS NULL
+            ORDER BY 
+              CASE severity 
+                WHEN 'CRITICAL' THEN 0 
+                WHEN 'HIGH' THEN 1 
+                WHEN 'MEDIUM' THEN 2 
+                WHEN 'LOW' THEN 3 
+                ELSE 99 
+              END ASC,
+              created_at ASC
+        `).all().map(b => ({ ...b, projectSlug: b.project_slug, createdAt: b.created_at }));
+
+        const openedRecent = dl.db.prepare(`
+            SELECT * FROM blockers 
+            WHERE created_at >= ? AND created_at <= ?
+        `).all(startIso, endIso).map(b => ({ ...b, projectSlug: b.project_slug, createdAt: b.created_at }));
+
+        const resolvedRecent = dl.db.prepare(`
+            SELECT * FROM blockers 
+            WHERE resolved_at >= ? AND resolved_at <= ?
+        `).all(startIso, endIso).map(b => ({ ...b, projectSlug: b.project_slug, resolvedAt: b.resolved_at }));
+
+        return { blockers, open, openedRecent, resolvedRecent };
+    }
+
+    // --- Project Updates ---
+    getProjectUpdates(start, end) {
+        const toIso = (d) => d ? (d instanceof Date ? d.toISOString() : new Date(d).toISOString()) : null;
+        const startIso = toIso(start);
+        const endIso = toIso(end);
+
+        const updates = [];
+
+        // Find active projects with recent history
+        const activeProjects = dl.db.prepare(`SELECT * FROM projects WHERE is_active = 1`).all();
+
+        for (const proj of activeProjects) {
+            const recentHistory = dl.db.prepare(`
+                SELECT status_text, date FROM project_status_history
+                WHERE project_id = ? AND date >= ? AND date <= ?
+                ORDER BY date DESC
+            `).all(proj.id, startIso, endIso).map(h => ({
+                content: h.status_text,
+                date: h.date
+            }));
+
+            // We determine currentStatus as the latest from history, or a placeholder if none
+            const latest = dl.db.prepare(`
+                SELECT status_text FROM project_status_history
+                WHERE project_id = ? ORDER BY date DESC LIMIT 1
+            `).get(proj.id);
+
+            const currentStatus = latest ? latest.status_text : "Initialized";
+
+            if (recentHistory.length > 0) {
+                updates.push({
+                    client: proj.client,
+                    project: proj.name,
+                    slug: proj.slug,
+                    currentStatus: currentStatus,
+                    events: recentHistory
+                });
+            }
+        }
+
+        return updates;
+    }
+
+    // --- Daily Logs ---
+    getDailyLogs(start, end) {
+        function toIso(d) { return (d instanceof Date ? d : new Date(d)).toISOString().slice(0, 10); }
+        const startIso = formatDate ? formatDate(start) : toIso(start);
+        const endIso = formatDate ? formatDate(end) : toIso(end);
+
+        return dl.db.prepare(`
+            SELECT date, raw_markdown as content FROM daily_logs
+            WHERE date >= ? AND date <= ?
+            ORDER BY date ASC
+        `).all(startIso, endIso);
+    }
+
+    // --- RAG (Vector Search) ---
+    async semanticSearch(query, topK = 10) {
+        const queryVector = await defaultEmbedder.embedText(query);
+
+        // Fetch all stored embeddings. For a local personal tool with < 100k chunks, in-memory cosine sim is perfectly fast.
+        const rows = dl.db.prepare(`
+            SELECT reference_type, reference_id, chunk_index, text_chunk, embedding 
+            FROM document_embeddings
+        `).all();
+
+        const scored = [];
+        for (const row of rows) {
+            const chunkVector = defaultEmbedder.bufferToVector(row.embedding);
+            const score = defaultEmbedder.cosineSimilarity(queryVector, chunkVector);
+            scored.push({ ...row, score });
+        }
+
+        // Sort pure descending by similarity score
+        scored.sort((a, b) => b.score - a.score);
+
+        return scored.slice(0, topK);
+    }
+}
+
+module.exports = DataManager;

package/scripts/lib/Embedder.js

@@ -0,0 +1,59 @@
+const { pipeline } = require('@xenova/transformers');
+
+class Embedder {
+    constructor() {
+        this.modelName = 'Xenova/all-MiniLM-L6-v2';
+        this.extractorInfo = null;
+        this.initPromise = null;
+    }
+
+    async init() {
+        if (this.extractorInfo) return;
+        if (!this.initPromise) {
+            this.initPromise = pipeline('feature-extraction', this.modelName, { quantized: true })
+                .then((ex) => {
+                    this.extractorInfo = ex;
+                })
+                .catch((err) => {
+                    this.initPromise = null;
+                    throw err;
+                });
+        }
+        await this.initPromise;
+    }
+
+    async embedText(text) {
+        await this.init();
+        const cleanText = String(text || '').trim();
+        if (!cleanText) return new Float32Array(384); // Empty zero vector for model
+
+        const output = await this.extractorInfo(cleanText, { pooling: 'mean', normalize: true });
+        return new Float32Array(output.data);
+    }
+
+    // Prepares a Float32Array into a Node.js Buffer for SQLite BLOB storage
+    vectorToBuffer(float32Array) {
+        return Buffer.from(float32Array.buffer);
+    }
+
+    // Parses a SQLite BLOB Buffer back into a Float32Array
+    bufferToVector(buffer) {
+        return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / Float32Array.BYTES_PER_ELEMENT);
+    }
+
+    cosineSimilarity(vecA, vecB) {
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < vecA.length; i++) {
+            dotProduct += vecA[i] * vecB[i];
+            normA += vecA[i] * vecA[i];
+            normB += vecB[i] * vecB[i];
+        }
+        if (normA === 0 || normB === 0) return 0;
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+}
+
+const defaultEmbedder = new Embedder();
+module.exports = { Embedder, defaultEmbedder };

package/scripts/lib/schema.js

@@ -0,0 +1,23 @@
+/**
+ * schema.js
+ * Centralized schema constants for F.R.E.Y.A. data structures.
+ * This is the single source of truth for validation and ingestion.
+ */
+
+module.exports = {
+    TASK: {
+        CATEGORIES: ['DO_NOW', 'SCHEDULE', 'DELEGATE', 'IGNORE'],
+        STATUSES: ['PENDING', 'COMPLETED', 'ARCHIVED'],
+        PRIORITIES: ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']
+    },
+    BLOCKER: {
+        STATUSES: ['OPEN', 'MITIGATING', 'RESOLVED'],
+        SEVERITIES: ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']
+    },
+    CAREER: {
+        TYPES: ['Achievement', 'Feedback', 'Certification', 'Goal']
+    },
+    PROJECT: {
+        HISTORY_TYPES: ['Status', 'Decision', 'Risk', 'Achievement', 'Feedback', 'Goal', 'Blocker']
+    }
+};