@cccarv82/freya 2.3.13 → 2.4.1

package/templates/base/scripts/lib/DataManager.js

@@ -0,0 +1,198 @@
+/**
+ * DataManager.js (V2 - SQLite Powered)
+ * Centralized data access layer for F.R.E.Y.A. reports and scripts.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { isWithinRange, formatDate } = require('./date-utils');
+const SCHEMA = require('./schema');
+const { defaultInstance: dl } = require('./DataLayer');
+const { defaultEmbedder } = require('./Embedder');
+
+class DataManager {
+    constructor(dataDir, logsDir) {
+        // Keeping args for backwards compatibility, though SQLite handles the storage now
+        this.dataDir = dataDir;
+        this.logsDir = logsDir;
+    }
+
+    // --- Helpers (Deprecated but kept for backwards compat if needed elsewhere) ---
+    readJsonOrQuarantine(filePath, fallback) {
+        return fallback;
+    }
+
+    // --- Tasks ---
+    getTasksRaw() {
+        return dl.db.prepare('SELECT * FROM tasks').all().map(t => {
+            const meta = t.metadata ? JSON.parse(t.metadata) : {};
+            return {
+                id: t.id,
+                projectSlug: t.project_slug,
+                description: t.description,
+                category: t.category,
+                status: t.status,
+                createdAt: t.created_at,
+                completedAt: t.completed_at,
+                priority: meta.priority,
+                streamSlug: meta.streamSlug
+            };
+        });
+    }
+
+    getTasks(start, end) {
+        // SQLite date func: use ISO format
+        const toIso = (d) => d ? (d instanceof Date ? d.toISOString() : new Date(d).toISOString()) : null;
+        const startIso = toIso(start);
+        const endIso = toIso(end);
+
+        const tasks = this.getTasksRaw();
+        const completed = dl.db.prepare(`
+            SELECT * FROM tasks 
+            WHERE status = 'COMPLETED' 
+            AND completed_at >= ? AND completed_at <= ?
+        `).all(startIso, endIso).map(t => ({ ...t, completedAt: t.completed_at, createdAt: t.created_at }));
+
+        const pending = dl.db.prepare(`
+            SELECT * FROM tasks 
+            WHERE status = 'PENDING' AND category = 'DO_NOW'
+        `).all().map(t => ({ ...t, createdAt: t.created_at }));
+
+        return { tasks, completed, pending };
+    }
+
+    // --- Blockers ---
+    getBlockersRaw() {
+        return dl.db.prepare('SELECT * FROM blockers').all().map(b => {
+            const meta = b.metadata ? JSON.parse(b.metadata) : {};
+            return {
+                id: b.id,
+                projectSlug: b.project_slug,
+                title: b.title,
+                severity: b.severity,
+                status: b.status,
+                owner: b.owner,
+                nextAction: b.next_action,
+                createdAt: b.created_at,
+                resolvedAt: b.resolved_at,
+                streamSlug: meta.streamSlug
+            };
+        });
+    }
+
+    getBlockers(start, end) {
+        const toIso = (d) => d ? (d instanceof Date ? d.toISOString() : new Date(d).toISOString()) : null;
+        const startIso = toIso(start);
+        const endIso = toIso(end);
+
+        const blockers = this.getBlockersRaw();
+
+        const open = dl.db.prepare(`
+            SELECT * FROM blockers 
+            WHERE status NOT IN ('RESOLVED', 'CLOSED', 'DONE', 'FIXED')
+            AND resolved_at IS NULL
+            ORDER BY 
+              CASE severity 
+                WHEN 'CRITICAL' THEN 0 
+                WHEN 'HIGH' THEN 1 
+                WHEN 'MEDIUM' THEN 2 
+                WHEN 'LOW' THEN 3 
+                ELSE 99 
+              END ASC,
+              created_at ASC
+        `).all().map(b => ({ ...b, projectSlug: b.project_slug, createdAt: b.created_at }));
+
+        const openedRecent = dl.db.prepare(`
+            SELECT * FROM blockers 
+            WHERE created_at >= ? AND created_at <= ?
+        `).all(startIso, endIso).map(b => ({ ...b, projectSlug: b.project_slug, createdAt: b.created_at }));
+
+        const resolvedRecent = dl.db.prepare(`
+            SELECT * FROM blockers 
+            WHERE resolved_at >= ? AND resolved_at <= ?
+        `).all(startIso, endIso).map(b => ({ ...b, projectSlug: b.project_slug, resolvedAt: b.resolved_at }));
+
+        return { blockers, open, openedRecent, resolvedRecent };
+    }
+
+    // --- Project Updates ---
+    getProjectUpdates(start, end) {
+        const toIso = (d) => d ? (d instanceof Date ? d.toISOString() : new Date(d).toISOString()) : null;
+        const startIso = toIso(start);
+        const endIso = toIso(end);
+
+        const updates = [];
+
+        // Find active projects with recent history
+        const activeProjects = dl.db.prepare(`SELECT * FROM projects WHERE is_active = 1`).all();
+
+        for (const proj of activeProjects) {
+            const recentHistory = dl.db.prepare(`
+                SELECT status_text, date FROM project_status_history
+                WHERE project_id = ? AND date >= ? AND date <= ?
+                ORDER BY date DESC
+            `).all(proj.id, startIso, endIso).map(h => ({
+                content: h.status_text,
+                date: h.date
+            }));
+
+            // We determine currentStatus as the latest from history, or a placeholder if none
+            const latest = dl.db.prepare(`
+                SELECT status_text FROM project_status_history
+                WHERE project_id = ? ORDER BY date DESC LIMIT 1
+            `).get(proj.id);
+
+            const currentStatus = latest ? latest.status_text : "Initialized";
+
+            if (recentHistory.length > 0) {
+                updates.push({
+                    client: proj.client,
+                    project: proj.name,
+                    slug: proj.slug,
+                    currentStatus: currentStatus,
+                    events: recentHistory
+                });
+            }
+        }
+
+        return updates;
+    }
+
+    // --- Daily Logs ---
+    getDailyLogs(start, end) {
+        function toIso(d) { return (d instanceof Date ? d : new Date(d)).toISOString().slice(0, 10); }
+        const startIso = formatDate ? formatDate(start) : toIso(start);
+        const endIso = formatDate ? formatDate(end) : toIso(end);
+
+        return dl.db.prepare(`
+            SELECT date, raw_markdown as content FROM daily_logs
+            WHERE date >= ? AND date <= ?
+            ORDER BY date ASC
+        `).all(startIso, endIso);
+    }
+
+    // --- RAG (Vector Search) ---
+    async semanticSearch(query, topK = 10) {
+        const queryVector = await defaultEmbedder.embedText(query);
+
+        // Fetch all stored embeddings. For a local personal tool with < 100k chunks, in-memory cosine sim is perfectly fast.
+        const rows = dl.db.prepare(`
+            SELECT reference_type, reference_id, chunk_index, text_chunk, embedding 
+            FROM document_embeddings
+        `).all();
+
+        const scored = [];
+        for (const row of rows) {
+            const chunkVector = defaultEmbedder.bufferToVector(row.embedding);
+            const score = defaultEmbedder.cosineSimilarity(queryVector, chunkVector);
+            scored.push({ ...row, score });
+        }
+
+        // Sort pure descending by similarity score
+        scored.sort((a, b) => b.score - a.score);
+
+        return scored.slice(0, topK);
+    }
+}
+
+module.exports = DataManager;

package/templates/base/scripts/lib/Embedder.js

@@ -0,0 +1,59 @@
+const { pipeline } = require('@xenova/transformers');
+
+class Embedder {
+    constructor() {
+        this.modelName = 'Xenova/all-MiniLM-L6-v2';
+        this.extractorInfo = null;
+        this.initPromise = null;
+    }
+
+    async init() {
+        if (this.extractorInfo) return;
+        if (!this.initPromise) {
+            this.initPromise = pipeline('feature-extraction', this.modelName, { quantized: true })
+                .then((ex) => {
+                    this.extractorInfo = ex;
+                })
+                .catch((err) => {
+                    this.initPromise = null;
+                    throw err;
+                });
+        }
+        await this.initPromise;
+    }
+
+    async embedText(text) {
+        await this.init();
+        const cleanText = String(text || '').trim();
+        if (!cleanText) return new Float32Array(384); // Empty zero vector for model
+
+        const output = await this.extractorInfo(cleanText, { pooling: 'mean', normalize: true });
+        return new Float32Array(output.data);
+    }
+
+    // Prepares a Float32Array into a Node.js Buffer for SQLite BLOB storage
+    vectorToBuffer(float32Array) {
+        return Buffer.from(float32Array.buffer);
+    }
+
+    // Parses a SQLite BLOB Buffer back into a Float32Array
+    bufferToVector(buffer) {
+        return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / Float32Array.BYTES_PER_ELEMENT);
+    }
+
+    cosineSimilarity(vecA, vecB) {
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < vecA.length; i++) {
+            dotProduct += vecA[i] * vecB[i];
+            normA += vecA[i] * vecA[i];
+            normB += vecB[i] * vecB[i];
+        }
+        if (normA === 0 || normB === 0) return 0;
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+}
+
+const defaultEmbedder = new Embedder();
+module.exports = { Embedder, defaultEmbedder };

package/templates/base/scripts/lib/index-utils.js

@@ -0,0 +1,407 @@
+const fs = require('fs');
+const path = require('path');
+
+const ID_PATTERNS = [
+  /\bPTI\d{4,}-\d+\b/gi,
+  /\bINC\d+\b/gi,
+  /\bCHG\d+\b/gi
+];
+
+const TEXT_EXTS = new Set(['.md', '.txt', '.log', '.json', '.yaml', '.yml']);
+const TOKEN_RE = /[A-Za-z0-9_-]{3,}/g;
+
+const DEFAULT_MAX_SIZE = 2 * 1024 * 1024;
+const DEFAULT_TOKEN_LIMIT = 500;
+
+function extractIdTokens(query) {
+  const tokens = new Set();
+  const q = String(query || '');
+  for (const re of ID_PATTERNS) {
+    const matches = q.match(re);
+    if (matches) {
+      for (const m of matches) tokens.add(m.toUpperCase());
+    }
+  }
+  return Array.from(tokens);
+}
+
+function tokenizeQuery(query) {
+  const tokens = [];
+  const q = String(query || '');
+  const re = /[A-Za-z0-9_-]{2,}/g;
+  let m;
+  while ((m = re.exec(q)) !== null) {
+    tokens.push(m[0]);
+  }
+  return tokens;
+}
+
+function listFilesRecursive(dir, files = []) {
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return files;
+  }
+  for (const ent of entries) {
+    const full = path.join(dir, ent.name);
+    if (ent.isDirectory()) {
+      listFilesRecursive(full, files);
+    } else if (ent.isFile()) {
+      const ext = path.extname(ent.name).toLowerCase();
+      if (TEXT_EXTS.has(ext)) files.push(full);
+    }
+  }
+  return files;
+}
+
+function toDateString(ms) {
+  try {
+    const d = new Date(ms);
+    if (Number.isNaN(d.getTime())) return '';
+    const y = String(d.getFullYear());
+    const m = String(d.getMonth() + 1).padStart(2, '0');
+    const day = String(d.getDate()).padStart(2, '0');
+    return `${y}-${m}-${day}`;
+  } catch {
+    return '';
+  }
+}
+
+function inferDateFromPath(filePath, mtimeMs) {
+  const m = String(filePath).match(/\b(\d{4}-\d{2}-\d{2})\b/);
+  if (m && m[1]) return m[1];
+  return toDateString(mtimeMs);
+}
+
+function buildSnippet(text, index, length) {
+  if (index < 0) {
+    const clean = String(text || '').replace(/\s+/g, ' ').trim();
+    return clean.length > 220 ? clean.slice(0, 220) + '…' : clean;
+  }
+  const raw = String(text || '');
+  const start = Math.max(0, index - 80);
+  const end = Math.min(raw.length, index + length + 120);
+  let snippet = raw.slice(start, end).replace(/\s+/g, ' ').trim();
+  if (start > 0) snippet = '…' + snippet;
+  if (end < raw.length) snippet = snippet + '…';
+  return snippet;
+}
+
+function ensureDir(dir) {
+  fs.mkdirSync(dir, { recursive: true });
+}
+
+function normalizeRelPath(workspaceDir, fullPath) {
+  return path.relative(workspaceDir, fullPath).replace(/\\/g, '/');
+}
+
+function indexPathFor(workspaceDir) {
+  return path.join(workspaceDir, 'data', 'index', 'search-index.json');
+}
+
+function listTargetFiles(workspaceDir) {
+  const targetDirs = [
+    path.join(workspaceDir, 'logs', 'daily'),
+    path.join(workspaceDir, 'data', 'tasks'),
+    path.join(workspaceDir, 'data', 'Clients'),
+    path.join(workspaceDir, 'docs', 'reports')
+  ];
+
+  const files = [];
+  for (const dir of targetDirs) {
+    if (!fs.existsSync(dir)) continue;
+    const list = listFilesRecursive(dir, []);
+    for (const file of list) {
+      try {
+        const st = fs.statSync(file);
+        if (!st.isFile()) continue;
+        files.push({ file, mtimeMs: st.mtimeMs });
+      } catch {
+        // ignore
+      }
+    }
+  }
+  return files;
+}
+
+function extractIdMatches(text) {
+  const out = [];
+  for (const re of ID_PATTERNS) {
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      out.push({
+        key: String(m[0]).toUpperCase(),
+        index: m.index,
+        length: String(m[0]).length
+      });
+    }
+    re.lastIndex = 0;
+  }
+  return out;
+}
+
+function extractKeywordIndexMap(textLower, tokenLimit) {
+  const map = new Map();
+  let m;
+  while ((m = TOKEN_RE.exec(textLower)) !== null) {
+    const key = m[0];
+    if (!map.has(key)) {
+      map.set(key, m.index);
+      if (map.size >= tokenLimit) break;
+    }
+  }
+  TOKEN_RE.lastIndex = 0;
+  return map;
+}
+
+function addEntry(entriesMap, key, relPath, date, snippet) {
+  if (!entriesMap.has(key)) entriesMap.set(key, new Map());
+  const fileMap = entriesMap.get(key);
+  if (!fileMap.has(relPath)) {
+    fileMap.set(relPath, { path: relPath, date, snippet });
+  }
+}
+
+function removeFileFromEntries(entriesMap, relPath) {
+  for (const [key, fileMap] of entriesMap.entries()) {
+    if (fileMap.delete(relPath)) {
+      if (fileMap.size === 0) entriesMap.delete(key);
+    }
+  }
+}
+
+function indexSingleFile(workspaceDir, file, mtimeMs, opts, entriesMap) {
+  const maxSize = Math.max(1024, Number(opts.maxSize || DEFAULT_MAX_SIZE));
+  const tokenLimit = Math.max(50, Number(opts.tokenLimit || DEFAULT_TOKEN_LIMIT));
+
+  let st;
+  try {
+    st = fs.statSync(file);
+  } catch {
+    return;
+  }
+  if (!st.isFile() || st.size > maxSize) return;
+
+  let text;
+  try {
+    text = fs.readFileSync(file, 'utf8');
+  } catch {
+    return;
+  }
+  if (!text || text.includes('\u0000')) return;
+
+  const relPath = normalizeRelPath(workspaceDir, file);
+  const date = inferDateFromPath(relPath, mtimeMs);
+
+  const idMatches = extractIdMatches(text);
+  for (const match of idMatches) {
+    const snippet = buildSnippet(text, match.index, match.length);
+    addEntry(entriesMap, match.key, relPath, date, snippet);
+  }
+
+  const textLower = text.toLowerCase();
+  const tokenMap = extractKeywordIndexMap(textLower, tokenLimit);
+  for (const [token, index] of tokenMap.entries()) {
+    if (!token) continue;
+    const snippet = buildSnippet(text, index, token.length);
+    addEntry(entriesMap, token, relPath, date, snippet);
+  }
+}
+
+function entriesToMap(entries) {
+  const map = new Map();
+  if (!Array.isArray(entries)) return map;
+  for (const entry of entries) {
+    if (!entry || typeof entry.key !== 'string' || !Array.isArray(entry.files)) continue;
+    const fileMap = new Map();
+    for (const f of entry.files) {
+      if (!f || typeof f.path !== 'string') continue;
+      fileMap.set(f.path, {
+        path: f.path,
+        date: typeof f.date === 'string' ? f.date : '',
+        snippet: typeof f.snippet === 'string' ? f.snippet : ''
+      });
+    }
+    if (fileMap.size) map.set(entry.key, fileMap);
+  }
+  return map;
+}
+
+function mapToEntries(entriesMap) {
+  const out = [];
+  const keys = Array.from(entriesMap.keys()).sort();
+  for (const key of keys) {
+    const fileMap = entriesMap.get(key);
+    const files = Array.from(fileMap.values());
+    out.push({ key, files });
+  }
+  return out;
+}
+
+function readIndex(indexPath) {
+  try {
+    if (!fs.existsSync(indexPath)) return null;
+    const raw = fs.readFileSync(indexPath, 'utf8');
+    const json = JSON.parse(raw);
+    if (!json || typeof json !== 'object') return null;
+    return json;
+  } catch {
+    return null;
+  }
+}
+
+function buildIndex(workspaceDir, opts = {}) {
+  const entriesMap = new Map();
+  const files = listTargetFiles(workspaceDir);
+
+  for (const { file, mtimeMs } of files) {
+    indexSingleFile(workspaceDir, file, mtimeMs, opts, entriesMap);
+  }
+
+  const metaFiles = {};
+  for (const { file, mtimeMs } of files) {
+    const relPath = normalizeRelPath(workspaceDir, file);
+    metaFiles[relPath] = mtimeMs;
+  }
+
+  const index = {
+    meta: {
+      lastRun: new Date().toISOString(),
+      files: metaFiles
+    },
+    entries: mapToEntries(entriesMap)
+  };
+
+  const indexPath = indexPathFor(workspaceDir);
+  ensureDir(path.dirname(indexPath));
+  fs.writeFileSync(indexPath, JSON.stringify(index, null, 2) + '\n', 'utf8');
+
+  return { indexPath, fileCount: files.length, keyCount: index.entries.length };
+}
+
+function updateIndex(workspaceDir, opts = {}) {
+  const indexPath = indexPathFor(workspaceDir);
+  const currentFiles = listTargetFiles(workspaceDir);
+  const currentMap = new Map();
+  for (const { file, mtimeMs } of currentFiles) {
+    const relPath = normalizeRelPath(workspaceDir, file);
+    currentMap.set(relPath, { file, mtimeMs });
+  }
+
+  const existing = readIndex(indexPath);
+  if (!existing || !existing.meta || !existing.meta.files || !Array.isArray(existing.entries)) {
+    return buildIndex(workspaceDir, opts);
+  }
+
+  const entriesMap = entriesToMap(existing.entries);
+  const prevFiles = existing.meta.files || {};
+
+  const removed = new Set();
+  for (const relPath of Object.keys(prevFiles)) {
+    if (!currentMap.has(relPath)) removed.add(relPath);
+  }
+
+  const changed = [];
+  for (const [relPath, info] of currentMap.entries()) {
+    const prev = prevFiles[relPath];
+    if (!prev || Number(prev) !== Number(info.mtimeMs)) {
+      changed.push(info);
+    }
+  }
+
+  for (const relPath of removed) {
+    removeFileFromEntries(entriesMap, relPath);
+  }
+
+  for (const info of changed) {
+    const relPath = normalizeRelPath(workspaceDir, info.file);
+    removeFileFromEntries(entriesMap, relPath);
+    indexSingleFile(workspaceDir, info.file, info.mtimeMs, opts, entriesMap);
+  }
+
+  const metaFiles = {};
+  for (const [relPath, info] of currentMap.entries()) {
+    metaFiles[relPath] = info.mtimeMs;
+  }
+
+  const index = {
+    meta: {
+      lastRun: new Date().toISOString(),
+      files: metaFiles
+    },
+    entries: mapToEntries(entriesMap)
+  };
+
+  ensureDir(path.dirname(indexPath));
+  fs.writeFileSync(indexPath, JSON.stringify(index, null, 2) + '\n', 'utf8');
+
+  return { indexPath, fileCount: currentFiles.length, keyCount: index.entries.length, changed: changed.length, removed: removed.size };
+}
+
+function searchIndex(workspaceDir, query, opts = {}) {
+  const q = String(query || '').trim();
+  if (!q) return [];
+  const limit = Math.max(1, Math.min(20, Number(opts.limit || 8)));
+
+  const indexPath = indexPathFor(workspaceDir);
+  const index = readIndex(indexPath);
+  if (!index || !Array.isArray(index.entries)) return [];
+
+  const entriesMap = new Map();
+  for (const entry of index.entries) {
+    if (!entry || typeof entry.key !== 'string' || !Array.isArray(entry.files)) continue;
+    entriesMap.set(entry.key, entry.files);
+  }
+
+  const idTokens = extractIdTokens(q);
+  const tokens = tokenizeQuery(q).map((t) => t.toLowerCase());
+  const queryLower = q.toLowerCase();
+
+  const resultsMap = new Map();
+
+  function applyMatches(keys, weight) {
+    for (const key of keys) {
+      if (!key) continue;
+      const entryFiles = entriesMap.get(key);
+      if (!entryFiles) continue;
+      for (const f of entryFiles) {
+        if (!f || !f.path) continue;
+        const prev = resultsMap.get(f.path) || { file: f.path, date: f.date || '', score: 0, snippet: '' , weight: 0 };
+        const bonus = (key === queryLower) ? 10 : 0;
+        const nextScore = prev.score + weight + bonus;
+        const nextWeight = Math.max(prev.weight, weight);
+        const snippet = (nextWeight > prev.weight && f.snippet) ? f.snippet : (prev.snippet || f.snippet || '');
+        resultsMap.set(f.path, {
+          file: f.path,
+          date: f.date || prev.date || '',
+          score: nextScore,
+          snippet,
+          weight: nextWeight
+        });
+      }
+    }
+  }
+
+  applyMatches(idTokens.map((t) => t.toUpperCase()), 100);
+  applyMatches(tokens, 2);
+
+  const results = Array.from(resultsMap.values()).map((r) => {
+    const { weight, ...rest } = r;
+    return rest;
+  });
+
+  results.sort((a, b) => {
+    if (b.score !== a.score) return b.score - a.score;
+    return String(b.date || '').localeCompare(String(a.date || ''));
+  });
+
+  return results.slice(0, limit);
+}
+
+module.exports = {
+  buildIndex,
+  updateIndex,
+  searchIndex,
+  indexPathFor
+};

package/templates/base/scripts/lib/schema.js

@@ -0,0 +1,23 @@
+/**
+ * schema.js
+ * Centralized schema constants for F.R.E.Y.A. data structures.
+ * This is the single source of truth for validation and ingestion.
+ */
+
+module.exports = {
+    TASK: {
+        CATEGORIES: ['DO_NOW', 'SCHEDULE', 'DELEGATE', 'IGNORE'],
+        STATUSES: ['PENDING', 'COMPLETED', 'ARCHIVED'],
+        PRIORITIES: ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']
+    },
+    BLOCKER: {
+        STATUSES: ['OPEN', 'MITIGATING', 'RESOLVED'],
+        SEVERITIES: ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']
+    },
+    CAREER: {
+        TYPES: ['Achievement', 'Feedback', 'Certification', 'Goal']
+    },
+    PROJECT: {
+        HISTORY_TYPES: ['Status', 'Decision', 'Risk', 'Achievement', 'Feedback', 'Goal', 'Blocker']
+    }
+};