@stackmemoryai/stackmemory 1.10.5 → 1.14.0

package/dist/src/daemon/services/research-stream-service.js

@@ -0,0 +1,320 @@
+import { fileURLToPath as __fileURLToPath } from 'url';
+import { dirname as __pathDirname } from 'path';
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  appendFileSync,
+  mkdirSync
+} from "fs";
+import { join } from "path";
+import { homedir } from "os";
+const SM_DIR = join(homedir(), ".stackmemory");
+const DP_DIR = join(SM_DIR, "desire-paths");
+const STREAM_FILE = join(DP_DIR, "research-stream.jsonl");
+const DIGEST_FILE = join(DP_DIR, "research-digest.json");
+const HN_TOP_URL = "https://hacker-news.firebaseio.com/v0/topstories.json";
+const HN_ITEM_URL = "https://hacker-news.firebaseio.com/v0/item";
+const GH_SEARCH_URL = "https://api.github.com/search/repositories";
+const RATE_LIMIT_MS = 1100;
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isoWeek(date) {
+  const d = new Date(date.getTime());
+  d.setHours(0, 0, 0, 0);
+  d.setDate(d.getDate() + 3 - (d.getDay() + 6) % 7);
+  const week1 = new Date(d.getFullYear(), 0, 4);
+  const weekNum = 1 + Math.round(
+    ((d.getTime() - week1.getTime()) / 864e5 - 3 + (week1.getDay() + 6) % 7) / 7
+  );
+  return `${d.getFullYear()}-W${String(weekNum).padStart(2, "0")}`;
+}
+function scoreRelevance(title, keywords) {
+  const lower = title.toLowerCase();
+  const matched = [];
+  for (const kw of keywords) {
+    if (lower.includes(kw.toLowerCase())) {
+      matched.push(kw);
+    }
+  }
+  if (matched.length === 0) return { score: 0, matched: [] };
+  const score = Math.min(1, 0.3 + matched.length * 0.2);
+  return { score, matched };
+}
+async function safeFetch(url, timeoutMs = 1e4) {
+  try {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const res = await fetch(url, {
+      signal: controller.signal,
+      headers: { "User-Agent": "StackMemory-ResearchStream/1.0" }
+    });
+    clearTimeout(timer);
+    if (!res.ok) return null;
+    return await res.json();
+  } catch {
+    return null;
+  }
+}
+class ResearchStreamService {
+  config;
+  state;
+  intervalId;
+  isRunning = false;
+  onLog;
+  constructor(config, onLog) {
+    this.config = config;
+    this.onLog = onLog;
+    this.state = {
+      lastScanTime: 0,
+      signalsCollected: 0,
+      digestsGenerated: 0,
+      errors: []
+    };
+  }
+  isOptedOut() {
+    if (process.env.STACKMEMORY_RESEARCH_STREAM === "0" || process.env.STACKMEMORY_RESEARCH_STREAM === "false") {
+      return true;
+    }
+    return !this.config.enabled;
+  }
+  // ─── Source: Hacker News ──────────────────────────────────
+  async fetchHackerNews() {
+    const signals = [];
+    const topIds = await safeFetch(HN_TOP_URL);
+    if (!topIds || !Array.isArray(topIds)) {
+      this.onLog("WARN", "HN top stories fetch failed");
+      return signals;
+    }
+    const ids = topIds.slice(0, 10);
+    for (const id of ids) {
+      await sleep(200);
+      const item = await safeFetch(`${HN_ITEM_URL}/${id}.json`);
+      if (!item || !item.title) continue;
+      const { score: relevance, matched } = scoreRelevance(
+        item.title,
+        this.config.keywords
+      );
+      if (relevance === 0) continue;
+      signals.push({
+        ts: (/* @__PURE__ */ new Date()).toISOString(),
+        source: "hackernews",
+        type: "trending",
+        title: item.title,
+        url: item.url || `https://news.ycombinator.com/item?id=${id}`,
+        score: item.score || 0,
+        keywords_matched: matched,
+        relevance
+      });
+    }
+    return signals;
+  }
+  // ─── Source: GitHub Trending ───────────────────────────────
+  async fetchGitHubTrending() {
+    const signals = [];
+    const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1e3);
+    const dateStr = sevenDaysAgo.toISOString().split("T")[0];
+    const url = `${GH_SEARCH_URL}?q=created:>${dateStr}&sort=stars&order=desc&per_page=10`;
+    await sleep(RATE_LIMIT_MS);
+    const data = await safeFetch(url);
+    if (!data || !data.items) {
+      this.onLog("WARN", "GitHub trending fetch failed");
+      return signals;
+    }
+    for (const repo of data.items) {
+      const text = `${repo.full_name || ""} ${repo.description || ""}`;
+      const { score: relevance, matched } = scoreRelevance(
+        text,
+        this.config.keywords
+      );
+      if (relevance === 0) continue;
+      signals.push({
+        ts: (/* @__PURE__ */ new Date()).toISOString(),
+        source: "github",
+        type: "new_repo",
+        title: `${repo.full_name}: ${(repo.description || "").slice(0, 120)}`,
+        url: repo.html_url || "",
+        score: repo.stargazers_count || 0,
+        keywords_matched: matched,
+        relevance
+      });
+    }
+    return signals;
+  }
+  // ─── Source: Product Hunt (placeholder) ────────────────────
+  async fetchProductHunt() {
+    this.onLog("DEBUG", "Product Hunt source unavailable (no API key)");
+    return [];
+  }
+  // ─── Core Scan ────────────────────────────────────────────
+  async runScan() {
+    try {
+      mkdirSync(DP_DIR, { recursive: true });
+      const [hnSignals, ghSignals, phSignals] = await Promise.all([
+        this.fetchHackerNews(),
+        this.fetchGitHubTrending(),
+        this.fetchProductHunt()
+      ]);
+      const allSignals = [...hnSignals, ...ghSignals, ...phSignals];
+      allSignals.sort((a, b) => b.relevance - a.relevance || b.score - a.score);
+      const capped = allSignals.slice(0, this.config.maxSignalsPerScan);
+      const existingUrls = this.loadExistingUrls();
+      const newSignals = capped.filter((s) => !existingUrls.has(s.url));
+      if (newSignals.length > 0) {
+        const lines = newSignals.map((s) => JSON.stringify(s)).join("\n") + "\n";
+        appendFileSync(STREAM_FILE, lines, "utf-8");
+        this.state.signalsCollected += newSignals.length;
+      }
+      this.state.lastScanTime = Date.now();
+      this.onLog("INFO", "Research scan complete", {
+        hn: hnSignals.length,
+        gh: ghSignals.length,
+        ph: phSignals.length,
+        new: newSignals.length,
+        total: this.state.signalsCollected
+      });
+      this.updateDigest();
+    } catch (err) {
+      this.addError(String(err));
+      this.onLog("ERROR", "Research scan failed", { error: String(err) });
+    }
+  }
+  /** Load existing URLs from the stream file for dedup. */
+  loadExistingUrls() {
+    const urls = /* @__PURE__ */ new Set();
+    try {
+      if (!existsSync(STREAM_FILE)) return urls;
+      const lines = readFileSync(STREAM_FILE, "utf-8").trim().split("\n");
+      for (const line of lines) {
+        try {
+          const entry = JSON.parse(line);
+          if (entry.url) urls.add(entry.url);
+        } catch {
+        }
+      }
+    } catch {
+    }
+    return urls;
+  }
+  // ─── Weekly Digest ────────────────────────────────────────
+  updateDigest() {
+    try {
+      if (!existsSync(STREAM_FILE)) return;
+      const lines = readFileSync(STREAM_FILE, "utf-8").trim().split("\n");
+      const now = /* @__PURE__ */ new Date();
+      const currentWeek = isoWeek(now);
+      const weekStart = Date.now() - 7 * 24 * 60 * 60 * 1e3;
+      const weekSignals = [];
+      for (const line of lines) {
+        try {
+          const entry = JSON.parse(line);
+          if (new Date(entry.ts).getTime() >= weekStart) {
+            weekSignals.push(entry);
+          }
+        } catch {
+        }
+      }
+      if (weekSignals.length === 0) return;
+      weekSignals.sort(
+        (a, b) => b.relevance - a.relevance || b.score - a.score
+      );
+      const topSignals = weekSignals.slice(0, 20);
+      const keywordCounts = /* @__PURE__ */ new Map();
+      for (const signal of topSignals) {
+        for (const kw of signal.keywords_matched) {
+          keywordCounts.set(kw, (keywordCounts.get(kw) || 0) + 1);
+        }
+      }
+      const themes = [...keywordCounts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5).map(([kw, count]) => `${kw} (${count} signals)`);
+      const digest = {
+        week: currentWeek,
+        signals: topSignals,
+        themes,
+        generated_at: now.toISOString()
+      };
+      writeFileSync(DIGEST_FILE, JSON.stringify(digest, null, 2), "utf-8");
+      this.state.digestsGenerated++;
+      this.onLog("INFO", "Research digest updated", {
+        week: currentWeek,
+        signals: topSignals.length,
+        themes: themes.length
+      });
+    } catch (err) {
+      this.addError(String(err));
+    }
+  }
+  // ─── Lifecycle ────────────────────────────────────────────
+  start() {
+    if (this.isRunning || this.isOptedOut()) {
+      if (this.isOptedOut()) {
+        this.onLog("INFO", "Research stream disabled");
+      }
+      return;
+    }
+    this.isRunning = true;
+    mkdirSync(DP_DIR, { recursive: true });
+    const intervalMs = (this.config.interval || 360) * 60 * 1e3;
+    this.onLog("INFO", "Research stream service started", {
+      interval_min: this.config.interval,
+      keywords: this.config.keywords.length
+    });
+    setTimeout(() => {
+      if (!this.isRunning) return;
+      this.runScan();
+    }, 6e4);
+    this.intervalId = setInterval(() => {
+      if (!this.isRunning) return;
+      this.runScan();
+    }, intervalMs);
+    if (this.intervalId.unref) this.intervalId.unref();
+  }
+  stop() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = void 0;
+    }
+    this.isRunning = false;
+  }
+  getState() {
+    return { ...this.state };
+  }
+  /** Manually trigger a scan (for CLI/MCP). */
+  async triggerScan() {
+    const before = this.state.signalsCollected;
+    await this.runScan();
+    try {
+      if (!existsSync(STREAM_FILE)) return [];
+      const lines = readFileSync(STREAM_FILE, "utf-8").trim().split("\n");
+      return lines.slice(-(this.state.signalsCollected - before)).map((l) => {
+        try {
+          return JSON.parse(l);
+        } catch {
+          return null;
+        }
+      }).filter(Boolean);
+    } catch {
+      return [];
+    }
+  }
+  /** Get the latest digest. */
+  getDigest() {
+    try {
+      if (!existsSync(DIGEST_FILE)) return null;
+      return JSON.parse(readFileSync(DIGEST_FILE, "utf-8"));
+    } catch {
+      return null;
+    }
+  }
+  addError(err) {
+    this.state.errors.push(err);
+    if (this.state.errors.length > 10) {
+      this.state.errors = this.state.errors.slice(-10);
+    }
+  }
+}
+export {
+  ResearchStreamService
+};

package/dist/src/daemon/services/telemetry-service.js

@@ -0,0 +1,192 @@
+import { fileURLToPath as __fileURLToPath } from 'url';
+import { dirname as __pathDirname } from 'path';
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  readdirSync,
+  statSync,
+  mkdirSync
+} from "fs";
+import { join, dirname } from "path";
+import { homedir, platform } from "os";
+import { randomBytes } from "crypto";
+const SM_DIR = join(homedir(), ".stackmemory");
+const INSTANCE_ID_FILE = join(SM_DIR, "instance-id");
+const TELEMETRY_FILE = join(SM_DIR, "telemetry.json");
+const SESSIONS_DIR = join(SM_DIR, "sessions");
+const STALE_MS = 10 * 60 * 1e3;
+class DaemonTelemetryService {
+  config;
+  state;
+  intervalId;
+  isRunning = false;
+  onLog;
+  getDaemonState;
+  constructor(config, onLog, getDaemonState) {
+    this.config = config;
+    this.onLog = onLog;
+    this.getDaemonState = getDaemonState;
+    this.state = { lastSnapshotTime: 0, snapshotCount: 0, errors: [] };
+  }
+  isOptedOut() {
+    if (process.env.STACKMEMORY_TELEMETRY === "0" || process.env.STACKMEMORY_TELEMETRY === "false") {
+      return true;
+    }
+    return !this.config.enabled;
+  }
+  getInstanceId() {
+    try {
+      if (existsSync(INSTANCE_ID_FILE)) {
+        return readFileSync(INSTANCE_ID_FILE, "utf-8").trim();
+      }
+    } catch {
+    }
+    const id = randomBytes(16).toString("hex");
+    try {
+      writeFileSync(INSTANCE_ID_FILE, id, "utf-8");
+    } catch {
+    }
+    return id;
+  }
+  countSessions() {
+    try {
+      if (!existsSync(SESSIONS_DIR))
+        return { total_heartbeats: 0, active_now: 0 };
+      const files = readdirSync(SESSIONS_DIR).filter(
+        (f) => f.endsWith(".heartbeat")
+      );
+      const now = Date.now();
+      let active = 0;
+      for (const file of files) {
+        try {
+          const stat = statSync(join(SESSIONS_DIR, file));
+          if (now - stat.mtimeMs < STALE_MS) active++;
+        } catch {
+        }
+      }
+      return { total_heartbeats: files.length, active_now: active };
+    } catch {
+      return { total_heartbeats: 0, active_now: 0 };
+    }
+  }
+  countSkillAudit() {
+    try {
+      const auditPath = join(SM_DIR, "skill-audit.jsonl");
+      if (!existsSync(auditPath)) return 0;
+      return readFileSync(auditPath, "utf-8").trim().split("\n").length;
+    } catch {
+      return 0;
+    }
+  }
+  countHandoffs() {
+    try {
+      const handoffsDir = join(SM_DIR, "handoffs");
+      if (!existsSync(handoffsDir)) return 0;
+      return readdirSync(handoffsDir).filter((f) => f.endsWith(".md")).length;
+    } catch {
+      return 0;
+    }
+  }
+  collect() {
+    if (this.isOptedOut()) return { opted_out: true };
+    const daemonState = this.getDaemonState?.();
+    const sessions = this.countSessions();
+    return {
+      instance_id: this.getInstanceId(),
+      collected_at: (/* @__PURE__ */ new Date()).toISOString(),
+      platform: platform(),
+      node_version: process.version,
+      daemon: daemonState ? {
+        uptime_s: Math.round((daemonState.uptime || 0) / 1e3),
+        context_saves: daemonState.services?.context?.saveCount || 0,
+        memory_triggers: daemonState.services?.memory?.triggerCount || 0,
+        ram_percent: Math.round(
+          (daemonState.services?.memory?.currentRamPercent || 0) * 100
+        ),
+        errors: (daemonState.errors || []).length
+      } : null,
+      sessions,
+      skills: { audit_entries: this.countSkillAudit() },
+      handoffs: { total: this.countHandoffs() }
+    };
+  }
+  save() {
+    const snapshot = this.collect();
+    if ("opted_out" in snapshot) return null;
+    let history = [];
+    try {
+      if (existsSync(TELEMETRY_FILE)) {
+        const data = JSON.parse(readFileSync(TELEMETRY_FILE, "utf-8"));
+        history = Array.isArray(data.snapshots) ? data.snapshots : [];
+      }
+    } catch {
+      history = [];
+    }
+    history.push(snapshot);
+    const max = this.config.maxSnapshots || 90;
+    if (history.length > max) history = history.slice(-max);
+    try {
+      const dir = dirname(TELEMETRY_FILE);
+      if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+      writeFileSync(
+        TELEMETRY_FILE,
+        JSON.stringify({ version: 1, snapshots: history }, null, 2),
+        "utf-8"
+      );
+    } catch (err) {
+      this.state.errors.push(String(err));
+      if (this.state.errors.length > 5)
+        this.state.errors = this.state.errors.slice(-5);
+      this.onLog("ERROR", "Failed to save telemetry", { error: String(err) });
+      return null;
+    }
+    this.state.lastSnapshotTime = Date.now();
+    this.state.snapshotCount++;
+    return snapshot;
+  }
+  start() {
+    if (this.isRunning || this.isOptedOut()) {
+      if (this.isOptedOut()) {
+        this.onLog("INFO", "Telemetry disabled \u2014 opt-out active");
+      }
+      return;
+    }
+    this.isRunning = true;
+    const intervalMs = (this.config.interval || 1440) * 60 * 1e3;
+    this.onLog("INFO", "Telemetry service started", {
+      interval_min: this.config.interval
+    });
+    setTimeout(() => {
+      if (!this.isRunning) return;
+      const snap = this.save();
+      if (snap)
+        this.onLog("INFO", "Telemetry snapshot saved", {
+          sessions: snap.sessions.active_now
+        });
+    }, 3e4);
+    this.intervalId = setInterval(() => {
+      const snap = this.save();
+      if (snap)
+        this.onLog("INFO", "Telemetry snapshot saved", {
+          sessions: snap.sessions.active_now
+        });
+    }, intervalMs);
+    if (this.intervalId.unref) this.intervalId.unref();
+  }
+  stop() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = void 0;
+    }
+    this.isRunning = false;
+  }
+  getState() {
+    return { ...this.state };
+  }
+}
+export {
+  DaemonTelemetryService
+};

package/dist/src/daemon/unified-daemon.js

@@ -18,15 +18,23 @@ import {
 } from "./daemon-config.js";
 import { DaemonContextService } from "./services/context-service.js";
 import { DaemonLinearService } from "./services/linear-service.js";
+import { DaemonGitHubService } from "./services/github-service.js";
 import { DaemonMaintenanceService } from "./services/maintenance-service.js";
 import { DaemonMemoryService } from "./services/memory-service.js";
+import { DaemonTelemetryService } from "./services/telemetry-service.js";
+import { DaemonDesirePathService } from "./services/desire-path-service.js";
+import { ResearchStreamService } from "./services/research-stream-service.js";
 class UnifiedDaemon {
   config;
   paths;
   contextService;
   linearService;
+  githubService;
   maintenanceService;
   memoryService;
+  telemetryService;
+  desirePathService;
+  researchStreamService;
   heartbeatInterval;
   isShuttingDown = false;
   startTime = 0;
@@ -41,6 +49,10 @@ class UnifiedDaemon {
       this.config.linear,
       (level, msg, data) => this.log(level, "linear", msg, data)
     );
+    this.githubService = new DaemonGitHubService(
+      this.config.github,
+      (level, msg, data) => this.log(level, "github", msg, data)
+    );
     this.maintenanceService = new DaemonMaintenanceService(
       this.config.maintenance,
       (level, msg, data) => this.log(level, "maintenance", msg, data)
@@ -49,6 +61,19 @@ class UnifiedDaemon {
       this.config.memory,
       (level, msg, data) => this.log(level, "memory", msg, data)
     );
+    this.telemetryService = new DaemonTelemetryService(
+      this.config.telemetry,
+      (level, msg, data) => this.log(level, "telemetry", msg, data),
+      () => this.getStatus()
+    );
+    this.desirePathService = new DaemonDesirePathService(
+      this.config.desirePaths,
+      (level, msg, data) => this.log(level, "desire-paths", msg, data)
+    );
+    this.researchStreamService = new ResearchStreamService(
+      this.config.researchStream,
+      (level, msg, data) => this.log(level, "research-stream", msg, data)
+    );
   }
   log(level, service, message, data) {
     const logLevel = level.toUpperCase();
@@ -101,6 +126,7 @@ class UnifiedDaemon {
   updateStatus() {
     const maintenanceState = this.maintenanceService.getState();
     const memoryState = this.memoryService.getState();
+    const githubState = this.githubService.getState();
     const status = {
       running: true,
       pid: process.pid,
@@ -117,6 +143,12 @@ class UnifiedDaemon {
           lastRun: this.linearService.getState().lastSyncTime || void 0,
           syncCount: this.linearService.getState().syncCount
         },
+        github: {
+          enabled: this.config.github.enabled,
+          lastRun: githubState.lastSyncTime || void 0,
+          syncCount: githubState.syncCount,
+          lastProjectionState: githubState.lastProjectionState
+        },
         maintenance: {
           enabled: this.config.maintenance.enabled,
           lastRun: maintenanceState.lastRunTime || void 0,
@@ -139,6 +171,7 @@ class UnifiedDaemon {
       errors: [
         ...this.contextService.getState().errors.slice(-5),
         ...this.linearService.getState().errors.slice(-5),
+        ...githubState.errors.slice(-5),
         ...maintenanceState.errors.slice(-5),
         ...memoryState.errors.slice(-5)
       ]
@@ -190,6 +223,11 @@ class UnifiedDaemon {
           enabled: false,
           syncCount: this.linearService.getState().syncCount
         },
+        github: {
+          enabled: false,
+          syncCount: this.githubService.getState().syncCount,
+          lastProjectionState: this.githubService.getState().lastProjectionState
+        },
         maintenance: {
           enabled: false,
           staleFramesCleaned: this.maintenanceService.getState().staleFramesCleaned,
@@ -213,8 +251,11 @@ class UnifiedDaemon {
       uptime: Date.now() - this.startTime,
       contextSaves: this.contextService.getState().saveCount,
       linearSyncs: this.linearService.getState().syncCount,
+      githubSyncs: this.githubService.getState().syncCount,
       maintenanceRuns: this.maintenanceService.getState().ftsRebuilds,
-      memoryTriggers: this.memoryService.getState().triggerCount
+      memoryTriggers: this.memoryService.getState().triggerCount,
+      telemetrySnapshots: this.telemetryService.getState().snapshotCount,
+      researchSignals: this.researchStreamService.getState().signalsCollected
     });
     if (this.heartbeatInterval) {
       clearInterval(this.heartbeatInterval);
@@ -222,8 +263,12 @@ class UnifiedDaemon {
     }
     this.contextService.stop();
     this.linearService.stop();
+    this.githubService.stop();
     this.maintenanceService.stop();
     this.memoryService.stop();
+    this.telemetryService.stop();
+    this.desirePathService.stop();
+    this.researchStreamService.stop();
     this.cleanup();
     const exitCode = reason === "sigterm" || reason === "sigint" || reason === "sighup" ? 0 : 1;
     process.exit(exitCode);
@@ -241,6 +286,7 @@ class UnifiedDaemon {
       config: {
         context: this.config.context.enabled,
         linear: this.config.linear.enabled,
+        github: this.config.github.enabled,
         maintenance: this.config.maintenance.enabled,
         memory: this.config.memory.enabled,
         fileWatch: this.config.fileWatch.enabled
@@ -248,8 +294,12 @@ class UnifiedDaemon {
     });
     this.contextService.start();
     await this.linearService.start();
+    await this.githubService.start();
     this.maintenanceService.start();
     this.memoryService.start();
+    this.telemetryService.start();
+    this.desirePathService.start();
+    this.researchStreamService.start();
     this.heartbeatInterval = setInterval(() => {
       this.updateStatus();
     }, this.config.heartbeatInterval * 1e3);
@@ -258,6 +308,7 @@ class UnifiedDaemon {
   getStatus() {
     const maintenanceState = this.maintenanceService.getState();
     const memoryState = this.memoryService.getState();
+    const githubState = this.githubService.getState();
     return {
       running: !this.isShuttingDown,
       pid: process.pid,
@@ -274,6 +325,12 @@ class UnifiedDaemon {
           lastRun: this.linearService.getState().lastSyncTime || void 0,
           syncCount: this.linearService.getState().syncCount
         },
+        github: {
+          enabled: this.config.github.enabled,
+          lastRun: githubState.lastSyncTime || void 0,
+          syncCount: githubState.syncCount,
+          lastProjectionState: githubState.lastProjectionState
+        },
         maintenance: {
           enabled: this.config.maintenance.enabled,
           lastRun: maintenanceState.lastRunTime || void 0,