@geravant/sinain 1.0.19 → 1.2.0

package/sinain-core/src/util/task-store.ts

@@ -0,0 +1,84 @@
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { log, error } from "../log.js";
+
+const TAG = "task-store";
+
+/**
+ * Persistent task entry for spawn tasks.
+ */
+export interface PendingTaskEntry {
+  runId: string;
+  childSessionKey: string;
+  label?: string;
+  startedAt: number;
+  pollingEmitted: boolean;
+}
+
+const STORE_DIR = path.join(os.homedir(), ".sinain-core");
+const STORE_PATH = path.join(STORE_DIR, "pending-tasks.json");
+
+/**
+ * Load pending tasks from disk.
+ * Returns empty map if file doesn't exist or is corrupted.
+ */
+export function loadPendingTasks(): Map<string, PendingTaskEntry> {
+  try {
+    if (!fs.existsSync(STORE_PATH)) {
+      return new Map();
+    }
+    const data = fs.readFileSync(STORE_PATH, "utf-8");
+    const parsed = JSON.parse(data);
+    if (!Array.isArray(parsed)) {
+      return new Map();
+    }
+    const map = new Map<string, PendingTaskEntry>();
+    for (const [key, value] of parsed) {
+      if (typeof key === "string" && value && typeof value.runId === "string") {
+        map.set(key, value as PendingTaskEntry);
+      }
+    }
+    log(TAG, `loaded ${map.size} pending task(s) from disk`);
+    return map;
+  } catch (err: any) {
+    error(TAG, `failed to load pending tasks: ${err.message}`);
+    return new Map();
+  }
+}
+
+/**
+ * Save pending tasks to disk atomically.
+ * Uses write-then-rename for crash safety.
+ */
+export function savePendingTasks(tasks: Map<string, PendingTaskEntry>): void {
+  try {
+    // Ensure directory exists
+    if (!fs.existsSync(STORE_DIR)) {
+      fs.mkdirSync(STORE_DIR, { recursive: true });
+    }
+
+    // Convert map to array of entries for JSON serialization
+    const entries = Array.from(tasks.entries());
+    const tmpPath = STORE_PATH + ".tmp";
+
+    fs.writeFileSync(tmpPath, JSON.stringify(entries, null, 2), "utf-8");
+    fs.renameSync(tmpPath, STORE_PATH);
+  } catch (err: any) {
+    error(TAG, `failed to save pending tasks: ${err.message}`);
+  }
+}
+
+/**
+ * Delete the task store file (cleanup).
+ */
+export function clearPendingTasks(): void {
+  try {
+    if (fs.existsSync(STORE_PATH)) {
+      fs.unlinkSync(STORE_PATH);
+      log(TAG, "cleared pending tasks file");
+    }
+  } catch (err: any) {
+    error(TAG, `failed to clear pending tasks: ${err.message}`);
+  }
+}

package/sinain-core/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "eval"]
+}

package/sinain-knowledge/curation/engine.ts

@@ -6,11 +6,37 @@
  * external process execution.
  */
 
+import { readFileSync, existsSync, writeFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+
 import type { Logger, ScriptRunner } from "../data/schema.js";
 import type { KnowledgeStore } from "../data/store.js";
 import type { ResilienceManager } from "./resilience.js";
 import type { GitSnapshotStore } from "../data/git-store.js";
 
+// ============================================================================
+// Distillation State
+// ============================================================================
+
+interface DistillState {
+  lastDistilledFeedId: number;
+  lastDistilledTs: string;
+}
+
+function readDistillState(workspaceDir: string): DistillState {
+  const p = join(workspaceDir, "memory", "distill-state.json");
+  try {
+    if (existsSync(p)) return JSON.parse(readFileSync(p, "utf-8"));
+  } catch {}
+  return { lastDistilledFeedId: 0, lastDistilledTs: "" };
+}
+
+function writeDistillState(workspaceDir: string, state: DistillState): void {
+  const dir = join(workspaceDir, "memory");
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  writeFileSync(join(dir, "distill-state.json"), JSON.stringify(state, null, 2), "utf-8");
+}
+
 // ============================================================================
 // Types
 // ============================================================================
@@ -43,14 +69,17 @@ export type ContextAssemblyOpts = {
 export class CurationEngine {
   private curationInterval: ReturnType<typeof setInterval> | null = null;
   private gitSnapshotStore: GitSnapshotStore | null = null;
+  private sinainCoreUrl: string;
 
   constructor(
     private store: KnowledgeStore,
     private runScript: ScriptRunner,
     private resilience: ResilienceManager,
-    private config: { userTimezone: string },
+    private config: { userTimezone: string; sinainCoreUrl?: string },
     private logger: Logger,
-  ) {}
+  ) {
+    this.sinainCoreUrl = config.sinainCoreUrl || process.env.SINAIN_CORE_URL || "http://localhost:9500";
+  }
 
   /** Attach a git-backed snapshot store for periodic saves. */
   setGitSnapshotStore(gitStore: GitSnapshotStore): void {
@@ -143,15 +172,87 @@ export class CurationEngine {
         confidence: 0,
       };
 
-      // Fire-and-forget: ingest signal into triple store
-      const tickTs = new Date().toISOString();
-      runPythonScript([
-        "sinain-memory/triple_ingest.py",
-        "--memory-dir", "memory/",
-        "--tick-ts", tickTs,
-        "--signal-result", JSON.stringify(signalResult),
-        "--embed",
-      ], 15_000).catch(() => {});
+    }
+
+    // 2b. Session distillation check — fetch new feed items and distill if needed
+    try {
+      const distillState = readDistillState(workspaceDir);
+      const feedUrl = `${this.sinainCoreUrl}/feed?after=${distillState.lastDistilledFeedId}`;
+      const historyUrl = `${this.sinainCoreUrl}/agent/history?limit=10`;
+
+      const [feedResp, historyResp] = await Promise.all([
+        fetch(feedUrl).then(r => r.json()).catch(() => null),
+        fetch(historyUrl).then(r => r.json()).catch(() => null),
+      ]);
+
+      const feedItems = (feedResp as { messages?: unknown[] })?.messages ?? [];
+      const agentHistory = (historyResp as { results?: unknown[] })?.results ?? [];
+
+      // Only distill if there's meaningful new content (>3 non-trivial items)
+      const significantItems = (feedItems as Array<{ text?: string }>).filter(
+        (item) => item.text && !item.text.startsWith("[PERIODIC]") && item.text.length > 20,
+      );
+
+      if (significantItems.length > 3) {
+        this.logger.info(
+          `sinain-hud: distillation check — ${significantItems.length} new feed items since id=${distillState.lastDistilledFeedId}`,
+        );
+
+        // Combine feed + agent history as transcript
+        const transcript = JSON.stringify([...significantItems, ...agentHistory].slice(0, 100));
+        const sessionMeta = JSON.stringify({
+          ts: new Date().toISOString(),
+          sessionKey: params.sessionSummary,
+          feedItemCount: significantItems.length,
+        });
+
+        // Step 1: Distill
+        const distillT0 = Date.now();
+        const digest = await runPythonScript([
+          "sinain-memory/session_distiller.py",
+          "--memory-dir", "memory/",
+          "--transcript", transcript,
+          "--session-meta", sessionMeta,
+        ], 30_000);
+        latencyMs.distillation = Date.now() - distillT0;
+
+        if (digest && !digest.isEmpty && !digest.error) {
+          // Step 2: Integrate into playbook + knowledge graph
+          const integrateT0 = Date.now();
+          const integration = await runPythonScript([
+            "sinain-memory/knowledge_integrator.py",
+            "--memory-dir", "memory/",
+            "--digest", JSON.stringify(digest),
+          ], 60_000);
+          latencyMs.integration = Date.now() - integrateT0;
+
+          if (integration) {
+            this.logger.info(
+              `sinain-hud: knowledge integrated — changes=${JSON.stringify(integration.changes ?? {})}, graph=${JSON.stringify(integration.graphStats ?? {})}`,
+            );
+            result.distillation = { digest, integration };
+
+            // Regenerate effective playbook after integration
+            this.store.generateEffectivePlaybook();
+
+            // Render knowledge doc
+            this.store.renderKnowledgeDoc();
+          }
+        } else {
+          this.logger.info(`sinain-hud: distillation produced empty/error result, skipping integration`);
+        }
+
+        // Update watermark — always, even if distillation was empty, to advance past these items
+        const maxFeedId = (feedItems as Array<{ id?: number }>).reduce(
+          (max, item) => Math.max(max, item.id ?? 0), distillState.lastDistilledFeedId,
+        );
+        writeDistillState(workspaceDir, {
+          lastDistilledFeedId: maxFeedId,
+          lastDistilledTs: new Date().toISOString(),
+        });
+      }
+    } catch (err) {
+      this.logger.warn(`sinain-hud: distillation check error: ${String(err)}`);
     }
 
     // 3. Insight synthesis (60s timeout)
@@ -438,20 +539,32 @@ export class CurationEngine {
     const moduleGuidance = this.store.getActiveModuleGuidance();
     if (moduleGuidance) contextParts.push(moduleGuidance);
 
-    // Knowledge graph context (10s timeout)
+    // Knowledge document (pre-rendered, synchronous — replaces triple_query.py)
+    const knowledgeDocPath = join(workspaceDir, "memory", "sinain-knowledge.md");
     try {
-      const ragResult = await this.runScript(
-        ["uv", "run", "--with", "requests", "python3",
-         "sinain-memory/triple_query.py",
-         "--memory-dir", "memory",
-         "--context", "current session",
-         "--max-chars", "1500"],
-        { timeoutMs: 10_000, cwd: workspaceDir },
-      );
-      if (ragResult.code === 0) {
-        const parsed = JSON.parse(ragResult.stdout.trim());
-        if (parsed.context && parsed.context.length > 50) {
-          contextParts.push(`[KNOWLEDGE GRAPH CONTEXT]\n${parsed.context}`);
+      if (existsSync(knowledgeDocPath)) {
+        const knowledgeDoc = readFileSync(knowledgeDocPath, "utf-8");
+        if (knowledgeDoc.length > 50) {
+          contextParts.push(`[KNOWLEDGE]\n${knowledgeDoc}`);
+        }
+      }
+    } catch {}
+
+    // Dynamic graph enrichment for situation-specific facts (10s timeout)
+    try {
+      const situationEntities = this.store.extractSituationEntities();
+      if (situationEntities.length > 0) {
+        const graphResult = await this.runScript(
+          ["uv", "run", "--with", "requests", "python3",
+           "sinain-memory/graph_query.py",
+           "--db", "memory/knowledge-graph.db",
+           "--entities", JSON.stringify(situationEntities),
+           "--max-facts", "5",
+           "--format", "text"],
+          { timeoutMs: 10_000, cwd: workspaceDir },
+        );
+        if (graphResult.code === 0 && graphResult.stdout.trim().length > 20) {
+          contextParts.push(`[SITUATION-SPECIFIC KNOWLEDGE]\n${graphResult.stdout.trim()}`);
         }
       }
     } catch {}

package/sinain-knowledge/data/git-store.ts

@@ -49,6 +49,7 @@ export class GitSnapshotStore {
       cwd: this.repoPath,
       encoding: "utf-8",
       timeout: 15_000,
+      maxBuffer: 10 * 1024 * 1024, // 10 MB — snapshot.json can exceed default 1 MB
     }).trim();
   }
 
@@ -192,9 +193,33 @@ export class GitSnapshotStore {
     this.git("commit", "-m", message);
     const hash = this.git("rev-parse", "--short", "HEAD");
     this.logger.info(`sinain-knowledge: snapshot saved → ${hash}`);
+    await this.push();
     return hash;
   }
 
+  // ── Push ─────────────────────────────────────────────────────────────────
+
+  private async push(): Promise<void> {
+    try {
+      const remotes = this.git("remote");
+      if (!remotes) return;
+
+      // Use a longer timeout for network operations
+      execFileSync("git", ["push", "origin", "HEAD"], {
+        cwd: this.repoPath,
+        encoding: "utf-8",
+        timeout: 30_000,
+        stdio: "pipe",
+      });
+      this.logger.info("sinain-knowledge: snapshot pushed to remote");
+    } catch (err) {
+      // Warn only if there IS a remote but push failed
+      this.logger.warn(
+        `sinain-knowledge: push failed (${err instanceof Error ? err.message.split("\n")[0] : String(err)})`,
+      );
+    }
+  }
+
   // ── List ─────────────────────────────────────────────────────────────────
 
   /**
@@ -238,6 +263,7 @@ export class GitSnapshotStore {
         const dbBuf = execFileSync("git", ["show", `${ref}:${TRIPLES_FILE}`], {
           cwd: this.repoPath,
           timeout: 15_000,
+          maxBuffer: 50 * 1024 * 1024, // 50 MB — triplestore can be large
         });
         snapshot.triplestore = { dbBase64: dbBuf.toString("base64") };
       } catch {

package/sinain-knowledge/data/store.ts

@@ -485,4 +485,121 @@ export class KnowledgeStore {
     writeFileSync(tmpPath, content, "utf-8");
     renameSync(tmpPath, situationPath);
   }
+
+  // ── Knowledge Document ─────────────────────────────────────────────────
+
+  /**
+   * Render sinain-knowledge.md — the portable knowledge document (<8KB).
+   * Combines: playbook (working memory) + top graph facts (long-term memory)
+   * + recent session digests + active module guidance.
+   */
+  renderKnowledgeDoc(): boolean {
+    try {
+      const parts: string[] = [];
+      const now = new Date().toISOString();
+
+      parts.push(`# Sinain Knowledge`);
+      parts.push(`<!-- exported: ${now}, version: 3 -->\n`);
+
+      // Playbook (working memory)
+      const playbook = this.readPlaybook();
+      if (playbook) {
+        // Strip header/footer comments for the doc
+        const body = playbook
+          .split("\n")
+          .filter((l) => !l.trim().startsWith("<!--"))
+          .join("\n")
+          .trim();
+        if (body) {
+          parts.push(`## Playbook (Working Memory)\n${body}\n`);
+        }
+      }
+
+      // Recent session digests
+      const digestsPath = join(this.workspaceDir, "memory", "session-digests.jsonl");
+      if (existsSync(digestsPath)) {
+        try {
+          const lines = readFileSync(digestsPath, "utf-8")
+            .split("\n")
+            .filter((l) => l.trim())
+            .slice(-5);
+          if (lines.length > 0) {
+            parts.push(`## Recent Sessions`);
+            for (const line of lines) {
+              try {
+                const d = JSON.parse(line);
+                if (d.whatHappened) {
+                  parts.push(`- ${d.whatHappened}`);
+                }
+              } catch {}
+            }
+            parts.push("");
+          }
+        } catch {}
+      }
+
+      // Active module guidance (brief)
+      const guidance = this.getActiveModuleGuidance();
+      if (guidance && guidance.length > 20) {
+        // Truncate to keep doc under 8KB
+        const truncated = guidance.slice(0, 2000);
+        parts.push(`## Active Modules\n${truncated}\n`);
+      }
+
+      const doc = parts.join("\n").trim() + "\n";
+      const docPath = join(this.workspaceDir, "memory", "sinain-knowledge.md");
+      const memDir = join(this.workspaceDir, "memory");
+      if (!existsSync(memDir)) mkdirSync(memDir, { recursive: true });
+      writeFileSync(docPath, doc, "utf-8");
+      this.logger.info(`sinain-hud: rendered sinain-knowledge.md (${doc.length} chars)`);
+      return true;
+    } catch (err) {
+      this.logger.warn(`sinain-hud: failed to render knowledge doc: ${String(err)}`);
+      return false;
+    }
+  }
+
+  /**
+   * Extract entity keywords from SITUATION.md for dynamic graph enrichment.
+   * Returns lowercase entity names found in the situation text.
+   */
+  extractSituationEntities(): string[] {
+    const situation = this.readSituation();
+    if (!situation) return [];
+
+    const entities: Set<string> = new Set();
+
+    // Extract from "Active Application:" line
+    const appMatch = situation.match(/Active Application:\s*(.+)/i);
+    if (appMatch) {
+      const app = appMatch[1].trim().toLowerCase().replace(/\s+/g, "-");
+      if (app.length > 2) entities.add(app);
+    }
+
+    // Extract from "Detected Errors:" section
+    const errorMatch = situation.match(/Detected Errors:[\s\S]*?(?=\n##|\n---|\Z)/i);
+    if (errorMatch) {
+      // Look for common error type patterns
+      const errorPatterns = errorMatch[0].matchAll(
+        /(?:Error|Exception|TypeError|SyntaxError|ReferenceError|HTTP\s*\d{3}|ENOENT|EACCES|timeout)/gi,
+      );
+      for (const m of errorPatterns) {
+        entities.add(m[0].toLowerCase());
+      }
+    }
+
+    // Extract technology keywords
+    const techKeywords = [
+      "react-native", "react", "flutter", "swift", "kotlin", "python",
+      "typescript", "javascript", "node", "docker", "kubernetes",
+      "intellij", "vscode", "xcode", "metro", "gradle", "cocoapods",
+      "openrouter", "anthropic", "gemini", "openclaw", "sinain",
+    ];
+    const lowerSituation = situation.toLowerCase();
+    for (const kw of techKeywords) {
+      if (lowerSituation.includes(kw)) entities.add(kw);
+    }
+
+    return [...entities].slice(0, 10);
+  }
 }