@yugenlab/vaayu 0.1.0

package/chunks/chunk-JZU37VQ5.js

@@ -0,0 +1,714 @@
+import {
+  EmbeddingService,
+  cosineSimilarity,
+  estimateTokens
+} from "./chunk-JAWZ7ANC.js";
+import {
+  listSessions,
+  loadSession
+} from "./chunk-L7JICQBW.js";
+import {
+  DatabaseManager,
+  initVectorsSchema
+} from "./chunk-U6OLJ36B.js";
+import {
+  getChitraguptaHome
+} from "./chunk-KC6NRZ7U.js";
+
+// ../chitragupta/packages/smriti/src/recall.ts
+import fs2 from "fs";
+import path2 from "path";
+
+// ../chitragupta/packages/smriti/src/streams.ts
+import fs from "fs";
+import path from "path";
+var STREAM_CONFIGS = {
+  identity: {
+    type: "identity",
+    filename: "identity.md",
+    preservation: 0.95,
+    description: "WHO \u2014 personal preferences, corrections, facts, style. Near-immutable."
+  },
+  projects: {
+    type: "projects",
+    filename: "projects.md",
+    preservation: 0.8,
+    description: "WHAT \u2014 active projects, decisions, stack, architecture notes."
+  },
+  tasks: {
+    type: "tasks",
+    filename: "tasks.md",
+    preservation: 0.7,
+    description: "TODO \u2014 pending tasks, completed items, archived entries."
+  },
+  flow: {
+    type: "flow",
+    filename: "flow/default.md",
+    preservation: 0.3,
+    description: "HOW \u2014 ephemeral context, current topic, mood, open questions. Per-device."
+  }
+};
+var STREAM_ORDER = ["identity", "projects", "tasks", "flow"];
+var PRESERVATION_RATIOS = STREAM_ORDER.map(
+  (s) => STREAM_CONFIGS[s].preservation
+);
+function getStreamsRoot() {
+  return path.join(getChitraguptaHome(), "smriti", "streams");
+}
+function getStreamPath(streamType, deviceId) {
+  const root = getStreamsRoot();
+  if (streamType === "flow") {
+    const id = deviceId ?? "default";
+    return path.join(root, "flow", `${id}.md`);
+  }
+  return path.join(root, STREAM_CONFIGS[streamType].filename);
+}
+function buildStreamHeader(streamType, deviceId) {
+  const config = STREAM_CONFIGS[streamType];
+  const lines = [];
+  lines.push(`# ${streamType.charAt(0).toUpperCase() + streamType.slice(1)} Stream`);
+  lines.push("");
+  lines.push(`> ${config.description}`);
+  lines.push(`> Preservation ratio: ${config.preservation}`);
+  if (streamType === "flow" && deviceId) {
+    lines.push(`> Device: ${deviceId}`);
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+function buildStreamFooter(content) {
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const tokens = estimateTokens(content);
+  const lines = [];
+  lines.push("");
+  lines.push("---");
+  lines.push("");
+  lines.push("## Meta");
+  lines.push("");
+  lines.push(`- last_updated: ${now}`);
+  lines.push(`- token_count: ${tokens}`);
+  lines.push("");
+  return lines.join("\n");
+}
+function parseStreamFile(raw) {
+  const footerMarker = "\n---\n\n## Meta";
+  const footerIdx = raw.lastIndexOf(footerMarker);
+  let bodyWithHeader;
+  let footer;
+  if (footerIdx !== -1) {
+    bodyWithHeader = raw.slice(0, footerIdx);
+    footer = raw.slice(footerIdx);
+  } else {
+    bodyWithHeader = raw;
+    footer = "";
+  }
+  const headerEndPattern = /^(?:#[^\n]*\n\n(?:>[^\n]*\n)*)\n/m;
+  const headerMatch = bodyWithHeader.match(headerEndPattern);
+  let header;
+  let content;
+  if (headerMatch) {
+    header = headerMatch[0];
+    content = bodyWithHeader.slice(header.length);
+  } else {
+    const lines = bodyWithHeader.split("\n");
+    let headerEndLine = 0;
+    let pastTitle = false;
+    for (let i = 0; i < lines.length; i++) {
+      if (lines[i].startsWith("#")) {
+        pastTitle = true;
+        continue;
+      }
+      if (pastTitle && !lines[i].startsWith(">") && lines[i].trim() === "") {
+        headerEndLine = i + 1;
+        break;
+      }
+      if (pastTitle && !lines[i].startsWith(">") && lines[i].trim() !== "") {
+        headerEndLine = i;
+        break;
+      }
+    }
+    header = lines.slice(0, headerEndLine).join("\n") + "\n";
+    content = lines.slice(headerEndLine).join("\n");
+  }
+  return { header, content: content.trim(), footer };
+}
+var StreamManager = class {
+  root;
+  constructor() {
+    this.root = getStreamsRoot();
+  }
+  // ─── Ensure Directories ─────────────────────────────────────────
+  /**
+   * Ensure the streams directory structure exists.
+   */
+  ensureDirs() {
+    fs.mkdirSync(this.root, { recursive: true });
+    fs.mkdirSync(path.join(this.root, "flow"), { recursive: true });
+  }
+  // ─── Read ────────────────────────────────────────────────────────
+  /**
+   * Read the full content of a stream file.
+   * Returns empty string if the stream file does not exist.
+   */
+  read(streamType, deviceId) {
+    const filePath = getStreamPath(streamType, deviceId);
+    try {
+      if (fs.existsSync(filePath)) {
+        return fs.readFileSync(filePath, "utf-8");
+      }
+    } catch {
+    }
+    return "";
+  }
+  /**
+   * Read just the content body of a stream (without header/footer).
+   */
+  readContent(streamType, deviceId) {
+    const raw = this.read(streamType, deviceId);
+    if (!raw) return "";
+    const { content } = parseStreamFile(raw);
+    return content;
+  }
+  // ─── Write ───────────────────────────────────────────────────────
+  /**
+   * Overwrite a stream file with new content.
+   * Automatically adds the header and meta footer.
+   */
+  write(streamType, content, deviceId) {
+    this.ensureDirs();
+    const filePath = getStreamPath(streamType, deviceId);
+    const header = buildStreamHeader(streamType, deviceId);
+    const fullContent = header + content;
+    const footer = buildStreamFooter(fullContent);
+    fs.writeFileSync(filePath, fullContent + footer, "utf-8");
+  }
+  // ─── Append ──────────────────────────────────────────────────────
+  /**
+   * Append content to a stream file.
+   * If the stream doesn't exist, creates it with a header.
+   * Updates the meta footer.
+   */
+  append(streamType, entry, deviceId) {
+    this.ensureDirs();
+    const filePath = getStreamPath(streamType, deviceId);
+    const existing = this.read(streamType, deviceId);
+    if (!existing) {
+      const header2 = buildStreamHeader(streamType, deviceId);
+      const content2 = header2 + entry + "\n";
+      const footer2 = buildStreamFooter(content2);
+      fs.writeFileSync(filePath, content2 + footer2, "utf-8");
+      return;
+    }
+    const { header, content } = parseStreamFile(existing);
+    const timestamp = (/* @__PURE__ */ new Date()).toISOString();
+    const separator = content ? "\n\n" : "";
+    const newContent = content + separator + `*${timestamp}*
+
+${entry}`;
+    const fullContent = header + newContent + "\n";
+    const footer = buildStreamFooter(fullContent);
+    fs.writeFileSync(filePath, fullContent + footer, "utf-8");
+  }
+  // ─── Token Counting ──────────────────────────────────────────────
+  /**
+   * Get the estimated token count for a stream.
+   */
+  getTokenCount(streamType, deviceId) {
+    const raw = this.read(streamType, deviceId);
+    return estimateTokens(raw);
+  }
+  /**
+   * Get token counts for all streams (flow uses default device unless specified).
+   */
+  getAllTokenCounts(deviceId) {
+    return {
+      identity: this.getTokenCount("identity"),
+      projects: this.getTokenCount("projects"),
+      tasks: this.getTokenCount("tasks"),
+      flow: this.getTokenCount("flow", deviceId)
+    };
+  }
+  // ─── Budget Allocation ───────────────────────────────────────────
+  /**
+   * Compute per-stream token budgets from a total budget.
+   *
+   * Budget allocation is proportional to preservation ratios:
+   *   budget_i = totalBudget * (preservation_i / sum(preservation))
+   *
+   * This ensures higher-preservation streams (identity) get more budget
+   * than ephemeral streams (flow).
+   */
+  getStreamBudgets(totalBudget) {
+    const totalPreservation = STREAM_ORDER.reduce(
+      (sum, s) => sum + STREAM_CONFIGS[s].preservation,
+      0
+    );
+    const budgets = {};
+    for (const stream of STREAM_ORDER) {
+      const ratio = STREAM_CONFIGS[stream].preservation / totalPreservation;
+      budgets[stream] = Math.floor(totalBudget * ratio);
+    }
+    const allocated = Object.values(budgets).reduce((a, b) => a + b, 0);
+    const remainder = totalBudget - allocated;
+    if (remainder > 0) {
+      budgets.identity += remainder;
+    }
+    return budgets;
+  }
+  // ─── Preservation Enforcement ────────────────────────────────────
+  /**
+   * Trim a stream's content to fit within its preservation budget.
+   * Removes the oldest entries first (from the top of the content section).
+   *
+   * @param streamType - Which stream to trim
+   * @param maxTokens - Maximum token budget for this stream
+   * @param deviceId - Device ID for flow streams
+   * @returns Number of tokens trimmed
+   */
+  enforcePreservation(streamType, maxTokens, deviceId) {
+    const raw = this.read(streamType, deviceId);
+    if (!raw) return 0;
+    const currentTokens = estimateTokens(raw);
+    if (currentTokens <= maxTokens) return 0;
+    const { content } = parseStreamFile(raw);
+    const ENTRY_SEP = /(\n\n\*\d{4}-)/;
+    const parts = content.split(ENTRY_SEP);
+    if (parts.length <= 1) {
+      return 0;
+    }
+    const entries = [parts[0]];
+    for (let i = 1; i < parts.length; i += 2) {
+      entries.push((parts[i] ?? "") + (parts[i + 1] ?? ""));
+    }
+    if (entries.length <= 1) {
+      return 0;
+    }
+    let trimmedEntries = [...entries];
+    let currentContent = trimmedEntries.join("");
+    let tokens = estimateTokens(currentContent);
+    let totalTrimmed = 0;
+    while (tokens > maxTokens && trimmedEntries.length > 1) {
+      const removed = trimmedEntries.shift();
+      totalTrimmed += estimateTokens(removed);
+      currentContent = trimmedEntries.join("");
+      tokens = estimateTokens(currentContent);
+    }
+    if (totalTrimmed > 0) {
+      this.write(streamType, trimmedEntries.join(""), deviceId);
+    }
+    return totalTrimmed;
+  }
+  // ─── Stream Existence ────────────────────────────────────────────
+  /**
+   * Check if a stream file exists on disk.
+   */
+  exists(streamType, deviceId) {
+    const filePath = getStreamPath(streamType, deviceId);
+    return fs.existsSync(filePath);
+  }
+  /**
+   * List all flow device files.
+   */
+  listFlowDevices() {
+    const flowDir = path.join(this.root, "flow");
+    if (!fs.existsSync(flowDir)) return [];
+    try {
+      return fs.readdirSync(flowDir).filter((f) => f.endsWith(".md")).map((f) => f.replace(/\.md$/, ""));
+    } catch {
+      return [];
+    }
+  }
+  /**
+   * Get the streams root directory path.
+   */
+  getRoot() {
+    return this.root;
+  }
+};
+
+// ../chitragupta/packages/smriti/src/recall-scoring.ts
+var _embeddingService = new EmbeddingService();
+function configureRecallScoring(options) {
+  if (options.embeddingService) {
+    _embeddingService = options.embeddingService;
+  }
+}
+async function getEmbedding(text) {
+  return _embeddingService.getEmbedding(text);
+}
+function summarizeSession(session) {
+  const parts = [];
+  for (const turn of session.turns) {
+    if (turn.role === "user") {
+      parts.push(turn.content.slice(0, 200));
+    }
+    if (parts.join(" ").length > 400) break;
+  }
+  return parts.join(" | ").slice(0, 500);
+}
+function extractIndexText(session) {
+  const parts = [];
+  parts.push(session.meta.title);
+  parts.push(session.meta.title);
+  parts.push(session.meta.tags.join(" "));
+  for (const turn of session.turns) {
+    parts.push(turn.content.slice(0, 1e3));
+    if (turn.toolCalls) {
+      for (const tc of turn.toolCalls) {
+        parts.push(tc.name);
+        parts.push(tc.input.slice(0, 200));
+      }
+    }
+  }
+  return parts.join("\n").slice(0, 8e3);
+}
+function resetOllamaAvailability() {
+  _embeddingService.resetAvailability();
+}
+
+// ../chitragupta/packages/smriti/src/recall.ts
+var DEFAULT_TOP_K = 10;
+var DEFAULT_THRESHOLD = 0.3;
+function getIndexDir() {
+  return path2.join(getChitraguptaHome(), "smriti", "index");
+}
+function getEmbeddingsPath() {
+  return path2.join(getIndexDir(), "embeddings.json");
+}
+function vectorToBlob(vector) {
+  const float32 = new Float32Array(vector);
+  return Buffer.from(float32.buffer);
+}
+function blobToVector(blob) {
+  const float32 = new Float32Array(
+    blob.buffer,
+    blob.byteOffset,
+    blob.byteLength / 4
+  );
+  return Array.from(float32);
+}
+var _dbInitialized = false;
+function _resetRecallDbInit() {
+  _dbInitialized = false;
+}
+var RecallEngine = class {
+  entries = [];
+  loaded = false;
+  constructor() {
+    this.loadIndex();
+  }
+  // ─── SQLite Access ────────────────────────────────────────────────
+  getVectorsDb() {
+    const dbm = DatabaseManager.instance();
+    if (!_dbInitialized) {
+      try {
+        initVectorsSchema(dbm);
+        _dbInitialized = true;
+      } catch (err) {
+        process.stderr.write(`[chitragupta] vectors DB schema init failed: ${err instanceof Error ? err.message : err}
+`);
+        throw err;
+      }
+    }
+    return dbm.get("vectors");
+  }
+  // ─── Index Management ────────────────────────────────────────────
+  loadIndex() {
+    try {
+      const db = this.getVectorsDb();
+      const rows = db.prepare("SELECT * FROM embeddings").all();
+      this.entries = rows.map((row) => {
+        let metadata = {};
+        try {
+          metadata = JSON.parse(row.metadata ?? "{}");
+        } catch {
+        }
+        return {
+          id: row.id,
+          vector: blobToVector(row.vector),
+          source: row.source_type,
+          sourceId: row.source_id,
+          title: metadata.title ?? "",
+          text: row.text,
+          summary: metadata.summary ?? "",
+          tags: metadata.tags ?? [],
+          date: metadata.date ?? new Date(row.created_at).toISOString(),
+          deviceId: metadata.deviceId
+        };
+      });
+    } catch {
+      this.loadIndexJson();
+    }
+    this.loaded = true;
+  }
+  saveIndex() {
+    try {
+      const db = this.getVectorsDb();
+      const txn = db.transaction(() => {
+        db.prepare("DELETE FROM embeddings").run();
+        const insert = db.prepare(`
+          INSERT INTO embeddings (id, vector, text, source_type, source_id, dimensions, metadata, created_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        `);
+        for (const entry of this.entries) {
+          insert.run(
+            entry.id,
+            vectorToBlob(entry.vector),
+            entry.text,
+            entry.source,
+            entry.sourceId,
+            entry.vector.length,
+            JSON.stringify({
+              title: entry.title,
+              summary: entry.summary,
+              tags: entry.tags,
+              date: entry.date,
+              deviceId: entry.deviceId
+            }),
+            new Date(entry.date).getTime()
+          );
+        }
+      });
+      txn();
+    } catch {
+      this.saveIndexJson();
+    }
+  }
+  // ─── JSON Fallback (legacy) ────────────────────────────────────────
+  loadIndexJson() {
+    try {
+      const indexPath = getEmbeddingsPath();
+      if (fs2.existsSync(indexPath)) {
+        const raw = fs2.readFileSync(indexPath, "utf-8");
+        this.entries = JSON.parse(raw);
+      }
+    } catch {
+      this.entries = [];
+    }
+  }
+  saveIndexJson() {
+    try {
+      const dir = getIndexDir();
+      fs2.mkdirSync(dir, { recursive: true });
+      fs2.writeFileSync(
+        getEmbeddingsPath(),
+        JSON.stringify(this.entries, null, "	"),
+        "utf-8"
+      );
+    } catch {
+    }
+  }
+  // ─── Session Indexing ────────────────────────────────────────────
+  async indexSession(session) {
+    if (!this.loaded) this.loadIndex();
+    this.entries = this.entries.filter(
+      (e) => !(e.source === "session" && e.sourceId === session.meta.id)
+    );
+    const indexText = extractIndexText(session);
+    const summary = summarizeSession(session);
+    const chunkSize = 4e3;
+    const chunks = [];
+    if (indexText.length <= chunkSize) {
+      chunks.push(indexText);
+    } else {
+      for (let i = 0; i < indexText.length; i += chunkSize - 500) {
+        chunks.push(indexText.slice(i, i + chunkSize));
+        if (i + chunkSize >= indexText.length) break;
+      }
+    }
+    for (let i = 0; i < chunks.length; i++) {
+      const vector = await getEmbedding(chunks[i]);
+      this.entries.push({
+        id: chunks.length > 1 ? `${session.meta.id}-chunk-${i}` : session.meta.id,
+        vector,
+        source: "session",
+        sourceId: session.meta.id,
+        title: session.meta.title,
+        text: chunks[i],
+        summary,
+        tags: session.meta.tags,
+        date: session.meta.updated
+      });
+    }
+    this.saveIndex();
+  }
+  async indexStream(streamType, content, deviceId) {
+    if (!this.loaded) this.loadIndex();
+    const sourceId = deviceId ? `stream-${streamType}-${deviceId}` : `stream-${streamType}`;
+    this.entries = this.entries.filter(
+      (e) => !(e.source === "stream" && e.sourceId === sourceId)
+    );
+    if (!content || content.trim().length === 0) return;
+    const vector = await getEmbedding(content.slice(0, 8e3));
+    this.entries.push({
+      id: sourceId,
+      vector,
+      source: "stream",
+      sourceId,
+      title: `${streamType} stream${deviceId ? ` (${deviceId})` : ""}`,
+      text: content.slice(0, 4e3),
+      summary: content.slice(0, 300),
+      tags: [streamType],
+      date: (/* @__PURE__ */ new Date()).toISOString(),
+      deviceId
+    });
+    this.saveIndex();
+  }
+  // ─── Recall (Search) ─────────────────────────────────────────────
+  async recall(query, options) {
+    if (!this.loaded) this.loadIndex();
+    if (this.entries.length === 0) return [];
+    const topK = options?.topK ?? DEFAULT_TOP_K;
+    const threshold = options?.threshold ?? DEFAULT_THRESHOLD;
+    const queryVector = await getEmbedding(query);
+    const scored = [];
+    for (const entry of this.entries) {
+      if (options?.deviceFilter && entry.deviceId !== options.deviceFilter) continue;
+      if (options?.dateRange) {
+        const entryDate = new Date(entry.date).getTime();
+        const rangeStart = new Date(options.dateRange[0]).getTime();
+        const rangeEnd = new Date(options.dateRange[1]).getTime();
+        if (entryDate < rangeStart || entryDate > rangeEnd) continue;
+      }
+      if (options?.tagFilter && options.tagFilter.length > 0) {
+        const hasTag = options.tagFilter.some((tag) => entry.tags.includes(tag));
+        if (!hasTag) continue;
+      }
+      const score = cosineSimilarity(queryVector, entry.vector);
+      if (score >= threshold) {
+        scored.push({ entry, score });
+      }
+    }
+    scored.sort((a, b) => b.score - a.score);
+    const seen = /* @__PURE__ */ new Set();
+    const results = [];
+    for (const { entry, score } of scored) {
+      if (results.length >= topK) break;
+      if (seen.has(entry.sourceId)) continue;
+      seen.add(entry.sourceId);
+      results.push({
+        sessionId: entry.sourceId,
+        title: entry.title,
+        relevance: Math.max(0, Math.min(1, score)),
+        summary: entry.summary,
+        source: entry.source,
+        matchedContent: entry.text.slice(0, 1e3)
+      });
+    }
+    return results;
+  }
+  // ─── Re-index ────────────────────────────────────────────────────
+  async reindexAll() {
+    this.entries = [];
+    const allMetas = listSessions();
+    for (const meta of allMetas) {
+      try {
+        const session = loadSession(meta.id, meta.project);
+        await this.indexSession(session);
+      } catch {
+      }
+    }
+    const streamManager = new StreamManager();
+    for (const streamType of STREAM_ORDER) {
+      if (streamType === "flow") {
+        const devices = streamManager.listFlowDevices();
+        for (const device of devices) {
+          const content = streamManager.readContent("flow", device);
+          if (content) {
+            await this.indexStream("flow", content, device);
+          }
+        }
+      } else {
+        const content = streamManager.readContent(streamType);
+        if (content) {
+          await this.indexStream(streamType, content);
+        }
+      }
+    }
+    this.saveIndex();
+  }
+  getIndexSize() {
+    return this.entries.length;
+  }
+  resetOllamaAvailability() {
+    resetOllamaAvailability();
+  }
+};
+function migrateEmbeddingsJson() {
+  const jsonPath = getEmbeddingsPath();
+  if (!fs2.existsSync(jsonPath)) {
+    return { migrated: 0, skipped: 0 };
+  }
+  let entries;
+  try {
+    const raw = fs2.readFileSync(jsonPath, "utf-8");
+    entries = JSON.parse(raw);
+  } catch {
+    return { migrated: 0, skipped: 0 };
+  }
+  if (!entries || entries.length === 0) {
+    return { migrated: 0, skipped: 0 };
+  }
+  let migrated = 0;
+  let skipped = 0;
+  try {
+    const dbm = DatabaseManager.instance();
+    initVectorsSchema(dbm);
+    const db = dbm.get("vectors");
+    const txn = db.transaction(() => {
+      const insert = db.prepare(`
+        INSERT OR IGNORE INTO embeddings (id, vector, text, source_type, source_id, dimensions, metadata, created_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+      for (const entry of entries) {
+        try {
+          const result = insert.run(
+            entry.id,
+            vectorToBlob(entry.vector),
+            entry.text,
+            entry.source,
+            entry.sourceId,
+            entry.vector.length,
+            JSON.stringify({
+              title: entry.title,
+              summary: entry.summary,
+              tags: entry.tags,
+              date: entry.date,
+              deviceId: entry.deviceId
+            }),
+            new Date(entry.date).getTime()
+          );
+          if (result.changes > 0) {
+            migrated++;
+          } else {
+            skipped++;
+          }
+        } catch {
+          skipped++;
+        }
+      }
+    });
+    txn();
+    try {
+      fs2.renameSync(jsonPath, jsonPath + ".bak");
+    } catch {
+    }
+  } catch {
+    return { migrated: 0, skipped: entries.length };
+  }
+  return { migrated, skipped };
+}
+
+export {
+  STREAM_CONFIGS,
+  STREAM_ORDER,
+  PRESERVATION_RATIOS,
+  StreamManager,
+  configureRecallScoring,
+  vectorToBlob,
+  blobToVector,
+  _resetRecallDbInit,
+  RecallEngine,
+  migrateEmbeddingsJson
+};
+//# sourceMappingURL=chunk-JZU37VQ5.js.map