clawvault 3.1.0 → 3.2.0

package/dist/chunk-EK6S23ZB.js

@@ -0,0 +1,469 @@
+// src/lib/search.ts
+import { execFileSync, spawnSync } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+var QMD_INSTALL_URL = "https://github.com/tobi/qmd";
+var QMD_INSTALL_COMMAND = "bun install -g github:tobi/qmd";
+var QMD_INDEX_ENV_VAR = "CLAWVAULT_QMD_INDEX";
+var QMD_ERROR_MESSAGES = {
+  NOT_INSTALLED: {
+    code: "NOT_INSTALLED",
+    message: "qmd is not installed",
+    hint: `Install qmd to enable ClawVault search and indexing:
+  ${QMD_INSTALL_COMMAND}
+
+For more information: ${QMD_INSTALL_URL}`
+  },
+  NOT_CONFIGURED: {
+    code: "NOT_CONFIGURED",
+    message: "qmd collection is not configured",
+    hint: "Run `clawvault doctor` to diagnose configuration issues, or `clawvault migrate` to fix common setup problems."
+  },
+  COLLECTION_NOT_FOUND: {
+    code: "COLLECTION_NOT_FOUND",
+    message: "qmd collection not found",
+    hint: "The configured qmd collection does not exist. Run `clawvault migrate` to recreate it, or `qmd collection add <name> <path>` manually."
+  },
+  EXECUTION_FAILED: {
+    code: "EXECUTION_FAILED",
+    message: "qmd command failed",
+    hint: "Run `clawvault doctor` to diagnose qmd issues."
+  }
+};
+var QmdUnavailableError = class extends Error {
+  code;
+  hint;
+  constructor(code = "NOT_INSTALLED", additionalContext) {
+    const details = QMD_ERROR_MESSAGES[code];
+    const fullMessage = additionalContext ? `${details.message}: ${additionalContext}` : details.message;
+    super(fullMessage);
+    this.name = "QmdUnavailableError";
+    this.code = code;
+    this.hint = details.hint;
+  }
+  toUserMessage() {
+    return `Error: ${this.message}
+
+${this.hint}`;
+  }
+};
+function getQmdErrorDetails(code) {
+  return QMD_ERROR_MESSAGES[code];
+}
+var QmdConfigurationError = class extends Error {
+  constructor(message, hint) {
+    super(message);
+    this.hint = hint;
+    this.name = "QmdConfigurationError";
+  }
+};
+function ensureJsonArgs(args) {
+  return args.includes("--json") ? args : [...args, "--json"];
+}
+function resolveQmdIndexName(indexName) {
+  const explicit = indexName?.trim();
+  if (explicit) {
+    return explicit;
+  }
+  const fromEnv = process.env[QMD_INDEX_ENV_VAR]?.trim();
+  return fromEnv || void 0;
+}
+function withQmdIndexArgs(args, indexName) {
+  if (args.includes("--index")) {
+    return [...args];
+  }
+  const resolvedIndexName = resolveQmdIndexName(indexName);
+  if (!resolvedIndexName) {
+    return [...args];
+  }
+  return ["--index", resolvedIndexName, ...args];
+}
+function tryParseJson(raw) {
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+function extractJsonPayload(raw) {
+  const start = raw.search(/[\[{]/);
+  if (start === -1) return null;
+  const end = Math.max(raw.lastIndexOf("]"), raw.lastIndexOf("}"));
+  if (end <= start) return null;
+  return raw.slice(start, end + 1);
+}
+function stripQmdNoise(raw) {
+  return raw.split("\n").filter((line) => {
+    const t = line.trim();
+    if (!t) return true;
+    if (t.startsWith("[node-llama-cpp]")) return false;
+    if (t.startsWith("Expanding query")) return false;
+    if (t.startsWith("Searching ") && t.endsWith("queries...")) return false;
+    if (/^[├└─│]/.test(t)) return false;
+    return true;
+  }).join("\n");
+}
+function parseQmdOutput(raw) {
+  const trimmed = stripQmdNoise(raw).trim();
+  if (!trimmed) return [];
+  const direct = tryParseJson(trimmed);
+  const extracted = direct ? null : extractJsonPayload(trimmed);
+  const parsed = direct ?? (extracted ? tryParseJson(extracted) : null);
+  if (!parsed) {
+    throw new Error("qmd returned non-JSON output. Ensure qmd supports --json.");
+  }
+  if (Array.isArray(parsed)) {
+    return parsed;
+  }
+  if (parsed && typeof parsed === "object") {
+    const candidate = parsed.results ?? parsed.items ?? parsed.data;
+    if (Array.isArray(candidate)) {
+      return candidate;
+    }
+  }
+  throw new Error("qmd returned an unexpected JSON shape.");
+}
+function ensureQmdAvailable() {
+  if (!hasQmd()) {
+    throw new QmdUnavailableError("NOT_INSTALLED");
+  }
+}
+function detectQmdError(output, args) {
+  const lowerOutput = output.toLowerCase();
+  if (lowerOutput.includes("missing required arguments") || lowerOutput.includes("unknown option")) {
+    return new QmdConfigurationError(
+      'qmd does not support the search command with the expected arguments. This may indicate an incompatible qmd version or a different tool named "qmd".',
+      `Ensure you have the correct qmd installed: ${QMD_INSTALL_COMMAND}`
+    );
+  }
+  if (lowerOutput.includes("collection not found") || lowerOutput.includes("no collection")) {
+    const collectionArg = args.findIndex((a) => a === "-c");
+    const collectionName = collectionArg >= 0 && args[collectionArg + 1] ? args[collectionArg + 1] : "unknown";
+    return new QmdConfigurationError(
+      `qmd collection "${collectionName}" not found.`,
+      'Run `qmd update -c <collection>` to create the collection, or check your vault\'s .clawvault.json "name" field.'
+    );
+  }
+  if (lowerOutput.includes("no index") || lowerOutput.includes("index not found")) {
+    return new QmdConfigurationError(
+      "qmd index not found. The vault may not be indexed yet.",
+      "Run `clawvault rebuild` or `qmd update` to build the search index."
+    );
+  }
+  if (lowerOutput.includes("embedding") && (lowerOutput.includes("not found") || lowerOutput.includes("missing"))) {
+    return new QmdConfigurationError(
+      "qmd embeddings not found. Vector search requires embeddings to be generated.",
+      "Run `clawvault embed` or `qmd embed` to generate embeddings for semantic search."
+    );
+  }
+  return null;
+}
+function execQmd(args, indexName) {
+  ensureQmdAvailable();
+  const finalArgs = withQmdIndexArgs(ensureJsonArgs(args), indexName);
+  try {
+    const result = execFileSync("qmd", finalArgs, {
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "pipe"],
+      maxBuffer: 10 * 1024 * 1024,
+      // 10MB
+      shell: process.platform === "win32"
+    });
+    return parseQmdOutput(result);
+  } catch (err) {
+    if (err?.code === "ENOENT") {
+      throw new QmdUnavailableError("NOT_INSTALLED");
+    }
+    if (err?.status === 1 && err?.stdout) {
+      return parseQmdOutput(err.stdout);
+    }
+    const output = [err?.stdout, err?.stderr].filter(Boolean).join("\n");
+    const detectedError = detectQmdError(output, finalArgs);
+    if (detectedError) {
+      throw detectedError;
+    }
+    if (output) {
+      try {
+        return parseQmdOutput(output);
+      } catch {
+      }
+      if (output.includes("collection not found") || output.includes("no such collection")) {
+        throw new QmdUnavailableError("COLLECTION_NOT_FOUND", output.trim());
+      }
+    }
+    const errorDetail = err?.message || "unknown error";
+    throw new QmdUnavailableError("EXECUTION_FAILED", errorDetail);
+  }
+}
+function hasQmd() {
+  const result = spawnSync("qmd", ["--version"], { stdio: "ignore", shell: process.platform === "win32" });
+  return !result.error && (result.status === 0 || result.status === 1);
+}
+function qmdUpdate(collection, indexName) {
+  ensureQmdAvailable();
+  const args = ["update"];
+  if (collection) {
+    args.push("-c", collection);
+  }
+  execFileSync("qmd", withQmdIndexArgs(args, indexName), { stdio: "inherit", shell: process.platform === "win32" });
+}
+function qmdEmbed(collection, indexName) {
+  ensureQmdAvailable();
+  const args = ["embed"];
+  if (collection) {
+    args.push("-c", collection);
+  }
+  execFileSync("qmd", withQmdIndexArgs(args, indexName), { stdio: "inherit", shell: process.platform === "win32" });
+}
+var SearchEngine = class {
+  documents = /* @__PURE__ */ new Map();
+  collection = "";
+  vaultPath = "";
+  collectionRoot = "";
+  qmdIndexName;
+  /**
+   * Set the collection name (usually vault name)
+   */
+  setCollection(name) {
+    this.collection = name;
+  }
+  /**
+   * Get the current collection name
+   */
+  getCollection() {
+    return this.collection;
+  }
+  /**
+   * Set the vault path for file resolution
+   */
+  setVaultPath(vaultPath) {
+    this.vaultPath = vaultPath;
+  }
+  /**
+   * Set the collection root for qmd:// URI resolution
+   */
+  setCollectionRoot(root) {
+    this.collectionRoot = path.resolve(root);
+  }
+  /**
+   * Set qmd index name (defaults to qmd global default when omitted)
+   */
+  setIndexName(indexName) {
+    this.qmdIndexName = indexName;
+  }
+  /**
+   * Add or update a document in the local cache
+   * Note: qmd indexing happens via qmd update command
+   */
+  addDocument(doc) {
+    this.documents.set(doc.id, doc);
+  }
+  /**
+   * Remove a document from the local cache
+   */
+  removeDocument(id) {
+    this.documents.delete(id);
+  }
+  /**
+   * No-op for qmd - indexing is managed externally
+   */
+  rebuildIDF() {
+  }
+  /**
+   * BM25 search via qmd
+   */
+  search(query, options = {}) {
+    return this.runQmdQuery("search", query, options);
+  }
+  /**
+   * Vector/semantic search via qmd vsearch
+   */
+  vsearch(query, options = {}) {
+    return this.runQmdQuery("vsearch", query, options);
+  }
+  /**
+   * Combined search with query expansion (qmd query command)
+   */
+  query(query, options = {}) {
+    return this.runQmdQuery("query", query, options);
+  }
+  runQmdQuery(command, query, options) {
+    const {
+      limit = 10,
+      minScore = 0,
+      category,
+      tags,
+      fullContent = false,
+      temporalBoost = false
+    } = options;
+    if (!query.trim()) return [];
+    const args = [
+      command,
+      query,
+      "-n",
+      String(limit * 2),
+      "--json"
+    ];
+    if (this.collection) {
+      args.push("-c", this.collection);
+    }
+    const qmdResults = execQmd(args, this.qmdIndexName);
+    return this.convertResults(qmdResults, {
+      limit,
+      minScore,
+      category,
+      tags,
+      fullContent,
+      temporalBoost
+    });
+  }
+  /**
+   * Convert qmd results to ClawVault SearchResult format
+   */
+  convertResults(qmdResults, options) {
+    const { limit = 10, minScore = 0, category, tags, fullContent = false, temporalBoost = false } = options;
+    const results = [];
+    const maxScore = qmdResults[0]?.score || 1;
+    for (const qr of qmdResults) {
+      const filePath = this.qmdUriToPath(qr.file);
+      const relativePath = this.vaultPath ? path.relative(this.vaultPath, filePath) : filePath;
+      const normalizedRelativePath = relativePath.replace(/\\/g, "/");
+      if (normalizedRelativePath.startsWith("ledger/archive/") || normalizedRelativePath.includes("/ledger/archive/")) {
+        continue;
+      }
+      const docId = normalizedRelativePath.replace(/\.md$/, "");
+      let doc = this.documents.get(docId) ?? this.documents.get(docId.split("/").join(path.sep));
+      const modifiedAt = this.resolveModifiedAt(doc, filePath);
+      const parts = normalizedRelativePath.split("/");
+      const docCategory = parts.length > 1 ? parts[0] : "root";
+      if (category && docCategory !== category) continue;
+      if (tags && tags.length > 0 && doc) {
+        const docTags = new Set(doc.tags);
+        if (!tags.some((t) => docTags.has(t))) continue;
+      }
+      const normalizedScore = maxScore > 0 ? qr.score / maxScore : 0;
+      const finalScore = temporalBoost ? normalizedScore * this.getRecencyFactor(modifiedAt) : normalizedScore;
+      if (finalScore < minScore) continue;
+      if (!doc) {
+        doc = {
+          id: docId,
+          path: filePath,
+          category: docCategory,
+          title: qr.title || path.basename(relativePath, ".md"),
+          content: "",
+          // Content loaded separately if needed
+          frontmatter: {},
+          links: [],
+          tags: [],
+          modified: modifiedAt
+        };
+      }
+      results.push({
+        document: fullContent ? doc : { ...doc, content: "" },
+        score: finalScore,
+        snippet: this.cleanSnippet(qr.snippet),
+        matchedTerms: []
+        // qmd doesn't provide this
+      });
+    }
+    return results.sort((a, b) => b.score - a.score).slice(0, limit);
+  }
+  resolveModifiedAt(doc, filePath) {
+    if (doc) return doc.modified;
+    try {
+      return fs.statSync(filePath).mtime;
+    } catch {
+      return /* @__PURE__ */ new Date(0);
+    }
+  }
+  getRecencyFactor(modifiedAt) {
+    const ageMs = Math.max(0, Date.now() - modifiedAt.getTime());
+    const ageDays = ageMs / (24 * 60 * 60 * 1e3);
+    if (ageDays < 1) return 1;
+    if (ageDays <= 7) return 0.9;
+    return 0.7;
+  }
+  /**
+   * Convert qmd:// URI to file path
+   */
+  qmdUriToPath(uri) {
+    if (uri.startsWith("qmd://")) {
+      const withoutScheme = uri.slice(6);
+      const slashIndex = withoutScheme.indexOf("/");
+      if (slashIndex > -1) {
+        const relativePath = withoutScheme.slice(slashIndex + 1);
+        const root = this.collectionRoot || this.vaultPath;
+        if (root) {
+          return path.join(root, relativePath);
+        }
+        return relativePath;
+      }
+    }
+    return uri;
+  }
+  /**
+   * Clean up qmd snippet format
+   */
+  cleanSnippet(snippet) {
+    if (!snippet) return "";
+    return snippet.replace(/@@ [-+]?\d+,?\d* @@ \([^)]+\)/g, "").trim().split("\n").slice(0, 3).join("\n").slice(0, 300);
+  }
+  /**
+   * Get all cached documents
+   */
+  getAllDocuments() {
+    return [...this.documents.values()];
+  }
+  /**
+   * Get document count
+   */
+  get size() {
+    return this.documents.size;
+  }
+  /**
+   * Clear the local document cache
+   */
+  clear() {
+    this.documents.clear();
+  }
+  /**
+   * Export documents for persistence
+   */
+  export() {
+    return {
+      documents: [...this.documents.values()]
+    };
+  }
+  /**
+   * Import from persisted data
+   */
+  import(data) {
+    this.clear();
+    for (const doc of data.documents) {
+      this.addDocument(doc);
+    }
+  }
+};
+function extractWikiLinks(content) {
+  const matches = content.match(/\[\[([^\]]+)\]\]/g) || [];
+  return matches.map((m) => m.slice(2, -2).toLowerCase());
+}
+function extractTags(content) {
+  const matches = content.match(/#[\w-]+/g) || [];
+  return [...new Set(matches.map((m) => m.slice(1).toLowerCase()))];
+}
+
+export {
+  QMD_INSTALL_URL,
+  QMD_INSTALL_COMMAND,
+  QmdUnavailableError,
+  getQmdErrorDetails,
+  QmdConfigurationError,
+  withQmdIndexArgs,
+  hasQmd,
+  qmdUpdate,
+  qmdEmbed,
+  SearchEngine,
+  extractWikiLinks,
+  extractTags
+};

package/dist/{chunk-LNJA2UGL.js → chunk-ESFLMDRB.js}

@@ -1,9 +1,9 @@
-import {
-  Observer
-} from "./chunk-3NSBOUT3.js";
 import {
   getSessionsDir
 } from "./chunk-HRLWZGMA.js";
+import {
+  Observer
+} from "./chunk-77Q5CSPJ.js";
 
 // src/observer/active-session-observer.ts
 import * as fs from "fs";
@@ -18,11 +18,6 @@ var AGENT_ID_RE = /^[a-zA-Z0-9_-]{1,100}$/;
 var SESSION_ID_RE = /^[a-zA-Z0-9._-]{1,200}$/;
 var CURSOR_FILE_NAME = "observe-cursors.json";
 var STALE_CURSOR_THRESHOLD_MS = 12 * 60 * 60 * 1e3;
-function formatBytes(bytes) {
-  if (bytes >= ONE_MIB) return `${(bytes / ONE_MIB).toFixed(1)}MB`;
-  if (bytes >= ONE_KIB) return `${Math.round(bytes / ONE_KIB)}KB`;
-  return `${bytes}B`;
-}
 function isFiniteNonNegative(value) {
   return typeof value === "number" && Number.isFinite(value) && value >= 0;
 }
@@ -158,12 +153,7 @@ function getObserverStaleness(vaultPath, options = {}) {
     if (!sessionStat.isFile()) {
       continue;
     }
-    const newBytes = sessionStat.size - cursor.lastFileSize;
-    if (newBytes <= 0) {
-      continue;
-    }
-    const processThreshold = getScaledObservationThresholdBytes(sessionStat.size);
-    if (newBytes < processThreshold) {
+    if (cursor.lastFileSize >= sessionStat.size) {
       continue;
     }
     staleCount += 1;
@@ -191,32 +181,8 @@ function loadSessionIndex(sessionsDir) {
     return {};
   }
 }
-function findMostRecentResetFile(sessionsDir, sessionId) {
-  const prefix = `${sessionId}.jsonl.reset.`;
-  try {
-    const files = fs.readdirSync(sessionsDir).filter((f) => f.startsWith(prefix)).sort().reverse();
-    return files.length > 0 ? path.join(sessionsDir, files[0]) : null;
-  } catch {
-    return null;
-  }
-}
 function resolveTranscriptPath(sessionsDir, sessionId) {
-  const mainPath = path.join(sessionsDir, `${sessionId}.jsonl`);
-  try {
-    const stat = fs.statSync(mainPath);
-    if (stat.size < 100) {
-      const resetFile = findMostRecentResetFile(sessionsDir, sessionId);
-      if (resetFile) {
-        return resetFile;
-      }
-    }
-  } catch {
-    const resetFile = findMostRecentResetFile(sessionsDir, sessionId);
-    if (resetFile) {
-      return resetFile;
-    }
-  }
-  return mainPath;
+  return path.join(sessionsDir, `${sessionId}.jsonl`);
 }
 function discoverSessionDescriptors(sessionsDir, fallbackAgentId) {
   const descriptors = [];
@@ -293,21 +259,6 @@ function normalizeRole(role) {
   }
   return role.trim().toLowerCase();
 }
-function summarizeContentArray(content) {
-  const parts = [];
-  for (const block of content) {
-    if (!block || typeof block !== "object") continue;
-    const b = block;
-    if (b.type === "text" && typeof b.text === "string") {
-      const text = normalizeWhitespace(b.text);
-      if (text) parts.push(text);
-    } else if (b.type === "toolCall") {
-      const toolName = typeof b.name === "string" ? b.name : "tool";
-      parts.push(`[tool: ${toolName}]`);
-    }
-  }
-  return parts.join(" ");
-}
 function parseOpenClawJsonLine(line) {
   if (!line.trim()) {
     return "";
@@ -324,38 +275,14 @@ function parseOpenClawJsonLine(line) {
   const entry = parsed;
   if ("role" in entry && "content" in entry) {
     const role = normalizeRole(entry.role);
-    if (role === "system") return "";
-    if (role === "toolresult") {
-      const output = extractContentText(entry.content);
-      const preview = output.slice(0, 80) || "ok";
-      return `[tool_result \u2192 ${preview}]`;
-    }
-    const raw = entry.content;
-    let content;
-    if (Array.isArray(raw)) {
-      content = summarizeContentArray(raw);
-    } else {
-      content = extractContentText(raw);
-    }
+    const content = extractContentText(entry.content);
     if (!content) return "";
     return role ? `${role}: ${content}` : content;
   }
   if (entry.type === "message" && entry.message && typeof entry.message === "object") {
     const message = entry.message;
     const role = normalizeRole(message.role);
-    if (role === "system") return "";
-    if (role === "toolresult") {
-      const output = extractContentText(message.content);
-      const preview = output.slice(0, 80) || "ok";
-      return `[tool_result \u2192 ${preview}]`;
-    }
-    const raw = message.content;
-    let content;
-    if (Array.isArray(raw)) {
-      content = summarizeContentArray(raw);
-    } else {
-      content = extractContentText(raw);
-    }
+    const content = extractContentText(message.content);
     if (!content) return "";
     return role ? `${role}: ${content}` : content;
   }
@@ -518,9 +445,7 @@ async function observeActiveSessions(options, dependencies = {}) {
   const now = dependencies.now ?? (() => /* @__PURE__ */ new Date());
   const cursors = loadObserveCursorStore(vaultPath);
   const descriptors = discoverSessionDescriptors(sessionsDir, agentId);
-  const allCandidates = selectCandidates(descriptors, cursors, options.minNewBytes);
-  allCandidates.sort((a, b) => b.newBytes - a.newBytes);
-  const candidates = options.maxSessions != null && options.maxSessions > 0 ? allCandidates.slice(0, options.maxSessions) : allCandidates;
+  const candidates = selectCandidates(descriptors, cursors, options.minNewBytes);
   if (dryRun || candidates.length === 0) {
     return {
       agentId,
@@ -550,9 +475,7 @@ async function observeActiveSessions(options, dependencies = {}) {
   let observedNewBytes = 0;
   const routedCounts = {};
   const failedSessions = [];
-  for (let i = 0; i < candidates.length; i += 1) {
-    const candidate = candidates[i];
-    console.log(`[observer] processing session ${i + 1}/${candidates.length}: ${candidate.sessionKey} (${formatBytes(candidate.newBytes)} new)`);
+  for (const candidate of candidates) {
     try {
       const observer = observerFactory(vaultPath, observerOptions);
       const { messages, nextOffset } = await readIncrementalMessages(candidate.filePath, candidate.startOffset);

package/dist/{chunk-H34S76MB.js → chunk-ESVS6K2B.js}

@@ -1,14 +1,14 @@
+import {
+  normalizeObservationContent,
+  parseObservationMarkdown
+} from "./chunk-FHFUXL6G.js";
 import {
   requestLlmCompletion,
   resolveLlmProvider
-} from "./chunk-E7MFQB6D.js";
+} from "./chunk-NCKFNBHJ.js";
 import {
   archiveObservations
 } from "./chunk-MQUJNOHK.js";
-import {
-  normalizeObservationContent,
-  parseObservationMarkdown
-} from "./chunk-QK3UCXWL.js";
 import {
   formatIsoWeekKey,
   getIsoWeek,
@@ -213,7 +213,7 @@ function promoteWeekRecords(records) {
   return promoted;
 }
 async function maybeGenerateLlmReflection(weekKey, sections) {
-  const provider = await resolveLlmProvider();
+  const provider = resolveLlmProvider();
   if (!provider) {
     return null;
   }