@stackmemoryai/stackmemory 1.10.4 → 1.10.5

package/dist/src/cli/commands/wiki.js

@@ -173,6 +173,39 @@ function createWikiCommand() {
     console.log(chalk.gray(`  Articles updated:  ${result.updated.length}`));
     console.log(chalk.gray(`  Total articles:    ${result.totalArticles}`));
   });
+  cmd.command("ingest <source>").description("Ingest a URL or local path into the wiki").option("--wiki-dir <path>", "Override wiki directory").option("-n, --max-pages <n>", "Max pages to crawl for URLs", "20").option("--json", "Output as JSON").action(async (source, options) => {
+    const compiler = getCompiler(options.wikiDir);
+    await compiler.initialize();
+    const isUrl = source.startsWith("http://") || source.startsWith("https://");
+    let result;
+    if (isUrl) {
+      console.log(
+        chalk.cyan(`Crawling ${source} (max ${options.maxPages} pages)...`)
+      );
+      result = await compiler.ingestUrl(source, {
+        maxPages: parseInt(options.maxPages)
+      });
+    } else {
+      console.log(chalk.cyan(`Ingesting ${source}...`));
+      result = await compiler.ingestPath(source);
+    }
+    if (options.json) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    console.log(chalk.green("\nIngested."));
+    console.log(chalk.gray(`  Articles created: ${result.created.length}`));
+    console.log(chalk.gray(`  Total articles:   ${result.totalArticles}`));
+    if (result.created.length > 0) {
+      console.log(chalk.gray("\n  Created:"));
+      result.created.slice(0, 10).forEach((p) => console.log(chalk.gray(`    - ${p}`)));
+      if (result.created.length > 10) {
+        console.log(
+          chalk.gray(`    ...and ${result.created.length - 10} more`)
+        );
+      }
+    }
+  });
   cmd.command("lint").description("Health check the wiki for issues").option("--wiki-dir <path>", "Override wiki directory").option("--json", "Output as JSON").action(async (options) => {
     const compiler = getCompiler(options.wikiDir);
     await compiler.initialize();

package/dist/src/core/wiki/wiki-compiler.js

@@ -234,6 +234,225 @@ class WikiCompiler {
       compiledAt: Date.now()
     };
   }
+  /**
+   * Ingest a URL — fetch page content, convert to markdown, compile into wiki.
+   * Supports single pages and basic site crawling (follows internal links up to maxPages).
+   */
+  async ingestUrl(url, opts) {
+    const maxPages = opts?.maxPages ?? 20;
+    const created = [];
+    const visited = /* @__PURE__ */ new Set();
+    const queue = [url];
+    let baseHost;
+    try {
+      baseHost = new URL(url).hostname;
+    } catch {
+      return {
+        created: [],
+        updated: [],
+        totalArticles: this.countArticles(),
+        compiledAt: Date.now()
+      };
+    }
+    while (queue.length > 0 && visited.size < maxPages) {
+      const pageUrl = queue.shift();
+      if (!pageUrl || visited.has(pageUrl)) continue;
+      visited.add(pageUrl);
+      try {
+        const { title, content, links } = await this.fetchPage(pageUrl);
+        if (!content || content.length < 50) continue;
+        const slug = this.slugify(title || this.urlToSlug(pageUrl));
+        const sourcePath = `sources/${slug}.md`;
+        const article = [
+          "---",
+          `title: "${this.escapeYaml(title || pageUrl)}"`,
+          `category: source`,
+          `url: "${pageUrl}"`,
+          `created: ${(/* @__PURE__ */ new Date()).toISOString()}`,
+          `updated: ${(/* @__PURE__ */ new Date()).toISOString()}`,
+          `tags: [source, web-ingest, ${baseHost}]`,
+          "---",
+          "",
+          `# ${title || pageUrl}`,
+          "",
+          `> Source: ${pageUrl}`,
+          "",
+          content.slice(0, 8e3),
+          content.length > 8e3 ? "\n\n_...truncated..._" : "",
+          ""
+        ].join("\n");
+        this.writeArticle(sourcePath, article);
+        created.push(sourcePath);
+        for (const link of links) {
+          try {
+            const parsed = new URL(link, pageUrl);
+            if (parsed.hostname === baseHost && !visited.has(parsed.href)) {
+              queue.push(parsed.href);
+            }
+          } catch {
+          }
+        }
+      } catch {
+      }
+    }
+    if (created.length > 0) {
+      this.updateIndex();
+      this.appendLog(
+        `## [${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}] ingest-url | ${baseHost}`,
+        `Crawled ${visited.size} pages from ${url} \u2014 ${created.length} articles created`
+      );
+    }
+    return {
+      created,
+      updated: [],
+      totalArticles: this.countArticles(),
+      compiledAt: Date.now()
+    };
+  }
+  /**
+   * Ingest a local file or directory into the wiki.
+   */
+  async ingestPath(filePath) {
+    const created = [];
+    const { statSync } = await import("fs");
+    const stat = statSync(filePath);
+    const processFile = (fp) => {
+      if (!existsSync(fp)) return;
+      const content = readFileSync(fp, "utf-8");
+      const basename = fp.split("/").pop() ?? fp;
+      const ext = basename.split(".").pop() ?? "";
+      if (![
+        "md",
+        "txt",
+        "json",
+        "yaml",
+        "yml",
+        "toml",
+        "ts",
+        "js",
+        "py"
+      ].includes(ext))
+        return;
+      const title = basename.replace(/\.[^.]+$/, "");
+      const slug = this.slugify(title);
+      const sourcePath = `sources/${slug}.md`;
+      const article = [
+        "---",
+        `title: "${this.escapeYaml(title)}"`,
+        `category: source`,
+        `source_file: "${fp}"`,
+        `created: ${(/* @__PURE__ */ new Date()).toISOString()}`,
+        `updated: ${(/* @__PURE__ */ new Date()).toISOString()}`,
+        `tags: [source, local-ingest]`,
+        "---",
+        "",
+        `# ${title}`,
+        "",
+        `> Source: \`${fp}\``,
+        "",
+        content.slice(0, 8e3),
+        content.length > 8e3 ? "\n\n_...truncated..._" : "",
+        ""
+      ].join("\n");
+      this.writeArticle(sourcePath, article);
+      created.push(sourcePath);
+    };
+    if (stat.isDirectory()) {
+      const entries = readdirSync(filePath, { recursive: true });
+      for (const entry of entries) {
+        processFile(join(filePath, entry));
+      }
+    } else {
+      processFile(filePath);
+    }
+    if (created.length > 0) {
+      this.updateIndex();
+      this.appendLog(
+        `## [${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)}] ingest-path | ${filePath}`,
+        `${created.length} articles from local path`
+      );
+    }
+    return {
+      created,
+      updated: [],
+      totalArticles: this.countArticles(),
+      compiledAt: Date.now()
+    };
+  }
+  /** Fetch a web page and extract title, markdown content, and links */
+  async fetchPage(url) {
+    const res = await fetch(url, {
+      headers: { "User-Agent": "StackMemory-Wiki/1.0" },
+      signal: AbortSignal.timeout(1e4)
+    });
+    if (!res.ok) return { title: "", content: "", links: [] };
+    const html = await res.text();
+    const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+    const title = titleMatch?.[1]?.trim() ?? "";
+    const links = [];
+    const linkRe = /href="([^"]+)"/g;
+    let linkMatch;
+    while ((linkMatch = linkRe.exec(html)) !== null) {
+      const href = linkMatch[1] ?? "";
+      if (href && !href.startsWith("#") && !href.startsWith("javascript:") && !href.startsWith("mailto:")) {
+        links.push(href);
+      }
+    }
+    const content = this.htmlToMarkdown(html);
+    return { title, content, links };
+  }
+  /** Simple HTML to markdown conversion */
+  htmlToMarkdown(html) {
+    let text = html;
+    text = text.replace(
+      /<(script|style|nav|footer|header|aside)[^>]*>[\s\S]*?<\/\1>/gi,
+      ""
+    );
+    const mainMatch = text.match(
+      /<(?:main|article)[^>]*>([\s\S]*?)<\/(?:main|article)>/i
+    );
+    if (mainMatch) text = mainMatch[1] ?? text;
+    text = text.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, "\n# $1\n");
+    text = text.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, "\n## $1\n");
+    text = text.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, "\n### $1\n");
+    text = text.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, "\n#### $1\n");
+    text = text.replace(/<p[^>]*>/gi, "\n");
+    text = text.replace(/<\/p>/gi, "\n");
+    text = text.replace(/<br\s*\/?>/gi, "\n");
+    text = text.replace(/<li[^>]*>/gi, "- ");
+    text = text.replace(/<\/li>/gi, "\n");
+    text = text.replace(
+      /<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi,
+      "**$1**"
+    );
+    text = text.replace(/<(?:em|i)[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, "*$1*");
+    text = text.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, "`$1`");
+    text = text.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, "\n```\n$1\n```\n");
+    text = text.replace(
+      /<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi,
+      "[$2]($1)"
+    );
+    text = text.replace(/<[^>]+>/g, "");
+    text = text.replace(/&amp;/g, "&");
+    text = text.replace(/&lt;/g, "<");
+    text = text.replace(/&gt;/g, ">");
+    text = text.replace(/&quot;/g, '"');
+    text = text.replace(/&#39;/g, "'");
+    text = text.replace(/&nbsp;/g, " ");
+    text = text.replace(/\n{3,}/g, "\n\n");
+    text = text.trim();
+    return text;
+  }
+  /** Convert URL path to a readable slug */
+  urlToSlug(url) {
+    try {
+      const parsed = new URL(url);
+      const path = parsed.pathname.replace(/^\/|\/$/g, "");
+      return path ? this.slugify(path.replace(/\//g, "-")) : this.slugify(parsed.hostname);
+    } catch {
+      return this.slugify(url);
+    }
+  }
   /** Lint the wiki for health issues */
   async lint() {
     const allArticles = this.listAllArticles();

package/dist/src/utils/hook-installer.js

@@ -89,6 +89,14 @@ const CANONICAL_HOOKS = [
     timeout: 10,
     commandPrefix: "node",
     required: false
+  },
+  {
+    scriptName: "doc-ingest.js",
+    eventType: "PostToolUse",
+    matcher: "WebFetch",
+    timeout: 15,
+    commandPrefix: "node",
+    required: false
   }
 ];
 const DEAD_HOOKS = ["sms-response-handler.js"];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackmemoryai/stackmemory",
-  "version": "1.10.4",
+  "version": "1.10.5",
   "description": "Lossless, project-scoped memory for AI coding tools. Durable context across sessions with 56 MCP tools, FTS5 search, conductor orchestrator, loop/watch monitoring, snapshot capture, pre-flight overlap checks, Claude/Codex/OpenCode wrappers, Linear sync, and automatic hooks.",
   "engines": {
     "node": ">=20.0.0",

package/templates/claude-hooks/doc-ingest.js

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+
+/**
+ * Doc Ingest Hook
+ *
+ * Fires on PostToolUse when WebFetch is used. If the fetched URL looks like
+ * documentation (docs.*, /api/, /reference/, /guide/), auto-ingests it into
+ * the project wiki via stackmemory wiki ingest.
+ *
+ * Lightweight: only runs if .stackmemory/config.yaml has obsidian.vaultPath.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+
+// Doc URL patterns
+const DOC_PATTERNS = [
+  /^https?:\/\/docs\./i,
+  /^https?:\/\/[^/]+\/docs\//i,
+  /^https?:\/\/[^/]+\/api\//i,
+  /^https?:\/\/[^/]+\/reference\//i,
+  /^https?:\/\/[^/]+\/guide/i,
+  /^https?:\/\/[^/]+\/tutorial/i,
+  /^https?:\/\/developer\./i,
+];
+
+// Rate limit: max 1 ingest per URL per session
+const seen = new Set();
+
+function main() {
+  try {
+    // Only fire on WebFetch tool
+    const toolName = process.env.TOOL_NAME || '';
+    if (toolName !== 'WebFetch') return;
+
+    // Check wiki is configured
+    const configPath = path.join(process.cwd(), '.stackmemory', 'config.yaml');
+    if (!fs.existsSync(configPath)) return;
+    const config = fs.readFileSync(configPath, 'utf-8');
+    if (!config.includes('vaultPath:')) return;
+
+    // Extract URL from tool input
+    const input = process.env.TOOL_INPUT || '';
+    let url;
+    try {
+      const parsed = JSON.parse(input);
+      url = parsed.url;
+    } catch {
+      // Try regex fallback
+      const match = input.match(/https?:\/\/[^\s"']+/);
+      url = match ? match[0] : null;
+    }
+
+    if (!url) return;
+
+    // Check if URL matches doc patterns
+    const isDoc = DOC_PATTERNS.some((p) => p.test(url));
+    if (!isDoc) return;
+
+    // Dedupe per session
+    const host = new URL(url).hostname;
+    if (seen.has(host)) return;
+    seen.add(host);
+
+    // Run ingest (fire-and-forget, max 1 page for auto-ingest)
+    execSync(`stackmemory wiki ingest "${url}" -n 5`, {
+      timeout: 15000,
+      stdio: 'ignore',
+    });
+  } catch {
+    // Silent fail
+  }
+}
+
+main();