@agentmemory/fs-watcher 0.1.0

package/README.md

@@ -0,0 +1,63 @@
+# @agentmemory/fs-watcher
+
+Filesystem connector for agentmemory. Watches one or more directories and emits an observation to the running agentmemory server every time a file changes.
+
+Part of the data-source-connectors effort tracked in issue #62.
+
+## Install
+
+```bash
+npm install -g @agentmemory/fs-watcher
+```
+
+Or run without installing:
+
+```bash
+npx @agentmemory/fs-watcher ~/work/my-repo
+```
+
+## Usage
+
+```bash
+# CLI args win over env.
+agentmemory-fs-watcher ~/work/my-repo ~/notes
+
+# Or set env once in your shell.
+export AGENTMEMORY_FS_WATCH_DIRS=~/work/my-repo,~/notes
+export AGENTMEMORY_URL=http://localhost:3111
+export AGENTMEMORY_SECRET=...   # only if the server requires auth
+agentmemory-fs-watcher
+```
+
+Every file change inside the watched roots becomes a `post_tool_use` observation whose `data.changeKind` is `file_change` or `file_delete`. The first 4 KB of each text file is included as `data.content` so retrieval can match by substring; larger files are truncated with `data.truncated: true`. Binary files are not read (set `AGENTMEMORY_FS_WATCH_ALLOW_BINARY=1` to override).
+
+Session id and project are required by the observe endpoint — set them via env, or the watcher generates a per-process `fs-watcher-<ts>-<rand>` session id and uses the first root's directory name as the project.
+
+Requires Node.js **>=20 LTS**. Recursive `fs.watch` needs Node 19.1.0+ on Linux; Node 20 is the minimum supported LTS line.
+
+## Configuration
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `AGENTMEMORY_FS_WATCH_DIRS` | — | Comma-separated list of directories to watch |
+| `AGENTMEMORY_FS_WATCH_IGNORE` | — | Comma-separated regex patterns to ignore (applied to relative paths) |
+| `AGENTMEMORY_FS_WATCH_ALLOW_BINARY` | `0` | `1` to include binary files in the preview read |
+| `AGENTMEMORY_URL` | `http://localhost:3111` | agentmemory server URL |
+| `AGENTMEMORY_SECRET` | — | Bearer token, required if the server has `AGENTMEMORY_SECRET` set |
+| `AGENTMEMORY_PROJECT` | — | Optional project label attached to each observation |
+| `AGENTMEMORY_SESSION_ID` | — | Optional session id to attribute observations to |
+
+## Defaults
+
+Ignored out of the box: `.git/`, `node_modules/`, `dist/`, `build/`, `.next/`, `.turbo/`, `coverage/`, `.DS_Store`, `*.log`, `*.lock`. Extend with `AGENTMEMORY_FS_WATCH_IGNORE`.
+
+Text extensions read for preview: common source, config, and docs (`.ts/.js/.py/.go/.rs/.md/.yaml/...`). Unknown extensions are recorded as a path-only observation without content.
+
+Writes are debounced 500 ms per path so a stream of saves from your editor becomes a single observation.
+
+## Notes
+
+- Uses Node's built-in `fs.watch` with `{ recursive: true }`. Works natively on macOS, Linux, and Windows 10+. No native deps.
+- If `fs.watch` errors on a specific root (permission, platform quirk), the watcher logs and continues on the others.
+- The process must keep running. Use a process manager (`launchd`, `systemd`, `pm2`) to supervise it.
+- This connector is intentionally one-way: it writes observations and never reads the agentmemory store.

package/bin.mjs

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+import { FilesystemWatcher, configFromEnv } from "./watcher.mjs";
+
+const cliArgs = process.argv.slice(2);
+const envCfg = configFromEnv(process.env);
+
+const roots = cliArgs.length > 0 ? cliArgs : envCfg.roots;
+if (!roots || roots.length === 0) {
+  process.stderr.write(
+    "agentmemory-fs-watcher: no directories to watch.\n" +
+      "Usage: agentmemory-fs-watcher <dir> [<dir>...]\n" +
+      "Or set AGENTMEMORY_FS_WATCH_DIRS=path1,path2\n",
+  );
+  process.exit(2);
+}
+
+const watcher = new FilesystemWatcher({ ...envCfg, roots });
+watcher.start();
+process.stderr.write(
+  `[fs-watcher] emitting to ${envCfg.baseUrl || "http://localhost:3111"}\n`,
+);
+
+const shutdown = () => {
+  watcher.stop();
+  process.exit(0);
+};
+process.on("SIGINT", shutdown);
+process.on("SIGTERM", shutdown);

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@agentmemory/fs-watcher",
+  "version": "0.1.0",
+  "description": "Filesystem connector for agentmemory — emits observations on file changes.",
+  "type": "module",
+  "bin": {
+    "agentmemory-fs-watcher": "./bin.mjs"
+  },
+  "main": "./watcher.mjs",
+  "exports": {
+    ".": "./watcher.mjs"
+  },
+  "files": ["watcher.mjs", "bin.mjs", "README.md"],
+  "engines": { "node": ">=20" },
+  "license": "Apache-2.0",
+  "homepage": "https://github.com/rohitg00/agentmemory/tree/main/integrations/filesystem-watcher",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/rohitg00/agentmemory.git",
+    "directory": "integrations/filesystem-watcher"
+  }
+}

package/watcher.mjs

@@ -0,0 +1,229 @@
+import { watch, promises as fsp, statSync } from "node:fs";
+import { resolve, relative, join, extname, sep, basename } from "node:path";
+import { randomBytes } from "node:crypto";
+
+const TEXT_EXTENSIONS = new Set([
+  ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
+  ".py", ".rb", ".go", ".rs", ".java", ".kt", ".swift",
+  ".c", ".cc", ".cpp", ".h", ".hpp",
+  ".md", ".mdx", ".txt", ".rst",
+  ".json", ".yaml", ".yml", ".toml", ".ini", ".env",
+  ".html", ".css", ".scss", ".vue", ".svelte",
+  ".sh", ".bash", ".zsh", ".fish",
+  ".sql", ".graphql", ".proto",
+]);
+
+const DEFAULT_IGNORE = [
+  /(?:^|\/)\.git(?:\/|$)/,
+  /(?:^|\/)node_modules(?:\/|$)/,
+  /(?:^|\/)dist(?:\/|$)/,
+  /(?:^|\/)build(?:\/|$)/,
+  /(?:^|\/)\.next(?:\/|$)/,
+  /(?:^|\/)\.turbo(?:\/|$)/,
+  /(?:^|\/)coverage(?:\/|$)/,
+  /(?:^|\/)\.DS_Store$/,
+  /\.log$/,
+  /\.lock$/,
+];
+
+const MAX_PREVIEW_BYTES = 4096;
+const DEBOUNCE_MS = 500;
+
+export class FilesystemWatcher {
+  constructor(config = {}) {
+    this.roots = (config.roots || []).map((r) => resolve(r));
+    this.baseUrl = (config.baseUrl || "http://localhost:3111").replace(/\/+$/, "");
+    this.secret = config.secret;
+    this.project =
+      config.project ||
+      (this.roots[0] ? basename(this.roots[0]) : "filesystem-watcher");
+    this.sessionId =
+      config.sessionId ||
+      `fs-watcher-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
+    this.ignore = [...DEFAULT_IGNORE, ...(config.ignorePatterns || [])];
+    this.allowBinary = Boolean(config.allowBinary);
+    this.logger = config.logger || console;
+    this.watchers = [];
+    this.pendingByPath = new Map();
+  }
+
+  isIgnored(path) {
+    return this.ignore.some((re) => re.test(path));
+  }
+
+  isTextFile(path) {
+    if (this.allowBinary) return true;
+    const ext = extname(path).toLowerCase();
+    return TEXT_EXTENSIONS.has(ext);
+  }
+
+  async readPreview(path) {
+    try {
+      const fh = await fsp.open(path, "r");
+      try {
+        const buf = Buffer.alloc(MAX_PREVIEW_BYTES);
+        const { bytesRead } = await fh.read(buf, 0, MAX_PREVIEW_BYTES, 0);
+        return buf.slice(0, bytesRead).toString("utf-8");
+      } finally {
+        await fh.close();
+      }
+    } catch {
+      return null;
+    }
+  }
+
+  async emit(event) {
+    const headers = { "content-type": "application/json" };
+    if (this.secret) headers.authorization = `Bearer ${this.secret}`;
+    try {
+      const res = await fetch(`${this.baseUrl}/agentmemory/observe`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(event),
+        signal: AbortSignal.timeout(5000),
+      });
+      if (!res.ok) {
+        this.logger.warn?.(
+          `[fs-watcher] observe ${res.status}: ${await res.text().catch(() => "")}`,
+        );
+      }
+    } catch (err) {
+      this.logger.warn?.(`[fs-watcher] observe failed: ${err?.message || err}`);
+    }
+  }
+
+  schedule(rootDir, relPath) {
+    const key = join(rootDir, relPath);
+    const existing = this.pendingByPath.get(key);
+    if (existing) clearTimeout(existing.timer);
+    const timer = setTimeout(() => {
+      this.pendingByPath.delete(key);
+      this.flush(rootDir, relPath).catch((err) =>
+        this.logger.warn?.(`[fs-watcher] flush failed: ${err?.message || err}`),
+      );
+    }, DEBOUNCE_MS);
+    this.pendingByPath.set(key, { timer });
+  }
+
+  async flush(rootDir, relPath) {
+    const absPath = join(rootDir, relPath);
+    if (this.isIgnored(relPath)) return;
+    let exists = true;
+    let size = 0;
+    try {
+      const st = statSync(absPath);
+      if (!st.isFile()) return;
+      size = st.size;
+    } catch {
+      exists = false;
+    }
+    const changeKind = exists ? "file_change" : "file_delete";
+    let preview = null;
+    if (exists && this.isTextFile(absPath)) {
+      preview = await this.readPreview(absPath);
+    }
+    const truncated = exists && size > MAX_PREVIEW_BYTES;
+    const payload = {
+      hookType: "post_tool_use",
+      sessionId: this.sessionId,
+      project: this.project,
+      cwd: rootDir,
+      timestamp: new Date().toISOString(),
+      data: {
+        source: "filesystem-watcher",
+        changeKind,
+        files: [relPath],
+        content: this.formatContent(relPath, changeKind, preview, {
+          size,
+          truncated,
+        }),
+        rootDir,
+        absPath,
+        size,
+        truncated,
+      },
+    };
+    await this.emit(payload);
+  }
+
+  formatContent(relPath, changeKind, preview, { size, truncated }) {
+    if (changeKind === "file_delete") return `deleted: ${relPath}`;
+    const head = `${relPath} (${size} bytes${truncated ? ", truncated" : ""})`;
+    if (preview === null) return head;
+    return `${head}\n\n${preview}`;
+  }
+
+  start() {
+    if (this.roots.length === 0) {
+      throw new Error("filesystem-watcher: at least one root directory is required");
+    }
+    const failures = [];
+    for (const root of this.roots) {
+      try {
+        const handle = watch(
+          root,
+          { recursive: true, persistent: true },
+          (_eventType, filename) => {
+            if (!filename) return;
+            const rel = filename.split(sep).join("/");
+            if (this.isIgnored(rel)) return;
+            this.schedule(root, rel);
+          },
+        );
+        handle.on("error", (err) => {
+          this.logger.warn?.(`[fs-watcher] watch error on ${root}: ${err?.message || err}`);
+        });
+        this.watchers.push(handle);
+        this.logger.info?.(`[fs-watcher] watching ${root}`);
+      } catch (err) {
+        const msg = err?.message || String(err);
+        failures.push(`${root}: ${msg}`);
+        this.logger.error?.(`[fs-watcher] failed to watch ${root}: ${msg}`);
+      }
+    }
+    if (this.watchers.length === 0) {
+      throw new Error(
+        `filesystem-watcher: could not watch any of the configured roots. ` +
+          `If you are on Node 18 + Linux, recursive fs.watch requires Node >=19.1.0; upgrade to Node 20 LTS or newer. ` +
+          `Failures: ${failures.join("; ")}`,
+      );
+    }
+  }
+
+  stop() {
+    for (const w of this.watchers) {
+      try {
+        w.close();
+      } catch {}
+    }
+    this.watchers = [];
+    for (const { timer } of this.pendingByPath.values()) {
+      clearTimeout(timer);
+    }
+    this.pendingByPath.clear();
+  }
+}
+
+// Small helper used by tests and bin.mjs to parse env.
+export function configFromEnv(env = process.env) {
+  const roots = (env.AGENTMEMORY_FS_WATCH_DIRS || "")
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+  const extraIgnore = (env.AGENTMEMORY_FS_WATCH_IGNORE || "")
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean)
+    .map((s) => new RegExp(s));
+  return {
+    roots,
+    baseUrl: env.AGENTMEMORY_URL,
+    secret: env.AGENTMEMORY_SECRET,
+    project: env.AGENTMEMORY_PROJECT || null,
+    sessionId: env.AGENTMEMORY_SESSION_ID || null,
+    ignorePatterns: extraIgnore,
+    allowBinary: env.AGENTMEMORY_FS_WATCH_ALLOW_BINARY === "1",
+  };
+}
+
+export { relative as _relativeForTests };