membot 0.1.1 → 0.1.2

package/.claude/skills/membot.md

@@ -75,7 +75,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 
 - Defaults always operate on the current, non-tombstoned version.
 - Pass an explicit `--version <timestamp>` (from `membot versions`) to read or diff history.
-- `membot_add`, refresh-with-changes, `write`, and `mv` each create a new version. The previous version is preserved.
+- `membot_add` (when source bytes have changed), refresh-with-changes, `write`, and `mv` each create a new version. The previous version is preserved. Re-running `membot_add` against an unchanged source is a no-op (status `unchanged`, same `version_id`); pass `force=true` to force a new version.
 - Mutating an existing version is not possible — corrections are new versions.
 
 ## When to use this skill
@@ -99,7 +99,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 
 | Command                               | Purpose                                                                        |
 | ------------------------------------- | ------------------------------------------------------------------------------ |
-| `membot add <source>`                 | Ingest file, directory, glob, URL, or `inline:<text>` (one new version each)   |
+| `membot add <source>`                 | Ingest file, directory, glob, URL, or `inline:<text>`. Skips unchanged sources; pass `--force` to re-ingest |
 | `membot ls [prefix]`                  | List current files (size, mime, refresh status)                                |
 | `membot tree [prefix]`                | Render the synthesised logical-path tree                                       |
 | `membot read <path>`                  | Read current markdown surrogate (or `--bytes` for original)                    |

package/.cursor/rules/membot.mdc

@@ -75,7 +75,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 
 - Defaults always operate on the current, non-tombstoned version.
 - Pass an explicit `--version <timestamp>` (from `membot versions`) to read or diff history.
-- `membot_add`, refresh-with-changes, `write`, and `mv` each create a new version. The previous version is preserved.
+- `membot_add` (when source bytes have changed), refresh-with-changes, `write`, and `mv` each create a new version. The previous version is preserved. Re-running `membot_add` against an unchanged source is a no-op (status `unchanged`, same `version_id`); pass `force=true` to force a new version.
 - Mutating an existing version is not possible — corrections are new versions.
 
 ## When to use this rule
@@ -99,7 +99,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 
 | Command                               | Purpose                                                                        |
 | ------------------------------------- | ------------------------------------------------------------------------------ |
-| `membot add <source>`                 | Ingest file, directory, glob, URL, or `inline:<text>` (one new version each)   |
+| `membot add <source>`                 | Ingest file, directory, glob, URL, or `inline:<text>`. Skips unchanged sources; pass `--force` to re-ingest |
 | `membot ls [prefix]`                  | List current files (size, mime, refresh status)                                |
 | `membot tree [prefix]`                | Render the synthesised logical-path tree                                       |
 | `membot read <path>`                  | Read current markdown surrogate (or `--bytes` for original)                    |

package/README.md

@@ -15,18 +15,13 @@
 ## Install
 
 ```bash
-# macOS / Linux — pre-built binary
-curl -fsSL https://raw.githubusercontent.com/evantahler/membot/main/install.sh | bash
-
-# Windows — PowerShell
-iwr -useb https://raw.githubusercontent.com/evantahler/membot/main/install.ps1 | iex
-
-# From npm (requires Bun or Node)
-bun add -g membot
+bun install -g membot
 # or
 npm install -g membot
 ```
 
+This pulls in DuckDB's per-platform native bindings alongside membot. The build externalizes `@duckdb/*` (those `.node` bindings can't be embedded by `bun build --compile`), so a global npm/bun install is the supported path.
+
 ## Quick start
 
 ```bash
@@ -55,7 +50,7 @@ The skill files describe the discover → ingest → search → read → write w
 
 | Command                         | Description                                                                       |
 | ------------------------------- | --------------------------------------------------------------------------------- |
-| `membot add <source>`           | Ingest a file, directory, glob, URL, or `inline:<text>`. Each match → new version |
+| `membot add <source>`           | Ingest a file, directory, glob, URL, or `inline:<text>`. Skips on unchanged source bytes; pass `--force` to re-ingest |
 | `membot ls [prefix]`            | List current files (size, mime, refresh status)                                   |
 | `membot tree [prefix]`          | Render the synthesised logical-path tree                                          |
 | `membot read <path>`            | Read the markdown surrogate (or `--bytes` for original bytes, base64)             |

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "membot",
-	"version": "0.1.1",
+	"version": "0.1.2",
 	"description": "Versioned context store with hybrid search for AI agents. Stdio + HTTP MCP server and CLI.",
 	"type": "module",
 	"exports": {
@@ -26,7 +26,7 @@
 		"test": "bun test",
 		"lint": "biome ci . && tsc --noEmit",
 		"format": "biome check --write .",
-		"prebuild": "bash scripts/apply-transformers-patch.sh",
+		"prebuild": "bash scripts/apply-patches.sh",
 		"build": "bun build --compile --minify --sourcemap --external '@duckdb/*' ./src/cli.ts --outfile dist/membot"
 	},
 	"keywords": [

package/patches/@evantahler%2Fmcpx@0.21.4.patch

@@ -0,0 +1,44 @@
+diff --git a/src/search/onnx-wasm-paths.ts b/src/search/onnx-wasm-paths.ts
+--- a/src/search/onnx-wasm-paths.ts
++++ b/src/search/onnx-wasm-paths.ts
+@@ -1,31 +1,9 @@
+-// Embed the onnxruntime-web WASM runtime files into the compiled binary
+-// (`bun build --compile`) so they survive in a single-binary distribution
+-// where the user has no node_modules.
+-//
+-// This file is loaded **dynamically** by semantic.ts. The relative paths
+-// only resolve in the local repo / compiled binary; for npm/bun-installed
+-// mcpx the parent directory layout is different (deps are hoisted), the
+-// dynamic import throws, and we fall back to letting transformers.js
+-// load WASM via its default mechanism — which works fine because in
+-// that environment node_modules exists and onnxruntime-web is reachable
+-// through normal module resolution.
+-
+-// The relative `../../node_modules/...` paths only resolve from the local repo
+-// layout (and inside `bun build --compile`). When this file is shipped via npm,
+-// deps are hoisted, so consumer `tsc` runs hit TS2307. The `ts-ignore` directive
+-// below silences that for consumers; we avoid the stricter `expect-error` form
+-// because in the local repo the path resolves fine and there would be no error
+-// to expect. At runtime the dynamic import in semantic.ts is wrapped in
+-// try/catch and falls back to transformers.js's default WASM loader (issue #85).
+-// biome-ignore lint/suspicious/noTsIgnore: must stay as ts-ignore per comment above
+-// @ts-ignore - dynamic-only import
+-import wasmMjsPath from "../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.asyncify.mjs" with {
+-	type: "file",
+-};
+-// biome-ignore lint/suspicious/noTsIgnore: must stay as ts-ignore per comment above
+-// @ts-ignore - dynamic-only import
+-import wasmBinPath from "../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.asyncify.wasm" with {
+-	type: "file",
+-};
+-
+-export { wasmBinPath, wasmMjsPath };
++// PATCHED (membot): upstream mcpx ships static `with { type: "file" }` imports
++// of onnxruntime-web WASM assets via `../../node_modules/...`, which only
++// resolves when mcpx is built standalone. When consumed as an npm dep those
++// paths are unreachable and `bun build --compile` fails at build time. membot
++// never invokes mcpx's semantic search (only `mcpx.exec()` for URL fetching),
++// so we stub the exports — semantic.ts wraps the dynamic import in try/catch
++// and falls back to transformers.js's default WASM loader.
++export const wasmMjsPath = "";
++export const wasmBinPath = "";

package/scripts/apply-patches.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Apply node_modules patches imperatively. We don't use package.json's
+# `patchedDependencies` field because that field, when present in a published
+# package, breaks `bun install` from a tarball.
+#
+# Each patch is gated by a marker file inside its target so reruns are no-ops.
+
+apply_patch() {
+	local patch="$1" target="$2" marker_name="$3"
+	local marker="$target/$marker_name"
+
+	if [ ! -d "$target" ]; then
+		echo "error: $target not found — run \`bun install\` first" >&2
+		exit 1
+	fi
+	if [ ! -f "$patch" ]; then
+		echo "error: $patch not found" >&2
+		exit 1
+	fi
+	if [ -f "$marker" ]; then
+		echo "patch $patch already applied — skipping"
+		return 0
+	fi
+
+	echo "Applying $patch to $target..."
+	git apply --directory="$target" "$patch"
+	touch "$marker"
+}
+
+# @huggingface/transformers — replace static `import 'onnxruntime-node'` with a
+# stub so `bun build --compile` produces a binary using the WASM backend
+# (onnxruntime-web) instead of onnxruntime-node, whose native bindings can't be
+# bundled into a single-binary distribution.
+apply_patch \
+	"patches/@huggingface%2Ftransformers@4.2.0.patch" \
+	"node_modules/@huggingface/transformers" \
+	".membot-transformers-patch-applied"
+
+# @evantahler/mcpx — stub `src/search/onnx-wasm-paths.ts` whose static
+# `with { type: "file" }` imports use a relative path that only resolves in
+# mcpx's own repo layout. When mcpx is consumed as an npm dep those paths are
+# unreachable and `bun build --compile` fails at build time. membot never
+# invokes mcpx's semantic search, so the stubbed exports are safe.
+apply_patch \
+	"patches/@evantahler%2Fmcpx@0.21.4.patch" \
+	"node_modules/@evantahler/mcpx" \
+	".membot-mcpx-patch-applied"

package/src/ingest/ingest.ts

@@ -21,13 +21,14 @@ export interface IngestInput {
 	refresh_frequency?: string;
 	fetcher_hint?: string;
 	change_note?: string;
+	force?: boolean;
 }
 
 export interface IngestEntryResult {
 	source_path: string;
 	logical_path: string;
 	version_id: string | null;
-	status: "ok" | "failed";
+	status: "ok" | "unchanged" | "failed";
 	error?: string;
 	mime_type: string | null;
 	size_bytes: number;
@@ -39,6 +40,7 @@ export interface IngestResult {
 	ingested: IngestEntryResult[];
 	total: number;
 	ok: number;
+	unchanged: number;
 	failed: number;
 }
 
@@ -57,14 +59,15 @@ export async function ingest(input: IngestInput, ctx: AppContext): Promise<Inges
 	});
 
 	const refreshSec = parseDuration(input.refresh_frequency);
+	const force = input.force === true;
 
 	if (resolved.kind === "inline") {
 		return ingestInline(resolved.text, input, ctx, refreshSec);
 	}
 	if (resolved.kind === "url") {
-		return ingestUrl(resolved.url, input, ctx, refreshSec);
+		return ingestUrl(resolved.url, input, ctx, refreshSec, force);
 	}
-	return ingestLocalFiles(resolved, input, ctx, refreshSec);
+	return ingestLocalFiles(resolved, input, ctx, refreshSec, force);
 }
 
 /** Ingest a single inline blob (source_type='inline'). */
@@ -119,6 +122,7 @@ async function ingestUrl(
 	input: IngestInput,
 	ctx: AppContext,
 	refreshSec: number | null,
+	force: boolean,
 ): Promise<IngestResult> {
 	const mcpxAdapter = ctx.mcpx
 		? {
@@ -151,6 +155,15 @@ async function ingestUrl(
 		result.fetcher = fetched.fetcher;
 		result.source_sha256 = fetched.sha256;
 
+		if (!force) {
+			const cur = await getCurrent(ctx.db, logicalPath);
+			if (cur && cur.source_sha256 === fetched.sha256) {
+				result.status = "unchanged";
+				result.version_id = cur.version_id;
+				return summarize([result]);
+			}
+		}
+
 		const versionId = await pipelineForBytes(ctx, {
 			logicalPath,
 			bytes: fetched.bytes,
@@ -181,6 +194,7 @@ async function ingestLocalFiles(
 	input: IngestInput,
 	ctx: AppContext,
 	refreshSec: number | null,
+	force: boolean,
 ): Promise<IngestResult> {
 	if (resolved.entries.length === 0) {
 		throw new HelpfulError({
@@ -213,6 +227,16 @@ async function ingestLocalFiles(
 			result.size_bytes = local.sizeBytes;
 			result.source_sha256 = local.sha256;
 
+			if (!force) {
+				const cur = await getCurrent(ctx.db, logicalPath);
+				if (cur && cur.source_sha256 === local.sha256) {
+					result.status = "unchanged";
+					result.version_id = cur.version_id;
+					results.push(result);
+					continue;
+				}
+			}
+
 			const versionId = await pipelineForBytes(ctx, {
 				logicalPath,
 				bytes: local.bytes,
@@ -236,7 +260,10 @@ async function ingestLocalFiles(
 		}
 		results.push(result);
 	}
-	ctx.progress.done(`ingested ${results.filter((r) => r.status === "ok").length}/${results.length}`);
+	const okCount = results.filter((r) => r.status === "ok").length;
+	const unchangedCount = results.filter((r) => r.status === "unchanged").length;
+	const suffix = unchangedCount > 0 ? ` (${unchangedCount} unchanged)` : "";
+	ctx.progress.done(`ingested ${okCount}/${results.length}${suffix}`);
 
 	return summarize(results);
 }
@@ -428,12 +455,14 @@ export function parseDuration(input: string | null | undefined): number | null {
 /** Roll a list of per-entry results into the top-level summary shape. */
 function summarize(entries: IngestEntryResult[]): IngestResult {
 	let ok = 0;
+	let unchanged = 0;
 	let failed = 0;
 	for (const e of entries) {
 		if (e.status === "ok") ok += 1;
+		else if (e.status === "unchanged") unchanged += 1;
 		else failed += 1;
 	}
-	return { ingested: entries, total: entries.length, ok, failed };
+	return { ingested: entries, total: entries.length, ok, unchanged, failed };
 }
 
 function errorMessage(err: unknown): string {

package/src/ingest/source-resolver.ts

@@ -43,10 +43,12 @@ export async function resolveSource(source: string, options: ResolveOptions = {}
 	}
 
 	const followSymlinks = options.followSymlinks !== false;
-	const includeMatchers = (options.include ?? "**/*")
-		.split(",")
-		.map((g) => g.trim())
-		.filter(Boolean);
+	const userIncludes = options.include
+		? options.include
+				.split(",")
+				.map((g) => g.trim())
+				.filter(Boolean)
+		: [];
 	const excludeMatchers = [
 		...DEFAULT_EXCLUDES,
 		...(options.exclude ?? "")
@@ -57,9 +59,14 @@ export async function resolveSource(source: string, options: ResolveOptions = {}
 
 	if (isGlob(source)) {
 		const base = globBase(source);
+		const remainder = globRemainder(source);
 		try {
 			const realBase = await realpath(base);
-			return walk(realBase, [source, ...includeMatchers], excludeMatchers, followSymlinks);
+			// Source glob acts as a hard filter; user includes (if any) further
+			// narrow the result via AND. Pass them as a separate matcher so the
+			// two sets aren't picomatch-OR'd together.
+			const extraIncludes = userIncludes.length > 0 ? [userIncludes] : [];
+			return walk(realBase, [remainder], excludeMatchers, followSymlinks, extraIncludes);
 		} catch (err) {
 			throw asHelpful(
 				err,
@@ -93,7 +100,8 @@ export async function resolveSource(source: string, options: ResolveOptions = {}
 
 	if (st.isDirectory()) {
 		const realBase = await realpath(abs);
-		return walk(realBase, includeMatchers, excludeMatchers, followSymlinks);
+		const dirIncludes = userIncludes.length > 0 ? userIncludes : ["**/*"];
+		return walk(realBase, dirIncludes, excludeMatchers, followSymlinks);
 	}
 
 	throw new HelpfulError({
@@ -120,22 +128,40 @@ export function globBase(glob: string): string {
 	return base.length === 0 || !isAbsolute(base) ? resolve(base || ".") : base;
 }
 
+/**
+ * Take the wildcard portion of a glob — everything from the first segment
+ * containing a wildcard onward. We strip the static prefix so the matcher
+ * runs against entry paths relative to `globBase`. Without this, a glob like
+ * `docs/star-star/star.md` never matches anything under base=`docs/`, since
+ * walk() exposes `sub/file.md` to picomatch, not `docs/sub/file.md`.
+ */
+export function globRemainder(glob: string): string {
+	const parts = glob.split(sep);
+	const wildcardIdx = parts.findIndex((p) => /[*?[\]{}!]/.test(p));
+	if (wildcardIdx === -1) return glob;
+	return parts.slice(wildcardIdx).join(sep);
+}
+
 /**
  * Recursively walk `base`, returning files matched by `includes` and not
  * matched by `excludes`. Both globsets match against the entry's path
  * relative to `base`. Symlinks are followed when `followSymlinks` is true,
- * with cycles detected via a realpath cache.
+ * with cycles detected via a realpath cache. `extraIncludeSets` is a list
+ * of additional include groups, each ANDed onto the primary `includes` —
+ * use it when two filters must both match (e.g. source glob + --include).
  */
 async function walk(
 	base: string,
 	includes: string[],
 	excludes: string[],
 	followSymlinks: boolean,
+	extraIncludeSets: string[][] = [],
 ): Promise<ResolvedSource> {
 	const seen = new Set<string>();
 	const entries: ResolvedLocalEntry[] = [];
 
 	const isInclude = picomatch(includes, { dot: false, nocase: false });
+	const extraMatchers = extraIncludeSets.map((set) => picomatch(set, { dot: false, nocase: false }));
 	const isExclude = excludes.length ? picomatch(excludes, { dot: false }) : null;
 
 	const queue: string[] = [base];
@@ -174,6 +200,7 @@ async function walk(
 		const relForMatch = rel.length === 0 ? (cur.split(sep).pop() ?? cur) : rel;
 		if (isExclude?.(relForMatch)) continue;
 		if (!isInclude(relForMatch)) continue;
+		if (extraMatchers.some((m) => !m(relForMatch))) continue;
 		entries.push({ absPath: real, relPath: relForMatch });
 	}
 

package/src/operations/add.ts

@@ -14,11 +14,16 @@ export const addOperation = defineOperation({
   - a glob pattern (e.g. "docs/**/*.md")
   - a URL (fetched via mcpx if configured, otherwise plain HTTP)
   - "inline:<text>" literal
-PDF, DOCX, HTML, images, and other binaries are converted to markdown — native libraries first, vision/OCR for images, LLM fallback for messy or scanned input. Original bytes are kept in the blobs table; \`membot_read bytes=true\` returns them. Setting \`refresh_frequency\` enables automatic refresh from the daemon. Each ingested file becomes a NEW version under its own logical_path; existing versions stay queryable via membot_versions. Directory/glob ingests stream one file at a time — partial failures do not abort the rest; the response lists per-entry status.`,
+PDF, DOCX, HTML, images, and other binaries are converted to markdown — native libraries first, vision/OCR for images, LLM fallback for messy or scanned input. Original bytes are kept in the blobs table; \`membot_read bytes=true\` returns them. Setting \`refresh_frequency\` enables automatic refresh from the daemon. By default, re-ingesting an unchanged source (same source_sha256 as the current version) is a no-op and reports \`status: "unchanged"\`; pass \`force=true\` to always create a new version. Each newly-ingested file becomes a new version under its own logical_path; existing versions stay queryable via membot_versions. Directory/glob ingests stream one file at a time — partial failures do not abort the rest; the response lists per-entry status.`,
 	inputSchema: z.object({
 		source: z.string().describe("Local path, directory, glob, URL, or `inline:<text>` literal"),
 		logical_path: z.string().optional().describe("Destination logical_path (single source) or prefix (directory/glob)"),
-		include: z.string().optional().describe("Glob include filter (comma-separated for multiple); default `**/*`"),
+		include: z
+			.string()
+			.optional()
+			.describe(
+				"Glob include filter (comma-separated for multiple). Defaults to `**/*` for directory sources, or the source pattern itself when source is a glob.",
+			),
 		exclude: z.string().optional().describe("Glob exclude filter (comma-separated for multiple)"),
 		follow_symlinks: z
 			.boolean()
@@ -30,6 +35,10 @@ PDF, DOCX, HTML, images, and other binaries are converted to markdown — native
 			.optional()
 			.describe("Free-form hint passed to mcpx tool search (e.g. 'firecrawl', 'github', 'google docs', 'http')"),
 		change_note: z.string().optional().describe("Free-text note attached to the new version"),
+		force: z
+			.boolean()
+			.optional()
+			.describe("Re-ingest even when source bytes are unchanged. Default skips and reports `unchanged`."),
 	}),
 	outputSchema: z.object({
 		ingested: z.array(
@@ -37,7 +46,7 @@ PDF, DOCX, HTML, images, and other binaries are converted to markdown — native
 				source_path: z.string(),
 				logical_path: z.string(),
 				version_id: z.string().nullable(),
-				status: z.enum(["ok", "failed"]),
+				status: z.enum(["ok", "unchanged", "failed"]),
 				error: z.string().optional(),
 				mime_type: z.string().nullable(),
 				size_bytes: z.number(),
@@ -47,23 +56,27 @@ PDF, DOCX, HTML, images, and other binaries are converted to markdown — native
 		),
 		total: z.number(),
 		ok: z.number(),
+		unchanged: z.number(),
 		failed: z.number(),
 	}),
 	cli: {
 		positional: ["source"],
-		aliases: { logical_path: "-p", refresh_frequency: "-r", change_note: "-m" },
+		aliases: { logical_path: "-p", refresh_frequency: "-r", change_note: "-m", force: "-f" },
 	},
 	console_formatter: (result) => {
 		const lines = result.ingested.map((e) => {
 			if (e.status === "ok") {
 				return `${colors.green("✓")} ${colors.cyan(e.logical_path)} ${colors.dim(`(${e.fetcher}, ${e.size_bytes}B)`)}`;
 			}
+			if (e.status === "unchanged") {
+				return `${colors.dim("≡")} ${colors.cyan(e.logical_path)} ${colors.dim("(unchanged)")}`;
+			}
 			return `${colors.red("✗")} ${e.source_path} ${colors.dim(e.error ?? "")}`;
 		});
-		const summary = result.failed
-			? `${colors.green(`added ${result.ok}`)}, ${colors.red(`failed ${result.failed}`)}`
-			: colors.green(`added ${result.ok}`);
-		return `${lines.join("\n")}\n${summary}`;
+		const parts: string[] = [colors.green(`added ${result.ok}`)];
+		if (result.unchanged > 0) parts.push(colors.dim(`unchanged ${result.unchanged}`));
+		if (result.failed > 0) parts.push(colors.red(`failed ${result.failed}`));
+		return `${lines.join("\n")}\n${parts.join(", ")}`;
 	},
 	handler: async (input, ctx) => ingest(input, ctx),
 });

package/scripts/apply-transformers-patch.sh

@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Apply the @huggingface/transformers patch to node_modules so that
-# `bun build --compile` produces a binary using the WASM backend
-# (onnxruntime-web) instead of onnxruntime-node, whose native bindings
-# can't be bundled into a single-binary distribution.
-#
-# We apply the patch imperatively (rather than via package.json
-# `patchedDependencies`) because that field, when present in a
-# published package, breaks `bun install` from a tarball.
-
-PATCH="patches/@huggingface%2Ftransformers@4.2.0.patch"
-TARGET="node_modules/@huggingface/transformers"
-MARKER="$TARGET/.membot-transformers-patch-applied"
-
-if [ ! -d "$TARGET" ]; then
-	echo "error: $TARGET not found — run \`bun install\` first" >&2
-	exit 1
-fi
-
-if [ ! -f "$PATCH" ]; then
-	echo "error: $PATCH not found" >&2
-	exit 1
-fi
-
-if [ -f "$MARKER" ]; then
-	echo "transformers patch already applied — skipping"
-	exit 0
-fi
-
-echo "Applying transformers patch ($PATCH) to $TARGET..."
-git apply --directory="$TARGET" "$PATCH"
-touch "$MARKER"
-echo "Patch applied."