membot 0.1.2 → 0.2.1

package/.claude/skills/membot.md

@@ -36,6 +36,15 @@ membot add ./docs --refresh-frequency 24h         # auto-refresh every day
 
 Each entry becomes a new version under its own `logical_path`. PDFs/DOCX/HTML are converted to markdown; images get vision captions; original bytes are kept and reachable via `membot read --bytes`.
 
+The default `logical_path` mirrors the source path so files with the same basename in different projects don't collide:
+
+- Local file → absolute path with leading `/` stripped (e.g. `/Users/me/projA/README.md` → `Users/me/projA/README.md`).
+- Local directory or glob → each entry's absolute path under the same shape.
+- URL → `remotes/{host}/{path}` with `/`'s preserved (e.g. `https://github.com/userA/projA/blob/main/README.md` → `remotes/github.com/userA/projA/blob/main/README.md`). Query strings and fragments are dropped from the logical_path (the full URL is still stored for refresh).
+- `inline:<text>` → `inline/{timestamp}.md`.
+
+Pass `-p <path>` (or `--logical-path`) to override. On a directory walk it's treated as a *prefix* — entries land at `{prefix}/{path-relative-to-walk-base}`. Re-running `membot add` on the same source reuses the same logical_path and creates a new version (correct refresh behavior).
+
 ## 3. Read
 
 ```bash
@@ -101,7 +110,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 | ------------------------------------- | ------------------------------------------------------------------------------ |
 | `membot add <source>`                 | Ingest file, directory, glob, URL, or `inline:<text>`. Skips unchanged sources; pass `--force` to re-ingest |
 | `membot ls [prefix]`                  | List current files (size, mime, refresh status)                                |
-| `membot tree [prefix]`                | Render the synthesised logical-path tree                                       |
+| `membot tree [prefix]`                | Render the synthesised logical-path tree (`--max-depth`, `--max-items` cap output) |
 | `membot read <path>`                  | Read current markdown surrogate (or `--bytes` for original)                    |
 | `membot write <path> --content <txt>` | Write inline agent-authored markdown as a new version                          |
 | `membot search <query>`               | Hybrid search (semantic + BM25); add `--include-history` to search older versions |

package/.cursor/rules/membot.mdc

@@ -36,6 +36,15 @@ membot add ./docs --refresh-frequency 24h         # auto-refresh every day
 
 Each entry becomes a new version under its own `logical_path`. PDFs/DOCX/HTML are converted to markdown; images get vision captions; original bytes are kept and reachable via `membot read --bytes`.
 
+The default `logical_path` mirrors the source path so files with the same basename in different projects don't collide:
+
+- Local file → absolute path with leading `/` stripped (e.g. `/Users/me/projA/README.md` → `Users/me/projA/README.md`).
+- Local directory or glob → each entry's absolute path under the same shape.
+- URL → `remotes/{host}/{path}` with `/`'s preserved (e.g. `https://github.com/userA/projA/blob/main/README.md` → `remotes/github.com/userA/projA/blob/main/README.md`). Query strings and fragments are dropped from the logical_path (the full URL is still stored for refresh).
+- `inline:<text>` → `inline/{timestamp}.md`.
+
+Pass `-p <path>` (or `--logical-path`) to override. On a directory walk it's treated as a *prefix* — entries land at `{prefix}/{path-relative-to-walk-base}`. Re-running `membot add` on the same source reuses the same logical_path and creates a new version (correct refresh behavior).
+
 ## 3. Read
 
 ```bash
@@ -101,7 +110,7 @@ Tombstones hide a path from `ls` / `tree` / `search` but `versions` and `read --
 | ------------------------------------- | ------------------------------------------------------------------------------ |
 | `membot add <source>`                 | Ingest file, directory, glob, URL, or `inline:<text>`. Skips unchanged sources; pass `--force` to re-ingest |
 | `membot ls [prefix]`                  | List current files (size, mime, refresh status)                                |
-| `membot tree [prefix]`                | Render the synthesised logical-path tree                                       |
+| `membot tree [prefix]`                | Render the synthesised logical-path tree (`--max-depth`, `--max-items` cap output) |
 | `membot read <path>`                  | Read current markdown surrogate (or `--bytes` for original)                    |
 | `membot write <path> --content <txt>` | Write inline agent-authored markdown as a new version                          |
 | `membot search <query>`               | Hybrid search (semantic + BM25); add `--include-history` to search older versions |

package/README.md

@@ -50,9 +50,9 @@ The skill files describe the discover → ingest → search → read → write w
 
 | Command                         | Description                                                                       |
 | ------------------------------- | --------------------------------------------------------------------------------- |
-| `membot add <source>`           | Ingest a file, directory, glob, URL, or `inline:<text>`. Skips on unchanged source bytes; pass `--force` to re-ingest |
+| `membot add <source>`           | Ingest a file, directory, glob, URL, or `inline:<text>`. Default `logical_path` mirrors the source (absolute path for local files, `remotes/{host}/{path}` for URLs) so files with the same basename in different projects don't collide. Pass `-p <path>` to override or, on a directory walk, to set a prefix. Skips on unchanged source bytes; pass `--force` to re-ingest. |
 | `membot ls [prefix]`            | List current files (size, mime, refresh status)                                   |
-| `membot tree [prefix]`          | Render the synthesised logical-path tree                                          |
+| `membot tree [prefix]`          | Render the synthesised logical-path tree (`--max-depth`, `--max-items` cap output) |
 | `membot read <path>`            | Read the markdown surrogate (or `--bytes` for original bytes, base64)             |
 | `membot search <query>`         | Hybrid search (semantic + BM25); `--include-history` searches older versions      |
 | `membot info <path>`            | Inspect metadata (source, fetcher, schedule, digests) without content             |

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "membot",
-	"version": "0.1.2",
+	"version": "0.2.1",
 	"description": "Versioned context store with hybrid search for AI agents. Stdio + HTTP MCP server and CLI.",
 	"type": "module",
 	"exports": {

package/src/ingest/ingest.ts

@@ -209,7 +209,7 @@ async function ingestLocalFiles(
 	const isMulti = resolved.entries.length > 1;
 
 	for (const entry of resolved.entries) {
-		ctx.progress.tick(entry.relPath);
+		ctx.progress.tick(entry.relPathFromBase);
 		const logicalPath = pickLogicalPath(input.logical_path, entry, isMulti);
 		const result: IngestEntryResult = {
 			source_path: entry.absPath,
@@ -404,26 +404,47 @@ async function persistVersion(ctx: AppContext, p: PersistParams): Promise<string
 }
 
 /**
- * Pick the logical path for a single matched entry. For a single-file
- * ingest with explicit `logical_path`, use it as-is. For multi-entry
- * ingests with `logical_path` set, treat it as a *prefix* under which
- * each entry's relative path is placed.
+ * Pick the logical path for a single matched entry.
+ *
+ * - Default (no explicit logical_path): use the entry's absolute filesystem
+ *   path with `\` normalized to `/` and the leading `/` stripped. This
+ *   keeps `~/projA/README.md` and `~/projB/README.md` from colliding under
+ *   a shared `README.md`. Two adds of the same absolute path produce the
+ *   same logical_path, so the second add correctly creates a new version.
+ * - Single-source with explicit logical_path: use it verbatim.
+ * - Multi-entry (directory/glob) with explicit logical_path: treat as a
+ *   prefix and append each entry's path relative to the walk base.
  */
-function pickLogicalPath(explicit: string | undefined, entry: ResolvedLocalEntry, isMulti: boolean): string {
-	if (!explicit) return entry.relPath.replaceAll("\\", "/");
+export function pickLogicalPath(explicit: string | undefined, entry: ResolvedLocalEntry, isMulti: boolean): string {
+	if (!explicit) return normalizeAbs(entry.absPath);
 	if (!isMulti) return explicit;
 	const prefix = explicit.endsWith("/") ? explicit.slice(0, -1) : explicit;
-	return `${prefix}/${entry.relPath.replaceAll("\\", "/")}`;
+	return `${prefix}/${entry.relPathFromBase.replaceAll("\\", "/")}`;
 }
 
-/** Default logical path for an ingested URL — host + path, sanitized. */
-function defaultLogicalForUrl(url: string): string {
+/**
+ * Normalize an absolute filesystem path into a logical_path:
+ * `\` → `/`, leading `/` stripped. Drive letters (Windows `C:`) are kept
+ * as the first path segment.
+ */
+export function normalizeAbs(absPath: string): string {
+	return absPath.replaceAll("\\", "/").replace(/^\/+/, "");
+}
+
+/**
+ * Default logical path for an ingested URL: `remotes/{host}/{pathname}`
+ * with slashes preserved so two projects on the same host (e.g.,
+ * github.com) don't collide. Query string and fragment are dropped from
+ * the logical_path for stable identity — the full URL is still preserved
+ * on the row in `source_path` and used for refresh.
+ */
+export function defaultLogicalForUrl(url: string): string {
 	try {
 		const u = new URL(url);
-		const tail = u.pathname.replace(/^\/+/, "").replaceAll("/", "_") || "root";
-		return `urls/${u.hostname}/${tail || "root"}`;
+		const tail = u.pathname.replace(/^\/+/, "").replace(/\/+$/, "") || "index";
+		return `remotes/${u.hostname}/${tail}`;
 	} catch {
-		return `urls/${url.replace(/[^a-z0-9.-]/gi, "_")}`;
+		return `remotes/${url.replace(/[^a-z0-9.-]/gi, "_")}`;
 	}
 }
 

package/src/ingest/source-resolver.ts

@@ -9,9 +9,15 @@ export type ResolvedSource =
 	| { kind: "local-files"; entries: ResolvedLocalEntry[]; basePath: string };
 
 export interface ResolvedLocalEntry {
+	/** Absolute filesystem path (post-realpath). */
 	absPath: string;
-	/** Path relative to the base; used to derive a default logical_path. */
-	relPath: string;
+	/**
+	 * Path relative to the walk base. Used when the caller passes an
+	 * explicit `logical_path` *prefix* (directory/glob mode) — entries land
+	 * at `{prefix}/{relPathFromBase}`. For default logical_paths we use
+	 * `absPath` directly so paths from different filesystems don't collide.
+	 */
+	relPathFromBase: string;
 }
 
 export interface ResolveOptions {
@@ -91,10 +97,11 @@ export async function resolveSource(source: string, options: ResolveOptions = {}
 	}
 
 	if (st.isFile()) {
+		const real = await realpath(abs);
 		return {
 			kind: "local-files",
-			basePath: abs,
-			entries: [{ absPath: abs, relPath: source.split(sep).pop() ?? source }],
+			basePath: real,
+			entries: [{ absPath: real, relPathFromBase: real.split(sep).pop() ?? real }],
 		};
 	}
 
@@ -201,7 +208,7 @@ async function walk(
 		if (isExclude?.(relForMatch)) continue;
 		if (!isInclude(relForMatch)) continue;
 		if (extraMatchers.some((m) => !m(relForMatch))) continue;
-		entries.push({ absPath: real, relPath: relForMatch });
+		entries.push({ absPath: real, relPathFromBase: relForMatch });
 	}
 
 	return { kind: "local-files", basePath: base, entries };

package/src/operations/add.ts

@@ -14,7 +14,14 @@ export const addOperation = defineOperation({
   - a glob pattern (e.g. "docs/**/*.md")
   - a URL (fetched via mcpx if configured, otherwise plain HTTP)
   - "inline:<text>" literal
-PDF, DOCX, HTML, images, and other binaries are converted to markdown — native libraries first, vision/OCR for images, LLM fallback for messy or scanned input. Original bytes are kept in the blobs table; \`membot_read bytes=true\` returns them. Setting \`refresh_frequency\` enables automatic refresh from the daemon. By default, re-ingesting an unchanged source (same source_sha256 as the current version) is a no-op and reports \`status: "unchanged"\`; pass \`force=true\` to always create a new version. Each newly-ingested file becomes a new version under its own logical_path; existing versions stay queryable via membot_versions. Directory/glob ingests stream one file at a time — partial failures do not abort the rest; the response lists per-entry status.`,
+PDF, DOCX, HTML, images, and other binaries are converted to markdown — native libraries first, vision/OCR for images, LLM fallback for messy or scanned input. Original bytes are kept in the blobs table; \`membot_read bytes=true\` returns them. Setting \`refresh_frequency\` enables automatic refresh from the daemon. By default, re-ingesting an unchanged source (same source_sha256 as the current version) is a no-op and reports \`status: "unchanged"\`; pass \`force=true\` to always create a new version. Each newly-ingested file becomes a new version under its own logical_path; existing versions stay queryable via membot_versions. Directory/glob ingests stream one file at a time — partial failures do not abort the rest; the response lists per-entry status.
+
+When \`logical_path\` is omitted, it is derived from the source so files with the same basename in different projects do not collide:
+  - Local sources use the entry's absolute filesystem path with the leading "/" stripped (e.g. "/Users/me/projA/README.md" → "Users/me/projA/README.md").
+  - URLs use "remotes/{host}/{path}" with slashes preserved (e.g. "https://github.com/u/p/blob/main/README.md" → "remotes/github.com/u/p/blob/main/README.md"). Query strings and fragments are dropped from the logical_path; the full URL is still stored on the row for refresh.
+  - "inline:<text>" defaults to "inline/{timestamp}.md".
+
+Pass \`logical_path\` to override. For a directory or glob walk it is treated as a PREFIX — each entry is placed at "{prefix}/{path-relative-to-walk-base}". Re-running \`membot_add\` on the same source resolves to the same logical_path; if bytes are unchanged the call is a no-op (status \`unchanged\`), otherwise a new version is created.`,
 	inputSchema: z.object({
 		source: z.string().describe("Local path, directory, glob, URL, or `inline:<text>` literal"),
 		logical_path: z.string().optional().describe("Destination logical_path (single source) or prefix (directory/glob)"),

package/src/operations/tree.ts

@@ -8,6 +8,7 @@ interface TreeNode {
 	full_path: string;
 	is_file: boolean;
 	children?: TreeNode[];
+	children_truncated?: number;
 }
 
 export const treeOperation = defineOperation({
@@ -18,6 +19,10 @@ export const treeOperation = defineOperation({
 	inputSchema: z.object({
 		prefix: z.string().optional().describe("Only show paths starting with this prefix"),
 		max_depth: z.number().default(4).describe("How many path segments deep to render"),
+		max_items: z
+			.number()
+			.default(20)
+			.describe("Max children to render at each level; remainder is summarised as '+N more'"),
 	}),
 	outputSchema: z.object({
 		root: z.string(),
@@ -27,77 +32,116 @@ export const treeOperation = defineOperation({
 				full_path: z.string(),
 				is_file: z.boolean(),
 				children: z.array(z.unknown()).optional(),
+				children_truncated: z.number().optional(),
 			}),
 		),
+		truncated: z.number().optional(),
 	}),
 	cli: { positional: ["prefix"] },
 	console_formatter: (result) => {
 		const lines: string[] = [colors.bold(result.root)];
 		const nodes = result.tree as TreeNode[];
-		renderNodes(nodes, "", lines);
+		const topTruncated = (result as { truncated?: number }).truncated ?? 0;
+		renderNodes(nodes, "", lines, topTruncated);
 		if (lines.length === 1) lines.push(colors.dim("(empty)"));
 		return lines.join("\n");
 	},
 	handler: async (input, ctx) => {
 		const allPaths = await listAllCurrentPaths(ctx.db);
 		const filtered = input.prefix ? allPaths.filter((p) => p.startsWith(input.prefix!)) : allPaths;
-		return { root: input.prefix ?? "/", tree: buildTree(filtered, input.max_depth) };
+		const tree = buildTree(filtered, input.max_depth);
+		const truncated = truncateTree(tree, input.max_items);
+		return {
+			root: input.prefix ?? "/",
+			tree,
+			...(truncated > 0 ? { truncated } : {}),
+		};
 	},
 });
 
 /**
  * Build a tree of TreeNode objects from a flat list of `/`-delimited paths.
- * Splits each path into segments and groups by common prefix; nodes deeper
- * than `maxDepth` are folded into their parent's `children` summary count.
+ * Splits each path into segments and groups by common prefix. Segments
+ * deeper than `maxDepth` are folded into the deepest visible ancestor —
+ * that ancestor is marked `is_file=true` so the renderer surfaces it as a
+ * leaf even though longer paths exist underneath. Children are sorted by
+ * name within each level so downstream truncation is deterministic.
  */
-function buildTree(paths: string[], maxDepth: number): TreeNode[] {
-	const root: Map<string, TreeNode> = new Map();
+export function buildTree(paths: string[], maxDepth: number): TreeNode[] {
+	interface MutableNode {
+		name: string;
+		full_path: string;
+		is_file: boolean;
+		children: Map<string, MutableNode>;
+	}
+	const root = new Map<string, MutableNode>();
 	for (const path of paths) {
 		const segs = path.split("/").filter(Boolean);
+		if (segs.length === 0) continue;
 		let level = root;
 		const trail: string[] = [];
-		for (let i = 0; i < segs.length && i < maxDepth; i++) {
+		const stop = Math.min(segs.length, maxDepth);
+		for (let i = 0; i < stop; i++) {
 			const seg = segs[i]!;
 			trail.push(seg);
-			const fullPath = trail.join("/");
 			let node = level.get(seg);
 			if (!node) {
-				node = { name: seg, full_path: fullPath, is_file: i === segs.length - 1 };
+				node = { name: seg, full_path: trail.join("/"), is_file: false, children: new Map() };
 				level.set(seg, node);
-			} else if (i === segs.length - 1) {
-				node.is_file = true;
-			}
-			if (i < segs.length - 1) {
-				if (!node.children) node.children = [];
-				const childMap = new Map(node.children.map((c) => [c.name, c] as const));
-				node.children = [...childMap.values()];
-				level = childMap;
-				if (childMap.size === 0) {
-					level = new Map();
-					node.children = [];
-				} else {
-					// rebuild level pointer
-					level = new Map(node.children.map((c) => [c.name, c] as const));
-				}
 			}
+			const isTerminal = i === segs.length - 1 || i === maxDepth - 1;
+			if (isTerminal) node.is_file = true;
+			level = node.children;
 		}
 	}
-	return [...root.values()].sort((a, b) => a.name.localeCompare(b.name));
+	const finalize = (m: Map<string, MutableNode>): TreeNode[] => {
+		const arr = [...m.values()].sort((a, b) => a.name.localeCompare(b.name));
+		return arr.map((n) => {
+			const out: TreeNode = { name: n.name, full_path: n.full_path, is_file: n.is_file };
+			if (n.children.size > 0) out.children = finalize(n.children);
+			return out;
+		});
+	};
+	return finalize(root);
+}
+
+/**
+ * Trim each child list (and the root list) to `maxItems`, mutating in place.
+ * Returns the number of root entries dropped; per-node drops are recorded on
+ * `node.children_truncated`. Input is assumed pre-sorted (by `buildTree`) so
+ * "first N" is stable.
+ */
+export function truncateTree(nodes: TreeNode[], maxItems: number): number {
+	for (const node of nodes) {
+		if (node.children?.length) {
+			const dropped = truncateTree(node.children, maxItems);
+			if (dropped > 0) node.children_truncated = dropped;
+		}
+	}
+	if (nodes.length > maxItems) {
+		const dropped = nodes.length - maxItems;
+		nodes.length = maxItems;
+		return dropped;
+	}
+	return 0;
 }
 
 /**
  * Walk a tree and append `├── name` / `└── name` lines with proper continuation
- * prefixes. Directories are rendered in cyan-bold; files in plain text.
+ * prefixes. Directories are rendered in cyan-bold; files in plain text. When a
+ * level was truncated, a dim trailing `+N more` line is appended at that level.
  */
-function renderNodes(nodes: TreeNode[], prefix: string, out: string[]): void {
-	const sorted = [...nodes].sort((a, b) => a.name.localeCompare(b.name));
-	sorted.forEach((node, i) => {
-		const last = i === sorted.length - 1;
+function renderNodes(nodes: TreeNode[], prefix: string, out: string[], truncatedCount = 0): void {
+	nodes.forEach((node, i) => {
+		const last = i === nodes.length - 1 && truncatedCount === 0;
 		const branch = last ? "└── " : "├── ";
 		const label = node.is_file && !node.children?.length ? node.name : colors.cyan(colors.bold(node.name));
 		out.push(`${prefix}${branch}${label}`);
 		if (node.children?.length) {
-			renderNodes(node.children, prefix + (last ? "    " : "│   "), out);
+			renderNodes(node.children, prefix + (last ? "    " : "│   "), out, node.children_truncated ?? 0);
 		}
 	});
+	if (truncatedCount > 0) {
+		out.push(`${prefix}└── ${colors.dim(`+${truncatedCount} more`)}`);
+	}
 }