mdcontext 0.1.0 → 0.2.0

package/.changeset/config.json

@@ -1,11 +1,11 @@
 {
-  "$schema": "https://unpkg.com/@changesets/config@3.0.0/schema.json",
-  "changelog": ["@changesets/changelog-github", { "repo": "alphab/mdcontext" }],
-  "commit": false,
-  "fixed": [],
-  "linked": [],
-  "access": "public",
-  "baseBranch": "main",
-  "updateInternalDependencies": "patch",
-  "ignore": []
+	"$schema": "https://unpkg.com/@changesets/config@3.0.0/schema.json",
+	"changelog": ["@changesets/changelog-github", { "repo": "alphab/mdcontext" }],
+	"commit": false,
+	"fixed": [],
+	"linked": [],
+	"access": "public",
+	"baseBranch": "main",
+	"updateInternalDependencies": "patch",
+	"ignore": []
 }

package/.claude/settings.local.json

@@ -0,0 +1,25 @@
+{
+	"permissions": {
+		"allow": ["mcp__linear-server", "Read", "Edit", "Write", "Bash", "WebFetch"]
+	},
+	"mcpServers": {
+		"linear-server": {
+			"type": "http",
+			"url": "https://mcp.linear.app/mcp"
+		}
+	},
+	"hooks": {
+		"PreToolUse": [
+			{
+				"matcher": "WebFetch|Read|Write|Edit",
+				"hooks": [
+					{
+						"type": "command",
+						"command": "/Users/alphab/Dev/LLM/DEV/TMP/nancy/skills/check-directives/check-directives.sh"
+					}
+				]
+			}
+		]
+	},
+	"outputStyle": "default"
+}

package/.github/workflows/claude-code-review.yml

@@ -0,0 +1,44 @@
+name: Claude Code Review
+
+on:
+  pull_request:
+    branches: [main]
+    # Optional: Only run on specific file changes
+    # paths:
+    #   - "src/**/*.ts"
+    #   - "src/**/*.tsx"
+    #   - "src/**/*.js"
+    #   - "src/**/*.jsx"
+
+jobs:
+  claude-review:
+    # Optional: Filter by PR author
+    # if: |
+    #   github.event.pull_request.user.login == 'external-contributor' ||
+    #   github.event.pull_request.user.login == 'new-developer' ||
+    #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code Review
+        id: claude-review
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
+          plugins: 'code-review@claude-code-plugins'
+          prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+

package/.github/workflows/claude.yml

@@ -0,0 +1,85 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (
+        github.event_name == 'issue_comment' &&
+        contains(github.event.comment.body, '@claude') &&
+        (
+          github.event.comment.author_association == 'MEMBER' ||
+          github.event.comment.author_association == 'OWNER' ||
+          github.event.comment.author_association == 'COLLABORATOR'
+        )
+      ) ||
+      (
+        github.event_name == 'pull_request_review_comment' &&
+        contains(github.event.comment.body, '@claude') &&
+        (
+          github.event.comment.author_association == 'MEMBER' ||
+          github.event.comment.author_association == 'OWNER' ||
+          github.event.comment.author_association == 'COLLABORATOR'
+        )
+      ) ||
+      (
+        github.event_name == 'pull_request_review' &&
+        contains(github.event.review.body, '@claude') &&
+        (
+          github.event.review.author_association == 'MEMBER' ||
+          github.event.review.author_association == 'OWNER' ||
+          github.event.review.author_association == 'COLLABORATOR'
+        )
+      ) ||
+      (
+        github.event_name == 'issues' &&
+        (
+          contains(github.event.issue.body, '@claude') ||
+          contains(github.event.issue.title, '@claude')
+        ) &&
+        (
+          github.event.issue.author_association == 'MEMBER' ||
+          github.event.issue.author_association == 'OWNER' ||
+          github.event.issue.author_association == 'COLLABORATOR'
+        )
+      )
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+      actions: read # Required for Claude to read CI results on PRs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
+          additional_permissions: |
+            actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+          # claude_args: '--allowed-tools Bash(gh pr:*)'
+

package/CONTRIBUTING.md

@@ -0,0 +1,186 @@
+# Contributing to mdcontext
+
+Thank you for your interest in contributing to mdcontext! This guide will help you get started.
+
+## Development Setup
+
+### Prerequisites
+
+- Node.js >= 18.0.0
+- pnpm (recommended, version 10+)
+- Python 3.12+ (for native module compilation)
+
+### Getting Started
+
+```bash
+# Clone the repository
+git clone https://github.com/mdcontext/mdcontext.git
+cd mdcontext
+
+# Install dependencies
+pnpm install
+
+# Build the project
+pnpm build
+
+# Run tests
+pnpm test
+```
+
+### Available Commands
+
+| Command | Description |
+|---------|-------------|
+| `pnpm build` | Build the project |
+| `pnpm test` | Run the test suite |
+| `pnpm test:all` | Run tests including semantic search tests (requires `OPENAI_API_KEY`) |
+| `pnpm typecheck` | Run TypeScript type checking |
+| `pnpm lint` | Run Biome linter |
+| `pnpm format` | Format code with Biome |
+| `pnpm check` | Run format, lint, and typecheck |
+
+## Making Changes
+
+1. **Create a branch** for your changes:
+   ```bash
+   git checkout -b feature/my-awesome-feature
+   ```
+
+2. **Make your changes** and ensure:
+   - All tests pass: `pnpm test`
+   - Type checking passes: `pnpm typecheck`
+   - Code is formatted: `pnpm format`
+
+3. **Create a changeset** (see below)
+
+4. **Submit a pull request** to the `main` branch
+
+## Changeset Workflow
+
+We use [Changesets](https://github.com/changesets/changesets) to manage versions and releases. **Every PR that affects the published package should include a changeset.**
+
+### Creating a Changeset
+
+After making your changes, run:
+
+```bash
+pnpm changeset
+```
+
+This will prompt you to:
+1. Select the package (mdcontext)
+2. Choose a bump type (patch/minor/major)
+3. Write a summary of your changes
+
+A markdown file will be created in the `.changeset/` directory. **Commit this file with your PR.**
+
+### Choosing a Bump Type
+
+| Type | When to use | Example |
+|------|-------------|---------|
+| **patch** | Bug fixes, documentation updates, internal refactors | Fixing a search result formatting bug |
+| **minor** | New features that are backwards compatible | Adding a new CLI flag or command |
+| **major** | Breaking changes to public API or CLI | Changing command syntax, removing flags |
+
+#### Examples
+
+**Patch** (bug fix):
+```markdown
+---
+"mdcontext": patch
+---
+
+Fixed section extraction to correctly handle nested headings
+```
+
+**Minor** (new feature):
+```markdown
+---
+"mdcontext": minor
+---
+
+Added --json flag to context command for structured output
+```
+
+**Major** (breaking change):
+```markdown
+---
+"mdcontext": major
+---
+
+Changed search command syntax: path argument now comes before query
+
+Migration: `mdcontext search "query" path/` becomes `mdcontext search path/ "query"`
+```
+
+### When You Don't Need a Changeset
+
+- Documentation-only changes (README, comments)
+- Test-only changes
+- CI/tooling changes that don't affect the package
+- Internal refactors with no user-facing changes
+
+If the changesets bot comments on your PR asking for a changeset and you believe one isn't needed, you can add an empty changeset:
+
+```bash
+pnpm changeset add --empty
+```
+
+## Release Process
+
+Here's what happens after your PR is merged:
+
+1. **Changesets are collected**: The release workflow detects changeset files in `main`
+
+2. **Version Packages PR is created**: GitHub Actions creates a PR titled "chore: release packages" that:
+   - Bumps the version in `package.json`
+   - Updates `CHANGELOG.md` with your changeset descriptions
+   - Deletes the changeset files
+
+3. **Publish on merge**: When a maintainer merges the "Version Packages" PR:
+   - The package is automatically published to npm
+   - A GitHub release is created
+   - Package provenance is attached via npm's OIDC support
+
+### For Maintainers
+
+To trigger a release:
+1. Review the auto-generated "Version Packages" PR
+2. Verify the version bump and changelog look correct
+3. Merge the PR
+4. The release happens automatically
+
+## Code Style
+
+- We use [Biome](https://biomejs.dev/) for formatting and linting
+- Run `pnpm format` before committing
+- TypeScript strict mode is enabled
+- Prefer functional patterns (Effect-TS is used throughout)
+
+## Testing
+
+- Write tests for new features and bug fixes
+- Tests are located in `*.test.ts` files alongside source files
+- Use [Vitest](https://vitest.dev/) for testing
+- Semantic search tests require `OPENAI_API_KEY` and are skipped by default
+
+### Running Specific Tests
+
+```bash
+# Run a specific test file
+pnpm vitest run src/search/searcher.test.ts
+
+# Run tests matching a pattern
+pnpm vitest run -t "keyword search"
+
+# Watch mode during development
+pnpm test:watch
+```
+
+## Questions?
+
+- Open an issue for bugs or feature requests
+- Check existing issues before creating new ones
+- For questions about using mdcontext, see the README and docs/
+
+Thank you for contributing!

package/NOTES/NOTES

@@ -0,0 +1,44 @@
+  ralph orchestrate search-enhancements
+  ralph orchestrate context-precision
+  ralph orchestrate embeddings-ux
+
+  - Semantic search with embeddings
+  - Boolean operators (AND, OR, NOT)
+  - Phrase search (exact match)
+  - Section-level context extraction
+  - Search context lines
+
+  Ideas for Part 3:
+  Option: A
+  Strategy: Feature Battle
+  Focus: Parallel agents each test ONE new feature (semantic,
+    boolean, phrase) - compare effectiveness
+  ────────────────────────────────────────
+  Option: B
+  Strategy: Meta-Dogfood
+  Focus: Use mdtldr to explore md-tldr's own codebase - eat our
+  own
+     dogfood
+  ────────────────────────────────────────
+  Option: C
+  Strategy: Query Refinement
+  Focus: Single agent that does iterative search, refining queries
+
+    based on results
+  ────────────────────────────────────────
+  Option: D
+  Strategy: Scale Test
+  Focus: Same docs corpus but with much larger token budgets, more
+
+-----
+
+
+Let have a subagent review
+  /Users/alphab/Dev/LLM/DEV/md-tldr/docs/002-research-embedding-
+  models.md
+  /Users/alphab/Dev/LLM/DEV/md-tldr/docs/003-research-rag-altern
+  atives.md
+  /Users/alphab/Dev/LLM/DEV/md-tldr/docs/004-research-vector-sea
+  rch.md
+
+  and document task candidates

package/README.md

@@ -8,6 +8,7 @@ QUICK REFERENCE
   mdcontext search <query> [path]  Search by meaning or structure
   mdcontext context <files...>     Get LLM-ready summary
   mdcontext tree [path|file]       Show files or document outline
+  mdcontext config <command>       Configuration management (init, show, check)
   mdcontext links <file>           Outgoing links
   mdcontext backlinks <file>       Incoming links
   mdcontext stats [path]           Index statistics
@@ -36,7 +37,7 @@ mdcontext context README.md           # Get LLM-ready summary
 npm install -g mdcontext
 ```
 
-Requires Node.js 18+. Semantic search requires `OPENAI_API_KEY`.
+Requires Node.js 18+. Semantic search requires an embedding provider (OpenAI, Ollama, LM Studio, OpenRouter, or Voyage). See [docs/CONFIG.md](./docs/CONFIG.md#embedding-providers) for provider setup.
 
 ---
 
@@ -63,7 +64,22 @@ Search by meaning (semantic) or keyword (text match).
 mdcontext search "how to authenticate"        # Semantic search (if embeddings exist)
 mdcontext search -k "auth.*flow"              # Keyword search (text match)
 mdcontext search -n 5 "setup"                 # Limit to 5 results
-mdcontext search --threshold 0.8 "deploy"     # Higher similarity threshold
+mdcontext search --threshold 0.25 "deploy"    # Lower threshold for more results
+```
+
+#### Similarity Threshold
+
+Semantic search filters results by similarity score (0-1). Default: **0.35** (35%).
+
+- **0 results?** Content may exist below the threshold. Try `--threshold 0.25`
+- **Typical scores**: Single-word queries score ~30-40%, multi-word phrases ~50-70%
+- **Higher threshold** = stricter matching, fewer results
+- **Lower threshold** = more results, possibly less relevant
+
+```bash
+mdcontext search "authentication"              # Uses default 0.35 threshold
+mdcontext search --threshold 0.25 "auth"       # Lower threshold for broad queries
+mdcontext search --threshold 0.6 "specific"    # Higher threshold for precision
 ```
 
 #### Context Lines
@@ -77,6 +93,37 @@ mdcontext search "error" -B 2 -A 5            # 2 lines before, 5 lines after
 
 Auto-detection: Uses semantic search if embeddings exist and query looks like natural language. Use `-k` to force keyword search.
 
+#### Advanced Search
+
+**Quality Modes** - Control speed vs. accuracy tradeoff:
+```bash
+mdcontext search "query" --quality fast       # 40% faster, good recall
+mdcontext search "query" -q thorough          # Best recall, 30% slower
+```
+
+**Re-ranking** - Boost precision by 20-35%:
+```bash
+mdcontext search "query" --rerank             # First use downloads 90MB model
+npm install @huggingface/transformers         # Required dependency
+```
+
+**HyDE** - Better results for complex questions:
+```bash
+mdcontext search "how to implement auth" --hyde   # Expands query semantically
+```
+
+#### AI Summarization
+
+Generate AI-powered summaries of search results:
+
+```bash
+mdcontext search "authentication" --summarize     # Get AI summary of results
+mdcontext search "error handling" -s --yes        # Skip cost confirmation
+mdcontext search "database" -s --stream           # Stream output in real-time
+```
+
+Uses your existing AI subscription (Claude Code, Copilot CLI) for free, or pay-per-use API providers. See [AI Summarization](#ai-summarization) for setup.
+
 ### context
 
 Get LLM-ready summaries from one or more files.
@@ -155,12 +202,29 @@ mdcontext search -k "Setup|Install"           # By keyword pattern
 
 ### Setting Up Semantic Search
 
+mdcontext supports multiple embedding providers for semantic search:
+
+- **OpenAI** (default) - Cloud-based, requires API key
+- **Ollama** - Free, local, daemon-based
+- **LM Studio** - Free, local, GUI-based (development only)
+- **OpenRouter** - Multi-provider gateway
+- **Voyage** - Premium quality, competitive pricing
+
+Quick start with OpenAI:
 ```bash
 export OPENAI_API_KEY=sk-...
 mdcontext index --embed                       # Build embeddings
 mdcontext search "how to deploy"              # Now works semantically
 ```
 
+Using Ollama (free, local):
+```bash
+ollama serve && ollama pull nomic-embed-text
+mdcontext index --embed --provider ollama --provider-model nomic-embed-text
+```
+
+See [docs/CONFIG.md](./docs/CONFIG.md#embedding-providers) for complete provider setup, comparison, and configuration options.
+
 ---
 
 ## MCP Integration
@@ -203,6 +267,35 @@ For Claude Code, add to `.claude/settings.json`:
 
 ## Configuration
 
+mdcontext supports a layered configuration system for persistent settings:
+
+```bash
+# Create a config file
+mdcontext config init
+
+# Check your configuration
+mdcontext config check
+
+# Customize settings in mdcontext.config.js
+```
+
+```javascript
+// mdcontext.config.js
+/** @type {import('mdcontext').PartialMdContextConfig} */
+export default {
+  index: {
+    excludePatterns: ['node_modules', '.git', 'dist', 'vendor']
+  },
+  search: {
+    defaultLimit: 20
+  }
+}
+```
+
+Configuration precedence: CLI flags > Environment variables > Config file > Defaults
+
+**See [docs/CONFIG.md](./docs/CONFIG.md) for the complete configuration reference.**
+
 ### Index Location
 
 Indexes are stored in `.mdcontext/` in your project root:
@@ -220,7 +313,117 @@ Indexes are stored in `.mdcontext/` in your project root:
 
 | Variable | Description |
 |----------|-------------|
-| `OPENAI_API_KEY` | Required for semantic search |
+| `OPENAI_API_KEY` | Required for OpenAI semantic search (default provider) |
+| `OPENROUTER_API_KEY` | Required for OpenRouter semantic search |
+| `MDCONTEXT_*` | Configuration overrides (see [CONFIG.md](./docs/CONFIG.md)) |
+
+---
+
+## AI Summarization
+
+Transform search results into actionable insights using AI.
+
+### Quick Start
+
+```bash
+# Basic usage (auto-detects installed CLI tools)
+mdcontext search "authentication" --summarize
+
+# Skip confirmation for scripts
+mdcontext search "error handling" --summarize --yes
+
+# Stream output in real-time
+mdcontext search "database" --summarize --stream
+```
+
+### First-Time Setup
+
+On first use, mdcontext auto-detects available providers:
+
+```
+Using claude (subscription - FREE)
+
+--- AI Summary ---
+
+Based on the search results, here are the key findings...
+```
+
+### Providers
+
+**CLI Providers (FREE with subscription):**
+
+| Provider | Command | Subscription Required |
+|----------|---------|----------------------|
+| Claude Code | `claude` | Claude Pro/Team |
+| GitHub Copilot | `copilot` | Copilot subscription |
+| OpenCode | `opencode` | BYOK (any provider) |
+
+**API Providers (pay-per-use):**
+
+| Provider | Cost per 1M tokens | Notes |
+|----------|-------------------|-------|
+| DeepSeek | $0.14-0.56 | Ultra-cheap |
+| Qwen | $0.03-0.12 | Budget option |
+| Google Gemini | $0.30-2.50 | Balanced |
+| OpenAI GPT | $1.75-14.00 | Premium |
+| Anthropic Claude | $3.00-15.00 | Premium |
+
+### Configuration
+
+**Option 1: Auto-detection (recommended)**
+
+Just run `--summarize` - mdcontext finds installed CLI tools automatically.
+
+**Option 2: Config file**
+
+```javascript
+// mdcontext.config.js
+/** @type {import('mdcontext').PartialMdContextConfig} */
+export default {
+  aiSummarization: {
+    mode: 'cli',        // 'cli' (free) or 'api' (paid)
+    provider: 'claude', // Provider name
+  },
+}
+```
+
+**Option 3: Environment variables**
+
+```bash
+export MDCONTEXT_AISUMMARIZATION_MODE=api
+export MDCONTEXT_AISUMMARIZATION_PROVIDER=deepseek
+export DEEPSEEK_API_KEY=sk-...
+```
+
+### CLI Flags
+
+| Flag | Short | Description |
+|------|-------|-------------|
+| `--summarize` | `-s` | Enable AI summarization |
+| `--yes` | `-y` | Skip cost confirmation |
+| `--stream` | | Stream output in real-time |
+
+### Cost Transparency
+
+API providers show cost estimates before proceeding:
+
+```
+Cost Estimate:
+  Provider: deepseek
+  Input tokens: ~2,500
+  Output tokens: ~500
+  Estimated cost: $0.0007
+
+Continue with summarization? [Y/n]:
+```
+
+CLI providers show free status:
+
+```
+Using claude (subscription - FREE)
+```
+
+See [docs/summarization.md](./docs/summarization.md) for architecture details and troubleshooting.
 
 ---
 

package/biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/2.3.11/schema.json",
+  "$schema": "https://biomejs.dev/schemas/2.3.12/schema.json",
   "assist": {
     "actions": {
       "source": {