memory-lancedb-pro 1.0.20 → 1.0.22

package/CHANGELOG.md

@@ -1,6 +1,33 @@
 # Changelog
 
 
+## 1.0.22
+
+**Storage Path Validation & Better Error Messages**
+
+- **Fix**: Validate `dbPath` at startup — resolve symlinks, auto-create missing directories, check write permissions (#26, #27)
+- **Fix**: Write/connection failures now include `errno`, resolved path, and actionable fix suggestions instead of generic errors (#28)
+- **New**: Exported `validateStoragePath()` utility for external tooling and diagnostics
+
+Breaking changes: None. Backward compatible.
+
+---
+
+## 1.0.21
+
+**Long Context Chunking**
+
+- **Feats**: Added automatic chunking for documents exceeding embedding context limits
+- **Feats**: Smart semantic-aware chunking at sentence boundaries with configurable overlap
+- **Feats**: Chunking adapts to different embedding model context limits (Jina, OpenAI, Gemini, etc.)
+- **Feats**: Parallel chunk embedding with averaged result for better semantic preservation
+- **Fixes**: Handles "Input length exceeds context length" errors gracefully
+- **Docs**: Added comprehensive documentation in docs/long-context-chunking.md
+
+Breaking changes: None. Backward compatible with existing configurations.
+
+---
+
 ## 1.0.20
 
 - Fix: reduce auto-capture noise by skipping memory-management prompts (delete/forget/cleanup memory entries).

package/README.md

@@ -720,6 +720,40 @@ upgrade to **memory-lancedb-pro >= 1.0.14**. This plugin now coerces these value
 
 ---
 
+## ⭐ Star History
+
+<a href="https://star-history.com/#win4r/memory-lancedb-pro&Date">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=win4r/memory-lancedb-pro&type=Date&theme=dark&transparent=true" />
+    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=win4r/memory-lancedb-pro&type=Date&transparent=true" />
+    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=win4r/memory-lancedb-pro&type=Date&transparent=true" />
+  </picture>
+</a>
+
+## Contributors
+
+Top contributors (from GitHub’s contributors list, sorted by commit contributions; bots excluded):
+
+<p>
+<a href="https://github.com/win4r"><img src="https://avatars.githubusercontent.com/u/42172631?v=4" width="48" height="48" alt="@win4r" /></a>
+<a href="https://github.com/kctony"><img src="https://avatars.githubusercontent.com/u/1731141?v=4" width="48" height="48" alt="@kctony" /></a>
+<a href="https://github.com/Akatsuki-Ryu"><img src="https://avatars.githubusercontent.com/u/8062209?v=4" width="48" height="48" alt="@Akatsuki-Ryu" /></a>
+<a href="https://github.com/JasonSuz"><img src="https://avatars.githubusercontent.com/u/612256?v=4" width="48" height="48" alt="@JasonSuz" /></a>
+<a href="https://github.com/Minidoracat"><img src="https://avatars.githubusercontent.com/u/11269639?v=4" width="48" height="48" alt="@Minidoracat" /></a>
+<a href="https://github.com/furedericca-lab"><img src="https://avatars.githubusercontent.com/u/263020793?v=4" width="48" height="48" alt="@furedericca-lab" /></a>
+<a href="https://github.com/joe2643"><img src="https://avatars.githubusercontent.com/u/19421931?v=4" width="48" height="48" alt="@joe2643" /></a>
+</p>
+
+- [@win4r](https://github.com/win4r) (3 commits)
+- [@kctony](https://github.com/kctony) (2 commits)
+- [@Akatsuki-Ryu](https://github.com/Akatsuki-Ryu) (1 commit)
+- [@JasonSuz](https://github.com/JasonSuz) (1 commit)
+- [@Minidoracat](https://github.com/Minidoracat) (1 commit)
+- [@furedericca-lab](https://github.com/furedericca-lab) (1 commit)
+- [@joe2643](https://github.com/joe2643) (1 commit)
+
+Full list: https://github.com/win4r/memory-lancedb-pro/graphs/contributors
+
 ## License
 
 MIT

package/README_CN.md

@@ -594,6 +594,40 @@ LanceDB 表 `memories`：
 
 ---
 
+## ⭐ Star 趋势
+
+<a href="https://star-history.com/#win4r/memory-lancedb-pro&Date">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=win4r/memory-lancedb-pro&type=Date&theme=dark&transparent=true" />
+    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=win4r/memory-lancedb-pro&type=Date&transparent=true" />
+    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=win4r/memory-lancedb-pro&type=Date&transparent=true" />
+  </picture>
+</a>
+
+## 主要贡献者
+
+按 GitHub Contributors 列表自动生成（按 commit 贡献数排序，已排除 bot）：
+
+<p>
+<a href="https://github.com/win4r"><img src="https://avatars.githubusercontent.com/u/42172631?v=4" width="48" height="48" alt="@win4r" /></a>
+<a href="https://github.com/kctony"><img src="https://avatars.githubusercontent.com/u/1731141?v=4" width="48" height="48" alt="@kctony" /></a>
+<a href="https://github.com/Akatsuki-Ryu"><img src="https://avatars.githubusercontent.com/u/8062209?v=4" width="48" height="48" alt="@Akatsuki-Ryu" /></a>
+<a href="https://github.com/JasonSuz"><img src="https://avatars.githubusercontent.com/u/612256?v=4" width="48" height="48" alt="@JasonSuz" /></a>
+<a href="https://github.com/Minidoracat"><img src="https://avatars.githubusercontent.com/u/11269639?v=4" width="48" height="48" alt="@Minidoracat" /></a>
+<a href="https://github.com/furedericca-lab"><img src="https://avatars.githubusercontent.com/u/263020793?v=4" width="48" height="48" alt="@furedericca-lab" /></a>
+<a href="https://github.com/joe2643"><img src="https://avatars.githubusercontent.com/u/19421931?v=4" width="48" height="48" alt="@joe2643" /></a>
+</p>
+
+- [@win4r](https://github.com/win4r)（3 次提交）
+- [@kctony](https://github.com/kctony)（2 次提交）
+- [@Akatsuki-Ryu](https://github.com/Akatsuki-Ryu)（1 次提交）
+- [@JasonSuz](https://github.com/JasonSuz)（1 次提交）
+- [@Minidoracat](https://github.com/Minidoracat)（1 次提交）
+- [@furedericca-lab](https://github.com/furedericca-lab)（1 次提交）
+- [@joe2643](https://github.com/joe2643)（1 次提交）
+
+完整列表：https://github.com/win4r/memory-lancedb-pro/graphs/contributors
+
 ## License
 
 MIT

package/docs/long-context-chunking.md

@@ -0,0 +1,258 @@
+# Long Context Chunking
+
+## Overview
+
+The long context chunking system automatically handles documents that exceed embedding model context limits by splitting them into manageable chunks and computing averaged embeddings.
+
+## Problem Solved
+
+When embedding very long documents or messages, you might encounter errors like:
+
+```
+Input length exceeds context length: 12453 tokens. Maximum length: 8192 tokens.
+```
+
+This plugin now handles such cases gracefully by:
+1. Detecting context length errors before they cause failures
+2. Automatically splitting the document into overlapping chunks
+3. Embedding each chunk separately
+4. Computing an averaged embedding that preserves semantic meaning
+
+## How It Works
+
+### Chunking Strategy
+
+The chunker uses a **semantic-aware** approach:
+
+- **Splits at sentence boundaries** when possible (better for preserving meaning)
+- **Configurable overlap** (default: 200 characters) to maintain context across chunks
+- **Adapts to model context limits** based on the embedding model
+- **Forced splits** at hard limits if sentence boundaries are not found
+
+### Chunking Flow
+
+```
+Long Document
+    │
+    ├── 8192+ characters ──┐
+                            │
+                            ▼
+                ┌─────────────────┐
+                │  Detect Overflow │
+                └────────┬────────┘
+                         │
+                         ▼
+                ┌─────────────────┐
+                │  Split into     │
+                │  Overlapping     │
+                │  Chunks          │
+                └────────┬────────┘
+                         │
+    ┌────────────────────┼────────────────────┐
+    │                    │                    │
+    ▼                    ▼                    ▼
+┌────────┐         ┌────────┐         ┌────────┐
+│ Chunk 1│         │ Chunk 2│         │ Chunk 3│
+│  [1-2k]│         │[1.8k-3.8k]│    │[3.6k-5.6k]│
+└───┬────┘         └───┬────┘         └───┬────┘
+    │                  │                  │
+    ▼                  ▼                  ▼
+Embedding          Embedding          Embedding
+    │                  │                  │
+    └──────────────────┼──────────────────┘
+                       │
+                       ▼
+              Compute Average
+                       │
+                       ▼
+              Final Embedding
+```
+
+## Configuration
+
+### Default Settings
+
+The chunker automatically adapts to your embedding model:
+
+- **maxChunkSize**: 70% of model context limit (e.g., 5734 for 8192-token model)
+- **overlapSize**: 5% of model context limit
+- **minChunkSize**: 10% of model context limit
+- **semanticSplit**: true (prefer sentence boundaries)
+- **maxLinesPerChunk**: 50 lines
+
+### Disabling Auto-Chunking
+
+If you prefer to handle chunking manually or want the model to fail on long documents:
+
+```json
+{
+  "plugins": {
+    "entries": {
+      "memory-lancedb-pro": {
+        "enabled": true,
+        "config": {
+          "embedding": {
+            "apiKey": "${JINA_API_KEY}",
+            "model": "jina-embeddings-v5-text-small",
+            "chunking": false  // Disable auto-chunking
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+### Custom Chunking Parameters
+
+For advanced users who want to tune chunking behavior:
+
+```json
+{
+  "plugins": {
+    "entries": {
+      "memory-lancedb-pro": {
+        "enabled": true,
+        "config": {
+          "embedding": {
+            "autoChunk": {
+              "maxChunkSize": 2000,      // Characters per chunk
+              "overlapSize": 500,          // Overlap between chunks
+              "minChunkSize": 500,         // Minimum acceptable chunk size
+              "semanticSplit": true,       // Prefer sentence boundaries
+              "maxLinesPerChunk": 100      // Max lines before forced split
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+## Supported Models
+
+The chunker automatically adapts to these embedding models:
+
+| Model | Context Limit | Chunk Size | Overlap |
+|-------|---------------|------------|----------|
+| Jina jina-embeddings-v5-text-small | 8192 | 5734 | 409 |
+| OpenAI text-embedding-3-small | 8192 | 5734 | 409 |
+| OpenAI text-embedding-3-large | 8192 | 5734 | 409 |
+| Gemini gemini-embedding-001 | 2048 | 1433 | 102 |
+
+## Performance Considerations
+
+### Token Savings
+
+- **Without chunking**: 1 failed embedding (retries required)
+- **With chunking**: 3-4 chunk embeddings (1 avg result)
+- **Net cost increase**: ~3x for long documents (>8k tokens)
+- **Trade-off**: Gracefully handling vs. processing smaller documents
+
+### Caching
+
+Chunked embeddings are cached by their original document hash, so:
+- Subsequent requests for the same document get the cached averaged embedding
+- Cache hit rate improves as long documents are processed repeatedly
+
+### Processing Time
+
+- **Small documents (<4k chars)**: No chunking, same as before
+- **Medium documents (4k-8k chars)**: No chunking, same as before
+- **Long documents (>8k chars)**: ~100-200ms additional chunking overhead
+
+## Logging & Debugging
+
+### Enable Debug Logging
+
+To see chunking in action, you can check the logs:
+
+```
+Document exceeded context limit (...), attempting chunking...
+Split document into 3 chunks for embedding
+Successfully embedded long document as 3 averaged chunks
+```
+
+### Common Scenarios
+
+**Scenario 1: Long memory text**
+- When a user's message or system prompt is very long
+- Automatically chunked before embedding
+- No error thrown, memory is still stored and retrievable
+
+**Scenario 2: Batch embedding long documents**
+- If some documents in a batch exceed limits
+- Only the long ones are chunked
+- Successful documents processed normally
+
+## Troubleshooting
+
+### Chunking Still Fails
+
+If you still see context length errors:
+
+1. **Verify model**: Check which embedding model you're using
+2. **Increase minChunkSize**: May need smaller chunks for some models
+3. **Disable autoChunk**: Handle chunking manually with explicit split
+
+### Too Many Small Chunks
+
+If chunking creates many tiny fragments:
+
+1. **Increase minChunkSize**: Larger minimum chunk size
+2. **Reduce overlap**: Less overlap between chunks means more efficient chunks
+
+### Embedding Quality Degradation
+
+If chunked embeddings seem less accurate:
+
+1. **Increase overlap**: More context between chunks preserves relationships
+2. **Use smaller maxChunkSize**: Split into more, smaller overlapping pieces
+3. **Consider hierarchical approach**: Use a two-pass retrieval (chunk → document → full text)
+
+## Future Enhancements
+
+Planned improvements:
+
+- [ ] **Hierarchical chunking**: Chunk → document-level embedding
+- [ ] **Sliding window**: Different overlap strategies per document complexity
+- [ ] **Smart summarization**: Summarize chunks before averaging for better quality
+- [ ] **Context-aware overlap**: Dynamic overlap based on document complexity
+- [ ] **Async chunking**: Process chunks in parallel for batch operations
+
+## Technical Details
+
+### Algorithm
+
+1. **Detect overflow**: Check if document exceeds maxChunkSize
+2. **Split semantically**: Find sentence boundaries within target range
+3. **Create overlap**: Include overlap with previous chunk's end
+4. **Embed in parallel**: Process all chunks simultaneously
+5. **Average the result**: Compute mean embedding across all chunks
+
+### Complexity
+
+- **Time**: O(n × k) where n = number of chunks, k = average chunk processing time
+- **Space**: O(n × d) where d = embedding dimension
+
+### Edge Cases
+
+| Case | Handling |
+|------|----------|
+| Empty document | Returns empty embedding immediately |
+| Very small documents | No chunking, normal processing |
+| Perfect boundaries | Split at sentence ends, no truncation |
+| No boundaries found | Hard split at max position |
+| Single oversized chunk | Process as-is, let provider error |
+| All chunks too small | Last chunk takes remaining text |
+
+## References
+
+- [LanceDB Documentation](https://lancedb.com)
+- [OpenAI Embedding Context Limits](https://platform.openai.com/docs/guides/embeddings)
+- [Semantic Chunking Research](https://arxiv.org/abs/2310.05970)
+
+---
+
+*This feature was added to handle long-context documents gracefully without losing memory quality.*

package/index.ts

@@ -10,7 +10,7 @@ import { readFile, readdir, writeFile, mkdir } from "node:fs/promises";
 import { readFileSync } from "node:fs";
 
 // Import core components
-import { MemoryStore } from "./src/store.js";
+import { MemoryStore, validateStoragePath } from "./src/store.js";
 import { createEmbedder, getVectorDimensions } from "./src/embedder.js";
 import { createRetriever, DEFAULT_RETRIEVAL_CONFIG } from "./src/retriever.js";
 import { createScopeManager } from "./src/scopes.js";
@@ -315,6 +315,18 @@ const memoryLanceDBProPlugin = {
     const config = parsePluginConfig(api.pluginConfig);
 
     const resolvedDbPath = api.resolvePath(config.dbPath || getDefaultDbPath());
+
+    // Pre-flight: validate storage path (symlink resolution, mkdir, write check).
+    // Runs synchronously and logs warnings; does NOT block gateway startup.
+    try {
+      validateStoragePath(resolvedDbPath);
+    } catch (err) {
+      api.logger.warn(
+        `memory-lancedb-pro: storage path issue — ${String(err)}\n` +
+        `  The plugin will still attempt to start, but writes may fail.`
+      );
+    }
+
     const vectorDim = getVectorDimensions(
       config.embedding.model || "text-embedding-3-small",
       config.embedding.dimensions

package/openclaw.plugin.json

@@ -1,8 +1,8 @@
 {
   "id": "memory-lancedb-pro",
   "name": "Memory (LanceDB Pro)",
-  "description": "Enhanced LanceDB-backed long-term memory with hybrid retrieval, multi-scope isolation, and management CLI",
-  "version": "1.0.20",
+  "description": "Enhanced LanceDB-backed long-term memory with hybrid retrieval, multi-scope isolation, long-context chunking, and management CLI",
+  "version": "1.0.21",
   "kind": "memory",
   "configSchema": {
     "type": "object",
@@ -40,6 +40,11 @@
           "normalized": {
             "type": "boolean",
             "description": "Request normalized embeddings when supported by the provider (e.g. Jina v5)"
+          },
+          "chunking": {
+            "type": "boolean",
+            "default": true,
+            "description": "Enable automatic chunking for documents exceeding embedding context limits"
           }
         },
         "required": [
@@ -260,6 +265,28 @@
       "help": "Override vector dimensions for custom models not in the built-in lookup table",
       "advanced": true
     },
+    "embedding.taskQuery": {
+      "label": "Query Task",
+      "placeholder": "retrieval.query",
+      "help": "Optional task selector for query embeddings (Jina: retrieval.query). If unset, no task field is sent.",
+      "advanced": true
+    },
+    "embedding.taskPassage": {
+      "label": "Passage Task",
+      "placeholder": "retrieval.passage",
+      "help": "Optional task selector for passage/document embeddings (Jina: retrieval.passage). If unset, no task field is sent.",
+      "advanced": true
+    },
+    "embedding.normalized": {
+      "label": "Normalized Embeddings",
+      "help": "Request normalized embeddings when the provider supports it (Jina v5). If unset, the field is not sent.",
+      "advanced": true
+    },
+    "embedding.chunking": {
+      "label": "Auto-Chunk Documents",
+      "help": "Automatically split long documents into chunks when they exceed embedding context limits. Set to false to disable and let the model fail on long documents.",
+      "advanced": true
+    },
     "dbPath": {
       "label": "Database Path",
       "placeholder": "~/.openclaw/memory/lancedb-pro",
@@ -338,6 +365,21 @@
       "help": "Number of candidates to fetch before fusion and reranking",
       "advanced": true
     },
+    "retrieval.lengthNormAnchor": {
+      "label": "Length Normalization Anchor",
+      "help": "Entries longer than this (chars) get score penalized to prevent long entries dominating. 0 = disabled.",
+      "advanced": true
+    },
+    "retrieval.hardMinScore": {
+      "label": "Hard Minimum Score",
+      "help": "Discard results below this score after all scoring stages. Higher = fewer but more relevant results.",
+      "advanced": true
+    },
+    "retrieval.timeDecayHalfLifeDays": {
+      "label": "Time Decay Half-Life",
+      "help": "Old entries lose score over this many days. Floor at 0.5x. 0 = disabled.",
+      "advanced": true
+    },
     "sessionMemory.enabled": {
       "label": "Session Memory",
       "help": "Store session summaries to LanceDB when /new is triggered (replaces built-in session-memory hook)"
@@ -366,38 +408,6 @@
       "label": "Management Tools",
       "help": "Enable memory_list and memory_stats tools for debugging and auditing",
       "advanced": true
-    },
-    "retrieval.lengthNormAnchor": {
-      "label": "Length Normalization Anchor",
-      "help": "Entries longer than this (chars) get score penalized to prevent long entries dominating. 0 = disabled.",
-      "advanced": true
-    },
-    "retrieval.hardMinScore": {
-      "label": "Hard Minimum Score",
-      "help": "Discard results below this score after all scoring stages. Higher = fewer but more relevant results.",
-      "advanced": true
-    },
-    "retrieval.timeDecayHalfLifeDays": {
-      "label": "Time Decay Half-Life",
-      "help": "Old entries lose score over this many days. Floor at 0.5x. 0 = disabled.",
-      "advanced": true
-    },
-    "embedding.taskQuery": {
-      "label": "Query Task",
-      "placeholder": "retrieval.query",
-      "help": "Optional task selector for query embeddings (Jina: retrieval.query). If unset, no task field is sent.",
-      "advanced": true
-    },
-    "embedding.taskPassage": {
-      "label": "Passage Task",
-      "placeholder": "retrieval.passage",
-      "help": "Optional task selector for passage/document embeddings (Jina: retrieval.passage). If unset, no task field is sent.",
-      "advanced": true
-    },
-    "embedding.normalized": {
-      "label": "Normalized Embeddings",
-      "help": "Request normalized embeddings when the provider supports it (Jina v5). If unset, the field is not sent.",
-      "advanced": true
     }
   }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "memory-lancedb-pro",
-  "version": "1.0.20",
-  "description": "OpenClaw enhanced LanceDB memory plugin with hybrid retrieval (Vector + BM25), cross-encoder rerank, multi-scope isolation, and management CLI",
+  "version": "1.0.22",
+  "description": "OpenClaw enhanced LanceDB memory plugin with hybrid retrieval (Vector + BM25), cross-encoder rerank, multi-scope isolation, long-context chunking, and management CLI",
   "type": "module",
   "main": "index.ts",
   "keywords": [
@@ -14,7 +14,9 @@
     "hybrid-retrieval",
     "rerank",
     "ai-memory",
-    "long-term-memory"
+    "long-term-memory",
+    "chunking",
+    "long-context"
   ],
   "repository": {
     "type": "git",

package/src/chunker.ts

@@ -0,0 +1,253 @@
+/**
+ * Long Context Chunking System
+ *
+ * Goal: split documents that exceed embedding model context limits into smaller,
+ * semantically coherent chunks with overlap.
+ *
+ * Notes:
+ * - We use *character counts* as a conservative proxy for tokens.
+ * - The embedder triggers this only after a provider throws a context-length error.
+ */
+
+// ============================================================================
+// Types & Constants
+// ============================================================================
+
+export interface ChunkMetadata {
+  startIndex: number;
+  endIndex: number;
+  length: number;
+}
+
+export interface ChunkResult {
+  chunks: string[];
+  metadatas: ChunkMetadata[];
+  totalOriginalLength: number;
+  chunkCount: number;
+}
+
+export interface ChunkerConfig {
+  /** Maximum characters per chunk. */
+  maxChunkSize: number;
+  /** Overlap between chunks in characters. */
+  overlapSize: number;
+  /** Minimum chunk size (except the final chunk). */
+  minChunkSize: number;
+  /** Attempt to split on sentence boundaries for better semantic coherence. */
+  semanticSplit: boolean;
+  /** Max lines per chunk before we try to split earlier on a line boundary. */
+  maxLinesPerChunk: number;
+}
+
+// Common embedding context limits (provider/model specific). These are typically
+// token limits, but we treat them as inputs to a conservative char-based heuristic.
+export const EMBEDDING_CONTEXT_LIMITS: Record<string, number> = {
+  // Jina v5
+  "jina-embeddings-v5-text-small": 8192,
+  "jina-embeddings-v5-text-nano": 8192,
+
+  // OpenAI
+  "text-embedding-3-small": 8192,
+  "text-embedding-3-large": 8192,
+
+  // Google
+  "text-embedding-004": 8192,
+  "gemini-embedding-001": 2048,
+
+  // Local/common
+  "nomic-embed-text": 8192,
+  "all-MiniLM-L6-v2": 512,
+  "all-mpnet-base-v2": 512,
+};
+
+export const DEFAULT_CHUNKER_CONFIG: ChunkerConfig = {
+  maxChunkSize: 4000,
+  overlapSize: 200,
+  minChunkSize: 200,
+  semanticSplit: true,
+  maxLinesPerChunk: 50,
+};
+
+// Sentence ending patterns (English + CJK-ish punctuation)
+const SENTENCE_ENDING = /[.!?。！？]/;
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+function clamp(n: number, lo: number, hi: number): number {
+  return Math.max(lo, Math.min(hi, n));
+}
+
+function countLines(s: string): number {
+  // Count \n (treat CRLF as one line break)
+  return s.split(/\r\n|\n|\r/).length;
+}
+
+function findLastIndexWithin(text: string, re: RegExp, start: number, end: number): number {
+  // Find last match start index for regex within [start, end).
+  // NOTE: `re` must NOT be global; we will scan manually.
+  let last = -1;
+  for (let i = end - 1; i >= start; i--) {
+    if (re.test(text[i])) return i;
+  }
+  return last;
+}
+
+function findSplitEnd(text: string, start: number, maxEnd: number, minEnd: number, config: ChunkerConfig): number {
+  const safeMinEnd = clamp(minEnd, start + 1, maxEnd);
+  const safeMaxEnd = clamp(maxEnd, safeMinEnd, text.length);
+
+  // Respect line limit: if we exceed maxLinesPerChunk, force earlier split at a line break.
+  if (config.maxLinesPerChunk > 0) {
+    const candidate = text.slice(start, safeMaxEnd);
+    if (countLines(candidate) > config.maxLinesPerChunk) {
+      // Find the position of the Nth line break.
+      let breaks = 0;
+      for (let i = start; i < safeMaxEnd; i++) {
+        const ch = text[i];
+        if (ch === "\n") {
+          breaks++;
+          if (breaks >= config.maxLinesPerChunk) {
+            // Split right after this newline.
+            return Math.max(i + 1, safeMinEnd);
+          }
+        }
+      }
+    }
+  }
+
+  if (config.semanticSplit) {
+    // Prefer a sentence boundary near the end.
+    // Scan backward from safeMaxEnd to safeMinEnd.
+    for (let i = safeMaxEnd - 1; i >= safeMinEnd; i--) {
+      if (SENTENCE_ENDING.test(text[i])) {
+        // Include trailing whitespace after punctuation.
+        let j = i + 1;
+        while (j < safeMaxEnd && /\s/.test(text[j])) j++;
+        return j;
+      }
+    }
+
+    // Next best: newline boundary.
+    for (let i = safeMaxEnd - 1; i >= safeMinEnd; i--) {
+      if (text[i] === "\n") return i + 1;
+    }
+  }
+
+  // Fallback: last whitespace boundary.
+  for (let i = safeMaxEnd - 1; i >= safeMinEnd; i--) {
+    if (/\s/.test(text[i])) return i;
+  }
+
+  return safeMaxEnd;
+}
+
+function sliceTrimWithIndices(text: string, start: number, end: number): { chunk: string; meta: ChunkMetadata } {
+  const raw = text.slice(start, end);
+  const leading = raw.match(/^\s*/)?.[0]?.length ?? 0;
+  const trailing = raw.match(/\s*$/)?.[0]?.length ?? 0;
+  const chunk = raw.trim();
+
+  const trimmedStart = start + leading;
+  const trimmedEnd = end - trailing;
+
+  return {
+    chunk,
+    meta: {
+      startIndex: trimmedStart,
+      endIndex: Math.max(trimmedStart, trimmedEnd),
+      length: chunk.length,
+    },
+  };
+}
+
+// ============================================================================
+// Chunking Core
+// ============================================================================
+
+export function chunkDocument(text: string, config: ChunkerConfig = DEFAULT_CHUNKER_CONFIG): ChunkResult {
+  if (!text || text.trim().length === 0) {
+    return { chunks: [], metadatas: [], totalOriginalLength: 0, chunkCount: 0 };
+  }
+
+  const totalOriginalLength = text.length;
+  const chunks: string[] = [];
+  const metadatas: ChunkMetadata[] = [];
+
+  let pos = 0;
+  const maxGuard = Math.max(4, Math.ceil(text.length / Math.max(1, config.maxChunkSize - config.overlapSize)) + 5);
+  let guard = 0;
+
+  while (pos < text.length && guard < maxGuard) {
+    guard++;
+
+    const remaining = text.length - pos;
+    if (remaining <= config.maxChunkSize) {
+      const { chunk, meta } = sliceTrimWithIndices(text, pos, text.length);
+      if (chunk.length > 0) {
+        chunks.push(chunk);
+        metadatas.push(meta);
+      }
+      break;
+    }
+
+    const maxEnd = Math.min(pos + config.maxChunkSize, text.length);
+    const minEnd = Math.min(pos + config.minChunkSize, maxEnd);
+
+    const end = findSplitEnd(text, pos, maxEnd, minEnd, config);
+    const { chunk, meta } = sliceTrimWithIndices(text, pos, end);
+
+    // If trimming made it too small, fall back to a hard split.
+    if (chunk.length < config.minChunkSize) {
+      const hardEnd = Math.min(pos + config.maxChunkSize, text.length);
+      const hard = sliceTrimWithIndices(text, pos, hardEnd);
+      if (hard.chunk.length > 0) {
+        chunks.push(hard.chunk);
+        metadatas.push(hard.meta);
+      }
+      if (hardEnd >= text.length) break;
+      pos = Math.max(hardEnd - config.overlapSize, pos + 1);
+      continue;
+    }
+
+    chunks.push(chunk);
+    metadatas.push(meta);
+
+    if (end >= text.length) break;
+
+    // Move forward with overlap.
+    const nextPos = Math.max(end - config.overlapSize, pos + 1);
+    pos = nextPos;
+  }
+
+  return {
+    chunks,
+    metadatas,
+    totalOriginalLength,
+    chunkCount: chunks.length,
+  };
+}
+
+/**
+ * Smart chunker that adapts to model context limits.
+ *
+ * We intentionally pick conservative char limits (70% of the reported limit)
+ * since token/char ratios vary.
+ */
+export function smartChunk(text: string, embedderModel?: string): ChunkResult {
+  const limit = embedderModel ? EMBEDDING_CONTEXT_LIMITS[embedderModel] : undefined;
+  const base = limit ?? 8192;
+
+  const config: ChunkerConfig = {
+    maxChunkSize: Math.max(1000, Math.floor(base * 0.7)),
+    overlapSize: Math.max(0, Math.floor(base * 0.05)),
+    minChunkSize: Math.max(100, Math.floor(base * 0.1)),
+    semanticSplit: true,
+    maxLinesPerChunk: 50,
+  };
+
+  return chunkDocument(text, config);
+}
+
+export default chunkDocument;

package/src/embedder.ts

@@ -1,6 +1,7 @@
 /**
  * Embedding Abstraction Layer
  * OpenAI-compatible API for various embedding providers.
+ * Supports automatic chunking for documents exceeding embedding context limits.
  *
  * Note: Some providers (e.g. Jina) support extra parameters like `task` and
  * `normalized` on the embeddings endpoint. The OpenAI SDK types do not include
@@ -9,6 +10,7 @@
 
 import OpenAI from "openai";
 import { createHash } from "node:crypto";
+import { smartChunk } from "./chunker.js";
 
 // ============================================================================
 // Embedding Cache (LRU with TTL)
@@ -94,6 +96,8 @@ export interface EmbeddingConfig {
   taskPassage?: string;
   /** Optional flag to request normalized embeddings (provider-dependent, e.g. Jina v5) */
   normalized?: boolean;
+  /** Enable automatic chunking for documents exceeding context limits (default: true) */
+  chunking?: boolean;
 }
 
 // Known embedding model dimensions
@@ -106,7 +110,7 @@ const EMBEDDING_DIMENSIONS: Record<string, number> = {
   "mxbai-embed-large": 1024,
   "BAAI/bge-m3": 1024,
   "all-MiniLM-L6-v2": 384,
-  "all-mpnet-base-v2": 768,
+  "all-mpnet-base-v2": 512,
 
   // Jina v5
   "jina-embeddings-v5-text-small": 1024,
@@ -158,8 +162,10 @@ export class Embedder {
 
   /** Optional requested dimensions to pass through to the embedding provider (OpenAI-compatible). */
   private readonly _requestDimensions?: number;
+  /** Enable automatic chunking for long documents (default: true) */
+  private readonly _autoChunk: boolean;
 
-  constructor(config: EmbeddingConfig) {
+  constructor(config: EmbeddingConfig & { chunking?: boolean }) {
     // Resolve environment variables in API key
     const resolvedApiKey = resolveEnvVars(config.apiKey);
 
@@ -168,6 +174,8 @@ export class Embedder {
     this._taskPassage = config.taskPassage;
     this._normalized = config.normalized;
     this._requestDimensions = config.dimensions;
+    // Enable auto-chunking by default for better handling of long documents
+    this._autoChunk = config.chunking !== false;
 
     this.client = new OpenAI({
       apiKey: resolvedApiKey,
@@ -273,6 +281,58 @@ export class Embedder {
       this._cache.set(text, task, embedding);
       return embedding;
     } catch (error) {
+      // Check if this is a context length exceeded error and try chunking
+      const errorMsg = error instanceof Error ? error.message : String(error);
+      const isContextError = /context|too long|exceed|length/i.test(errorMsg);
+
+      if (isContextError && this._autoChunk) {
+        try {
+          console.log(`Document exceeded context limit (${errorMsg}), attempting chunking...`);
+          const chunkResult = smartChunk(text, this._model);
+          
+          if (chunkResult.chunks.length === 0) {
+            throw new Error(`Failed to chunk document: ${errorMsg}`);
+          }
+
+          // Embed all chunks in parallel
+          console.log(`Split document into ${chunkResult.chunkCount} chunks for embedding`);
+          const chunkEmbeddings = await Promise.all(
+            chunkResult.chunks.map(async (chunk, idx) => {
+              try {
+                const embedding = await this.embedSingle(chunk, task);
+                return { embedding };
+              } catch (chunkError) {
+                console.warn(`Failed to embed chunk ${idx}:`, chunkError);
+                throw chunkError;
+              }
+            })
+          );
+
+          // Compute average embedding across chunks
+          const avgEmbedding = chunkEmbeddings.reduce(
+            (sum, { embedding }) => {
+              for (let i = 0; i < embedding.length; i++) {
+                sum[i] += embedding[i];
+              }
+              return sum;
+            },
+            new Array(this.dimensions).fill(0)
+          );
+
+          const finalEmbedding = avgEmbedding.map(v => v / chunkEmbeddings.length);
+          
+          // Cache the result for the original text (using its hash)
+          this._cache.set(text, task, finalEmbedding);
+          console.log(`Successfully embedded long document as ${chunkEmbeddings.length} averaged chunks`);
+          
+          return finalEmbedding;
+        } catch (chunkError) {
+          // If chunking fails, throw the original error
+          console.warn(`Chunking failed, using original error:`, chunkError);
+          throw new Error(`Failed to generate embedding: ${errorMsg}`, { cause: error });
+        }
+      }
+
       if (error instanceof Error) {
         throw new Error(`Failed to generate embedding: ${error.message}`, { cause: error });
       }
@@ -326,6 +386,71 @@ export class Embedder {
 
       return results;
     } catch (error) {
+      // Check if this is a context length exceeded error and try chunking each text
+      const errorMsg = error instanceof Error ? error.message : String(error);
+      const isContextError = /context|too long|exceed|length/i.test(errorMsg);
+
+      if (isContextError && this._autoChunk) {
+        try {
+          console.log(`Batch embedding failed with context error, attempting chunking...`);
+          
+          const chunkResults = await Promise.all(
+            validTexts.map(async (text, idx) => {
+              const chunkResult = smartChunk(text, this._model);
+              if (chunkResult.chunks.length === 0) {
+                throw new Error("Chunker produced no chunks");
+              }
+
+              // Embed all chunks in parallel, then average.
+              const embeddings = await Promise.all(
+                chunkResult.chunks.map((chunk) => this.embedSingle(chunk, task))
+              );
+
+              const avgEmbedding = embeddings.reduce(
+                (sum, emb) => {
+                  for (let i = 0; i < emb.length; i++) {
+                    sum[i] += emb[i];
+                  }
+                  return sum;
+                },
+                new Array(this.dimensions).fill(0)
+              );
+
+              const finalEmbedding = avgEmbedding.map((v) => v / embeddings.length);
+
+              // Cache the averaged embedding for the original (long) text.
+              this._cache.set(text, task, finalEmbedding);
+
+              return { embedding: finalEmbedding, index: validIndices[idx] };
+            })
+          );
+
+          console.log(`Successfully chunked and embedded ${chunkResults.length} long documents`);
+
+          // Build results array
+          const results: number[][] = new Array(texts.length);
+          chunkResults.forEach(({ embedding, index }) => {
+            if (embedding.length > 0) {
+              this.validateEmbedding(embedding);
+              results[index] = embedding;
+            } else {
+              results[index] = [];
+            }
+          });
+
+          // Fill empty arrays for invalid texts
+          for (let i = 0; i < texts.length; i++) {
+            if (!results[i]) {
+              results[i] = [];
+            }
+          }
+
+          return results;
+        } catch (chunkError) {
+          throw new Error(`Failed to embed documents after chunking attempt: ${errorMsg}`);
+        }
+      }
+
       if (error instanceof Error) {
         throw new Error(`Failed to generate batch embeddings: ${error.message}`, { cause: error });
       }

package/src/store.ts

@@ -4,6 +4,8 @@
 
 import type * as LanceDB from "@lancedb/lancedb";
 import { randomUUID } from "node:crypto";
+import { existsSync, accessSync, constants, mkdirSync, realpathSync, lstatSync } from "node:fs";
+import { dirname } from "node:path";
 
 // ============================================================================
 // Types
@@ -60,6 +62,72 @@ function escapeSqlLiteral(value: string): string {
   return value.replace(/'/g, "''");
 }
 
+// ============================================================================
+// Storage Path Validation
+// ============================================================================
+
+/**
+ * Validate and prepare the storage directory before LanceDB connection.
+ * Resolves symlinks, creates missing directories, and checks write permissions.
+ * Returns the resolved absolute path on success, or throws a descriptive error.
+ */
+export function validateStoragePath(dbPath: string): string {
+  let resolvedPath = dbPath;
+
+  // Resolve symlinks (including dangling symlinks)
+  try {
+    const stats = lstatSync(dbPath);
+    if (stats.isSymbolicLink()) {
+      try {
+        resolvedPath = realpathSync(dbPath);
+      } catch (err: any) {
+        throw new Error(
+          `dbPath "${dbPath}" is a symlink whose target does not exist.\n` +
+          `  Fix: Create the target directory, or update the symlink to point to a valid path.\n` +
+          `  Details: ${err.code || ""} ${err.message}`
+        );
+      }
+    }
+  } catch (err: any) {
+    // Missing path is OK (it will be created below)
+    if (err?.code === "ENOENT") {
+      // no-op
+    } else if (typeof err?.message === "string" && err.message.includes("symlink whose target does not exist")) {
+      throw err;
+    } else {
+      // Other lstat failures — continue with original path
+    }
+  }
+
+  // Create directory if it doesn't exist
+  if (!existsSync(resolvedPath)) {
+    try {
+      mkdirSync(resolvedPath, { recursive: true });
+    } catch (err: any) {
+      throw new Error(
+        `Failed to create dbPath directory "${resolvedPath}".\n` +
+        `  Fix: Ensure the parent directory "${dirname(resolvedPath)}" exists and is writable,\n` +
+        `       or create it manually: mkdir -p "${resolvedPath}"\n` +
+        `  Details: ${err.code || ""} ${err.message}`
+      );
+    }
+  }
+
+  // Check write permissions
+  try {
+    accessSync(resolvedPath, constants.W_OK);
+  } catch (err: any) {
+    throw new Error(
+      `dbPath directory "${resolvedPath}" is not writable.\n` +
+      `  Fix: Check permissions with: ls -la "${dirname(resolvedPath)}"\n` +
+      `       Or grant write access: chmod u+w "${resolvedPath}"\n` +
+      `  Details: ${err.code || ""} ${err.message}`
+    );
+  }
+
+  return resolvedPath;
+}
+
 // ============================================================================
 // Memory Store
 // ============================================================================
@@ -95,7 +163,19 @@ export class MemoryStore {
 
   private async doInitialize(): Promise<void> {
     const lancedb = await loadLanceDB();
-    const db = await lancedb.connect(this.config.dbPath);
+
+    let db: LanceDB.Connection;
+    try {
+      db = await lancedb.connect(this.config.dbPath);
+    } catch (err: any) {
+      const code = err.code || "";
+      const message = err.message || String(err);
+      throw new Error(
+        `Failed to open LanceDB at "${this.config.dbPath}": ${code} ${message}\n` +
+        `  Fix: Verify the path exists and is writable. Check parent directory permissions.`
+      );
+    }
+
     let table: LanceDB.Table;
 
     // Idempotent table init: try openTable first, create only if missing,
@@ -196,7 +276,15 @@ export class MemoryStore {
       metadata: entry.metadata || "{}",
     };
 
-    await this.table!.add([fullEntry]);
+    try {
+      await this.table!.add([fullEntry]);
+    } catch (err: any) {
+      const code = err.code || "";
+      const message = err.message || String(err);
+      throw new Error(
+        `Failed to store memory in "${this.config.dbPath}": ${code} ${message}`
+      );
+    }
     return fullEntry;
   }