@utilarium/overcontext 0.0.6 → 0.0.7-dev.20260220025748.d61df74

package/README.md

@@ -2,6 +2,34 @@
 
 > Schema-driven framework for context management
 
+## Why Overcontext?
+
+The world is waking up to the idea that **context is king** when interacting with Large Language Models. As Andrej Karpathy put it, "context engineering" is "the delicate art and science of filling the context window with just the right information." The industry is shifting from prompt engineering to context engineering -- and the tooling ecosystem is racing to catch up.
+
+The current landscape is converging on a common pattern: **large collections of Markdown files**. Whether it's [CLAUDE.md](https://docs.anthropic.com/en/docs/claude-code/memory) for Claude Code, [AGENTS.md](https://github.com/agentsmd/agents.md) as an open standard under the Linux Foundation, [.cursor/rules/](https://docs.cursor.com/context/rules-for-ai) for Cursor, or [copilot-instructions.md](https://docs.github.com/en/copilot/customizing-copilot/adding-repository-custom-instructions) for GitHub Copilot -- the default answer to "how do I give my AI more context?" is always the same: write more Markdown.
+
+Markdown is a fine starting point, but it has limits. It's unstructured, unsearchable at the entity level, hard to layer across scopes, and impossible to validate. As systems move toward more agentic interaction patterns -- where AI agents dynamically search for and compose context at runtime -- we need something more structured.
+
+**Overcontext is a reaction to that.** It started as a strategy for managing structured context files about people, projects, and terminology to support transcription tools like [Protokoll](https://github.com/redaksjon/protokoll) and other tools under development. Protokoll transforms voice memos into organized, context-aware notes by maintaining a knowledge base of people, projects, and organizations as YAML files -- using that context to correct proper names, classify content, and route transcriptions to the right places. As Protokoll's context needs grew, it became clear that a general-purpose framework was needed: one that could not only *store* context entities but *retrieve*, *search*, *layer*, and *validate* them with the rigor that agentic systems demand.
+
+Overcontext provides that framework. It gives you schema-validated, type-safe context entities with hierarchical discovery, namespace isolation, and a query API -- the building blocks for context engineering that goes beyond flat files.
+
+### How Overcontext Compares
+
+The context management landscape is broad. Here's where Overcontext fits:
+
+| Approach | Examples | What They Do | What's Missing |
+|---|---|---|---|
+| **Agent instruction files** | CLAUDE.md, AGENTS.md, .cursorrules | Behavioral instructions for AI agents | No entity model, no search, no validation |
+| **Codebase-to-prompt tools** | [Repomix](https://github.com/yamadashy/repomix), [files-to-prompt](https://github.com/simonw/files-to-prompt), [yek](https://github.com/mohsen1/yek) | Serialize code into LLM-friendly formats | One-shot dumps, no structured entities or queries |
+| **AI memory services** | [Mem0](https://mem0.ai), [Letta](https://letta.com), [Zep](https://getzep.com) | Persistent memory via embeddings and knowledge graphs | Cloud-dependent, opaque storage, not file-based |
+| **MCP servers** | [Model Context Protocol](https://modelcontextprotocol.io) | Standardized protocol for connecting AI to data sources | A transport layer, not a storage or schema framework |
+| **Overcontext** | This library | Schema-validated entities with hierarchical discovery, namespaces, and search | Designed to be the structured layer *underneath* all of the above |
+
+Overcontext isn't trying to replace Markdown instruction files or MCP -- it's the structured entity layer that those systems can build on. Your `CLAUDE.md` can reference Overcontext entities. Your MCP server can serve them. Your CLI tools can query them. The context you maintain is schema-validated, version-controllable, and yours.
+
+---
+
 Overcontext provides infrastructure for defining and managing custom entity schemas. Unlike a library with predefined types, overcontext lets you define your own entity schemas using Zod and provides the storage, validation, discovery, and CLI building blocks to work with them.
 
 ## Features

package/dist/api/slug.js

@@ -37,6 +37,15 @@
  * Generate a unique ID, appending a number if needed.
  */ const generateUniqueId = async (baseName, exists)=>{
     const baseId = slugify(baseName);
+    if (!baseId) {
+        // slugify produced an empty string (input was all special characters).
+        // Fall back to a random ID to avoid creating entities with empty IDs.
+        const fallbackId = `entity-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
+        if (!await exists(fallbackId)) {
+            return fallbackId;
+        }
+        return `${fallbackId}-${Math.random().toString(36).substring(2, 8)}`;
+    }
     if (!await exists(baseId)) {
         return baseId;
     }
@@ -47,8 +56,9 @@
             return candidateId;
         }
     }
-    // Fallback to timestamp
-    return `${baseId}-${Date.now()}`;
+    // Fallback to timestamp + random suffix to reduce predictability and
+    // avoid collisions in concurrent environments.
+    return `${baseId}-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
 };
 
 export { generateUniqueId, slugify };

package/dist/api/slug.js.map

@@ -1 +1 @@
-{"version":3,"file":"slug.js","sources":["../../src/api/slug.ts"],"sourcesContent":["/**\n * Generate a slug from a name.\n * \"John Doe\" -> \"john-doe\"\n * \"API Reference\" -> \"api-reference\"\n */\nexport const slugify = (name: string): string => {\n    const cleaned = name\n        .toLowerCase()\n        .trim()\n        .replace(/[^\\w\\s-]/g, '')     // Remove non-word chars\n        .replace(/[\\s_]+/g, '-');     // Replace spaces/underscores\n    \n    // Single-pass algorithm to collapse hyphens and trim (avoid ReDoS)\n    const result: string[] = [];\n    let prevWasHyphen = false;\n    let startIdx = 0;\n    let endIdx = cleaned.length;\n    \n    // Skip leading hyphens\n    while (startIdx < cleaned.length && cleaned[startIdx] === '-') {\n        startIdx++;\n    }\n    \n    // Skip trailing hyphens\n    while (endIdx > startIdx && cleaned[endIdx - 1] === '-') {\n        endIdx--;\n    }\n    \n    // Build result, collapsing consecutive hyphens\n    for (let i = startIdx; i < endIdx; i++) {\n        const char = cleaned[i];\n        if (char === '-') {\n            if (!prevWasHyphen) {\n                result.push('-');\n                prevWasHyphen = true;\n            }\n        } else {\n            result.push(char);\n            prevWasHyphen = false;\n        }\n    }\n    \n    return result.join('');\n};\n\n/**\n * Generate a unique ID, appending a number if needed.\n */\nexport const generateUniqueId = async (\n    baseName: string,\n    exists: (id: string) => Promise<boolean>\n): Promise<string> => {\n    const baseId = slugify(baseName);\n\n    if (!await exists(baseId)) {\n        return baseId;\n    }\n\n    // Try appending numbers\n    for (let i = 2; i <= 100; i++) {\n        const candidateId = `${baseId}-${i}`;\n        if (!await exists(candidateId)) {\n            return candidateId;\n        }\n    }\n\n    // Fallback to timestamp\n    return `${baseId}-${Date.now()}`;\n};\n"],"names":["slugify","name","cleaned","toLowerCase","trim","replace","result","prevWasHyphen","startIdx","endIdx","length","i","char","push","join","generateUniqueId","baseName","exists","baseId","candidateId","Date","now"],"mappings":"AAAA;;;;IAKO,MAAMA,OAAAA,GAAU,CAACC,IAAAA,GAAAA;IACpB,MAAMC,OAAAA,GAAUD,IAAAA,CACXE,WAAW,EAAA,CACXC,IAAI,GACJC,OAAO,CAAC,WAAA,EAAa,EAAA,CAAA;KACrBA,OAAO,CAAC,SAAA,EAAW,GAAA,CAAA,CAAA;;AAGxB,IAAA,MAAMC,SAAmB,EAAE;AAC3B,IAAA,IAAIC,aAAAA,GAAgB,KAAA;AACpB,IAAA,IAAIC,QAAAA,GAAW,CAAA;IACf,IAAIC,MAAAA,GAASP,QAAQQ,MAAM;;IAG3B,MAAOF,QAAAA,GAAWN,QAAQQ,MAAM,IAAIR,OAAO,CAACM,QAAAA,CAAS,KAAK,GAAA,CAAK;AAC3DA,QAAAA,QAAAA,EAAAA;AACJ,IAAA;;AAGA,IAAA,MAAOC,SAASD,QAAAA,IAAYN,OAAO,CAACO,MAAAA,GAAS,CAAA,CAAE,KAAK,GAAA,CAAK;AACrDA,QAAAA,MAAAA,EAAAA;AACJ,IAAA;;AAGA,IAAA,IAAK,IAAIE,CAAAA,GAAIH,QAAAA,EAAUG,CAAAA,GAAIF,QAAQE,CAAAA,EAAAA,CAAK;QACpC,MAAMC,IAAAA,GAAOV,OAAO,CAACS,CAAAA,CAAE;AACvB,QAAA,IAAIC,SAAS,GAAA,EAAK;AACd,YAAA,IAAI,CAACL,aAAAA,EAAe;AAChBD,gBAAAA,MAAAA,CAAOO,IAAI,CAAC,GAAA,CAAA;gBACZN,aAAAA,GAAgB,IAAA;AACpB,YAAA;QACJ,CAAA,MAAO;AACHD,YAAAA,MAAAA,CAAOO,IAAI,CAACD,IAAAA,CAAAA;YACZL,aAAAA,GAAgB,KAAA;AACpB,QAAA;AACJ,IAAA;IAEA,OAAOD,MAAAA,CAAOQ,IAAI,CAAC,EAAA,CAAA;AACvB;AAEA;;AAEC,IACM,MAAMC,gBAAAA,GAAmB,OAC5BC,QAAAA,EACAC,MAAAA,GAAAA;AAEA,IAAA,MAAMC,SAASlB,OAAAA,CAAQgB,QAAAA,CAAAA;IAEvB,IAAI,CAAC,MAAMC,MAAAA,CAAOC,MAAAA,CAAAA,EAAS;QACvB,OAAOA,MAAAA;AACX,IAAA;;AAGA,IAAA,IAAK,IAAIP,CAAAA,GAAI,CAAA,EAAGA,CAAAA,IAAK,KAAKA,CAAAA,EAAAA,CAAK;AAC3B,QAAA,MAAMQ,WAAAA,GAAc,CAAA,EAAGD,MAAAA,CAAO,CAAC,EAAEP,CAAAA,CAAAA,CAAG;QACpC,IAAI,CAAC,MAAMM,MAAAA,CAAOE,WAAAA,CAAAA,EAAc;YAC5B,OAAOA,WAAAA;AACX,QAAA;AACJ,IAAA;;AAGA,IAAA,OAAO,GAAGD,MAAAA,CAAO,CAAC,EAAEE,IAAAA,CAAKC,GAAG,EAAA,CAAA,CAAI;AACpC;;;;"}
+{"version":3,"file":"slug.js","sources":["../../src/api/slug.ts"],"sourcesContent":["/**\n * Generate a slug from a name.\n * \"John Doe\" -> \"john-doe\"\n * \"API Reference\" -> \"api-reference\"\n */\nexport const slugify = (name: string): string => {\n    const cleaned = name\n        .toLowerCase()\n        .trim()\n        .replace(/[^\\w\\s-]/g, '')     // Remove non-word chars\n        .replace(/[\\s_]+/g, '-');     // Replace spaces/underscores\n    \n    // Single-pass algorithm to collapse hyphens and trim (avoid ReDoS)\n    const result: string[] = [];\n    let prevWasHyphen = false;\n    let startIdx = 0;\n    let endIdx = cleaned.length;\n    \n    // Skip leading hyphens\n    while (startIdx < cleaned.length && cleaned[startIdx] === '-') {\n        startIdx++;\n    }\n    \n    // Skip trailing hyphens\n    while (endIdx > startIdx && cleaned[endIdx - 1] === '-') {\n        endIdx--;\n    }\n    \n    // Build result, collapsing consecutive hyphens\n    for (let i = startIdx; i < endIdx; i++) {\n        const char = cleaned[i];\n        if (char === '-') {\n            if (!prevWasHyphen) {\n                result.push('-');\n                prevWasHyphen = true;\n            }\n        } else {\n            result.push(char);\n            prevWasHyphen = false;\n        }\n    }\n    \n    return result.join('');\n};\n\n/**\n * Generate a unique ID, appending a number if needed.\n */\nexport const generateUniqueId = async (\n    baseName: string,\n    exists: (id: string) => Promise<boolean>\n): Promise<string> => {\n    const baseId = slugify(baseName);\n\n    if (!baseId) {\n        // slugify produced an empty string (input was all special characters).\n        // Fall back to a random ID to avoid creating entities with empty IDs.\n        const fallbackId = `entity-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;\n        if (!await exists(fallbackId)) {\n            return fallbackId;\n        }\n        return `${fallbackId}-${Math.random().toString(36).substring(2, 8)}`;\n    }\n\n    if (!await exists(baseId)) {\n        return baseId;\n    }\n\n    // Try appending numbers\n    for (let i = 2; i <= 100; i++) {\n        const candidateId = `${baseId}-${i}`;\n        if (!await exists(candidateId)) {\n            return candidateId;\n        }\n    }\n\n    // Fallback to timestamp + random suffix to reduce predictability and\n    // avoid collisions in concurrent environments.\n    return `${baseId}-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;\n};\n"],"names":["slugify","name","cleaned","toLowerCase","trim","replace","result","prevWasHyphen","startIdx","endIdx","length","i","char","push","join","generateUniqueId","baseName","exists","baseId","fallbackId","Date","now","Math","random","toString","substring","candidateId"],"mappings":"AAAA;;;;IAKO,MAAMA,OAAAA,GAAU,CAACC,IAAAA,GAAAA;IACpB,MAAMC,OAAAA,GAAUD,IAAAA,CACXE,WAAW,EAAA,CACXC,IAAI,GACJC,OAAO,CAAC,WAAA,EAAa,EAAA,CAAA;KACrBA,OAAO,CAAC,SAAA,EAAW,GAAA,CAAA,CAAA;;AAGxB,IAAA,MAAMC,SAAmB,EAAE;AAC3B,IAAA,IAAIC,aAAAA,GAAgB,KAAA;AACpB,IAAA,IAAIC,QAAAA,GAAW,CAAA;IACf,IAAIC,MAAAA,GAASP,QAAQQ,MAAM;;IAG3B,MAAOF,QAAAA,GAAWN,QAAQQ,MAAM,IAAIR,OAAO,CAACM,QAAAA,CAAS,KAAK,GAAA,CAAK;AAC3DA,QAAAA,QAAAA,EAAAA;AACJ,IAAA;;AAGA,IAAA,MAAOC,SAASD,QAAAA,IAAYN,OAAO,CAACO,MAAAA,GAAS,CAAA,CAAE,KAAK,GAAA,CAAK;AACrDA,QAAAA,MAAAA,EAAAA;AACJ,IAAA;;AAGA,IAAA,IAAK,IAAIE,CAAAA,GAAIH,QAAAA,EAAUG,CAAAA,GAAIF,QAAQE,CAAAA,EAAAA,CAAK;QACpC,MAAMC,IAAAA,GAAOV,OAAO,CAACS,CAAAA,CAAE;AACvB,QAAA,IAAIC,SAAS,GAAA,EAAK;AACd,YAAA,IAAI,CAACL,aAAAA,EAAe;AAChBD,gBAAAA,MAAAA,CAAOO,IAAI,CAAC,GAAA,CAAA;gBACZN,aAAAA,GAAgB,IAAA;AACpB,YAAA;QACJ,CAAA,MAAO;AACHD,YAAAA,MAAAA,CAAOO,IAAI,CAACD,IAAAA,CAAAA;YACZL,aAAAA,GAAgB,KAAA;AACpB,QAAA;AACJ,IAAA;IAEA,OAAOD,MAAAA,CAAOQ,IAAI,CAAC,EAAA,CAAA;AACvB;AAEA;;AAEC,IACM,MAAMC,gBAAAA,GAAmB,OAC5BC,QAAAA,EACAC,MAAAA,GAAAA;AAEA,IAAA,MAAMC,SAASlB,OAAAA,CAAQgB,QAAAA,CAAAA;AAEvB,IAAA,IAAI,CAACE,MAAAA,EAAQ;;;AAGT,QAAA,MAAMC,aAAa,CAAC,OAAO,EAAEC,IAAAA,CAAKC,GAAG,GAAG,CAAC,EAAEC,IAAAA,CAAKC,MAAM,GAAGC,QAAQ,CAAC,IAAIC,SAAS,CAAC,GAAG,CAAA,CAAA,CAAA,CAAI;QACvF,IAAI,CAAC,MAAMR,MAAAA,CAAOE,UAAAA,CAAAA,EAAa;YAC3B,OAAOA,UAAAA;AACX,QAAA;AACA,QAAA,OAAO,CAAA,EAAGA,UAAAA,CAAW,CAAC,EAAEG,IAAAA,CAAKC,MAAM,EAAA,CAAGC,QAAQ,CAAC,EAAA,CAAA,CAAIC,SAAS,CAAC,GAAG,CAAA,CAAA,CAAA,CAAI;AACxE,IAAA;IAEA,IAAI,CAAC,MAAMR,MAAAA,CAAOC,MAAAA,CAAAA,EAAS;QACvB,OAAOA,MAAAA;AACX,IAAA;;AAGA,IAAA,IAAK,IAAIP,CAAAA,GAAI,CAAA,EAAGA,CAAAA,IAAK,KAAKA,CAAAA,EAAAA,CAAK;AAC3B,QAAA,MAAMe,WAAAA,GAAc,CAAA,EAAGR,MAAAA,CAAO,CAAC,EAAEP,CAAAA,CAAAA,CAAG;QACpC,IAAI,CAAC,MAAMM,MAAAA,CAAOS,WAAAA,CAAAA,EAAc;YAC5B,OAAOA,WAAAA;AACX,QAAA;AACJ,IAAA;;;AAIA,IAAA,OAAO,GAAGR,MAAAA,CAAO,CAAC,EAAEE,IAAAA,CAAKC,GAAG,GAAG,CAAC,EAAEC,IAAAA,CAAKC,MAAM,GAAGC,QAAQ,CAAC,IAAIC,SAAS,CAAC,GAAG,CAAA,CAAA,CAAA,CAAI;AAClF;;;;"}

package/dist/index.cjs

@@ -411,6 +411,10 @@ class NamespaceNotFoundError extends StorageError {
     };
 };
 
+/**
+ * Maximum number of entities allowed in a single batch operation.
+ * Prevents resource exhaustion from unbounded batch sizes.
+ */ const MAX_BATCH_SIZE = 1000;
 const createFileSystemProvider = async (options)=>{
     const { basePath, registry, createIfMissing = true, extension = '.yaml', readonly = false, defaultNamespace, filenameStrategy } = options;
     // --- Helper Functions ---
@@ -521,10 +525,8 @@ const createFileSystemProvider = async (options)=>{
         const prefix = id.substring(0, Math.min(8, id.length));
         const files = await fs__namespace.readdir(dir);
         const candidates = files.filter((f)=>(f.endsWith('.yaml') || f.endsWith('.yml')) && f.startsWith(prefix));
-        if (candidates.length === 1) {
-            return path__namespace.join(dir, candidates[0]);
-        }
-        // Multiple prefix matches — read and verify the id field
+        // Always verify the entity ID inside the file to prevent
+        // returning the wrong entity when IDs share a common prefix.
         for (const file of candidates){
             const filePath = path__namespace.join(dir, file);
             const entity = await readEntity(filePath, type);
@@ -541,8 +543,35 @@ const createFileSystemProvider = async (options)=>{
             });
         }
     };
+    // Cache the real basePath to handle symlinked temp directories (e.g., macOS /var/folders -> /private/var/folders).
+    // Computed lazily on first readEntity call once the directory exists.
+    let realBasePath;
     const readEntity = async (filePath, type)=>{
         try {
+            // Resolve symlinks and verify the real path is within basePath
+            // to prevent symlink-based path traversal.
+            let realFilePath;
+            try {
+                realFilePath = await fs__namespace.realpath(filePath);
+            } catch  {
+                // If realpath fails (e.g., file doesn't exist), fall through to readFile
+                // which will throw ENOENT and be handled below.
+                realFilePath = filePath;
+            }
+            // Resolve basePath's real path to handle symlinked parent directories.
+            // Fall back to path.resolve if basePath doesn't exist yet.
+            if (!realBasePath) {
+                try {
+                    realBasePath = await fs__namespace.realpath(basePath);
+                } catch  {
+                    realBasePath = path__namespace.resolve(basePath);
+                }
+            }
+            if (!realFilePath.startsWith(realBasePath + path__namespace.sep) && realFilePath !== realBasePath) {
+                // eslint-disable-next-line no-console
+                console.warn(`Symlink traversal attempt detected: ${filePath} resolves to ${realFilePath}`);
+                return undefined;
+            }
             const content = await fs__namespace.readFile(filePath, 'utf-8');
             let parsed;
             try {
@@ -555,14 +584,29 @@ const createFileSystemProvider = async (options)=>{
             if (!parsed || typeof parsed !== 'object') {
                 return undefined;
             }
-            // Protect against prototype pollution from malicious YAML
-            // Only __proto__ is dangerous - constructor/prototype as string keys are safe
-            const parsedObj = parsed;
-            if (Object.prototype.hasOwnProperty.call(parsedObj, '__proto__')) {
+            // Protect against prototype pollution from malicious YAML.
+            // Recursively check for __proto__ keys at all nesting levels,
+            // since nested __proto__ can still cause pollution when objects
+            // are spread or assigned downstream.
+            const hasProtoKey = (obj)=>{
+                if (!obj || typeof obj !== 'object') return false;
+                if (Object.prototype.hasOwnProperty.call(obj, '__proto__')) return true;
+                for (const value of Object.values(obj)){
+                    if (hasProtoKey(value)) return true;
+                }
+                if (Array.isArray(obj)) {
+                    for (const item of obj){
+                        if (hasProtoKey(item)) return true;
+                    }
+                }
+                return false;
+            };
+            if (hasProtoKey(parsed)) {
                 // eslint-disable-next-line no-console
                 console.warn(`Potential prototype pollution attempt in ${filePath}: __proto__ key detected`);
                 return undefined;
             }
+            const parsedObj = parsed;
             // Validate against registered schema
             const result = registry.validateAs(type, {
                 ...parsedObj,
@@ -778,6 +822,9 @@ const createFileSystemProvider = async (options)=>{
             }
         },
         async saveBatch (entities, namespace) {
+            if (entities.length > MAX_BATCH_SIZE) {
+                throw new StorageAccessError(`Batch size ${entities.length} exceeds maximum of ${MAX_BATCH_SIZE}. Split into smaller batches.`);
+            }
             const saved = [];
             for (const entity of entities){
                 saved.push(await this.save(entity, namespace));
@@ -785,6 +832,9 @@ const createFileSystemProvider = async (options)=>{
             return saved;
         },
         async deleteBatch (refs, namespace) {
+            if (refs.length > MAX_BATCH_SIZE) {
+                throw new StorageAccessError(`Batch size ${refs.length} exceeds maximum of ${MAX_BATCH_SIZE}. Split into smaller batches.`);
+            }
             let count = 0;
             for (const ref of refs){
                 if (await this.delete(ref.type, ref.id, namespace)) {
@@ -1013,6 +1063,15 @@ const createMemoryProvider = (options)=>{
  * Generate a unique ID, appending a number if needed.
  */ const generateUniqueId = async (baseName, exists)=>{
     const baseId = slugify(baseName);
+    if (!baseId) {
+        // slugify produced an empty string (input was all special characters).
+        // Fall back to a random ID to avoid creating entities with empty IDs.
+        const fallbackId = `entity-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
+        if (!await exists(fallbackId)) {
+            return fallbackId;
+        }
+        return `${fallbackId}-${Math.random().toString(36).substring(2, 8)}`;
+    }
     if (!await exists(baseId)) {
         return baseId;
     }
@@ -1023,8 +1082,9 @@ const createMemoryProvider = (options)=>{
             return candidateId;
         }
     }
-    // Fallback to timestamp
-    return `${baseId}-${Date.now()}`;
+    // Fallback to timestamp + random suffix to reduce predictability and
+    // avoid collisions in concurrent environments.
+    return `${baseId}-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
 };
 
 /**