@prmichaelsen/remember-mcp 3.0.0 → 3.12.0

package/agent/tasks/task-76-fix-indexnullstate-schema-bug.md

@@ -0,0 +1,197 @@
+# Task 76: Fix indexNullState Schema Configuration Bug
+
+**Milestone**: M13 (Soft Delete System)
+**Estimated Time**: 1 hour
+**Dependencies**: Task 70 (Soft Delete Schema Fields)
+**Status**: Completed
+**Priority**: Critical
+
+---
+
+## Objective
+
+Fix critical Weaviate schema configuration bug where `deleted_at` field filtering fails because `indexNullState: true` was not configured in the inverted index. This prevents filtering on null values, which is required for the soft delete system's default behavior (exclude deleted memories).
+
+---
+
+## Problem
+
+**Error Message**:
+```
+WeaviateQueryError: Query call with protocol gRPC failed with message: 
+/weaviate.v1.Weaviate/Search UNKNOWN: object search at index memory_mnoyiarhz5b8n06tstovm582nsg2: 
+local shard object search memory_mnoyiarhz5b8n06tstovm582nsg2_12vqVNds7o70: 
+resolve doc ids for prop/value pair: nested AND/OR query: nested child 0: 
+Nullstate must be indexed to be filterable! 
+Add `indexNullState: true` to the invertedIndexConfig
+```
+
+**Root Cause**:
+- Soft delete system (v3.0.0) added `deleted_at` field to schema
+- Default filter uses `deleted_at IS NULL` to exclude deleted memories
+- Weaviate requires `indexNullState: true` to filter on null values
+- Schema was created without this configuration
+
+**Impact**:
+- 🚨 **CRITICAL**: All memory searches fail with gRPC error
+- Affects all users attempting to search memories
+- Soft delete system completely non-functional
+- Production service down
+
+---
+
+## Solution
+
+Add `invertedIndex` configuration with `indexNullState: true` to both schema files.
+
+**Note**: Existing collections must be recreated - this configuration cannot be updated on existing collections.
+
+---
+
+## Steps
+
+### 1. Fix Memory Schema ([`src/weaviate/schema.ts`](../../src/weaviate/schema.ts))
+
+Add inverted index configuration:
+
+```typescript
+await client.collections.create({
+  name: collectionName,
+  
+  // Vectorizer configuration
+  vectorizers: weaviate.configure.vectorizer.text2VecOpenAI({
+    model: 'text-embedding-3-small',
+    sourceProperties: ['content', 'title', 'summary', 'observation'],
+  }),
+
+  // Inverted index configuration
+  // indexNullState: true is required for filtering on null values (e.g., deleted_at IS NULL)
+  invertedIndex: weaviate.configure.invertedIndex({
+    indexNullState: true,
+  }),
+
+  properties: [
+    // ... existing properties
+  ]
+});
+```
+
+### 2. Fix Space Schema ([`src/weaviate/space-schema.ts`](../../src/weaviate/space-schema.ts))
+
+Add same configuration to public collection:
+
+```typescript
+await client.collections.create({
+  name: collectionName,
+
+  // Vectorizer configuration
+  vectorizers: weaviate.configure.vectorizer.text2VecOpenAI({
+    model: 'text-embedding-3-small',
+    sourceProperties: ['content', 'title', 'summary', 'observation'],
+  }),
+
+  // Inverted index configuration
+  // indexNullState: true is required for filtering on null values (e.g., deleted_at IS NULL)
+  invertedIndex: weaviate.configure.invertedIndex({
+    indexNullState: true,
+  }),
+
+  properties: [
+    // ... existing properties
+  ]
+});
+```
+
+### 3. Build and Test
+
+```bash
+npm run build
+npm test
+```
+
+**Expected**: All tests passing, TypeScript compiles without errors
+
+---
+
+## Verification
+
+- [x] Added `invertedIndex` configuration to [`src/weaviate/schema.ts`](../../src/weaviate/schema.ts)
+- [x] Added `invertedIndex` configuration to [`src/weaviate/space-schema.ts`](../../src/weaviate/space-schema.ts)
+- [x] Used correct API: `weaviate.configure.invertedIndex()` (not `invertedIndexConfig`)
+- [x] TypeScript compiles without errors
+- [x] Build successful
+- [x] All 93 tests passing
+
+---
+
+## Migration Notes
+
+**For Existing Collections**:
+
+⚠️ **CRITICAL**: Existing Weaviate collections must be recreated to apply this configuration change.
+
+**Migration Steps**:
+1. Export existing data from old collections
+2. Delete old collections
+3. Restart server (creates new collections with correct config)
+4. Re-import data
+
+**Alternative**: If no production data exists, simply delete collections and let them be recreated.
+
+**Command to delete collection** (use with caution):
+```typescript
+await client.collections.delete('Memory_{user_id}');
+await client.collections.delete('Memory_public');
+```
+
+---
+
+## Technical Details
+
+**Weaviate v3 API**:
+- Configuration uses `weaviate.configure.invertedIndex()` helper
+- Property is `invertedIndex` (not `invertedIndexConfig`)
+- `indexNullState: true` enables filtering on null values
+- Cannot be updated on existing collections (requires recreation)
+
+**Why This is Required**:
+- Soft delete uses `deleted_at IS NULL` filter to exclude deleted memories
+- Without `indexNullState: true`, Weaviate cannot filter on null values
+- Results in gRPC error: "Nullstate must be indexed to be filterable"
+
+**Performance Impact**:
+- Minimal - indexing null state adds small overhead
+- Required for soft delete functionality
+- Trade-off is acceptable for data safety
+
+---
+
+## Files Modified
+
+- [`src/weaviate/schema.ts`](../../src/weaviate/schema.ts) - Added invertedIndex config (line 55-58)
+- [`src/weaviate/space-schema.ts`](../../src/weaviate/space-schema.ts) - Added invertedIndex config (line 107-110)
+
+---
+
+## Related Issues
+
+**User Feedback**: "worked but gave this new error on next memory search"
+
+**Root Cause**: Task 70 added `deleted_at` field but didn't configure null state indexing
+
+**Prevention**: Always add `indexNullState: true` when creating nullable fields that will be filtered
+
+---
+
+## Next Steps
+
+1. Document this in CHANGELOG as patch release (v3.0.1)
+2. Notify users they need to recreate collections
+3. Add migration guide to README
+4. Consider adding collection recreation utility
+
+---
+
+**Status**: Completed (2026-02-25)
+**Version**: v3.0.1 (patch release)
+**Impact**: Critical bug fix - restores search functionality

package/dist/collections/composite-ids.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Composite ID Utilities
+ *
+ * Provides utilities for working with composite IDs in Memory Collection Pattern v2.
+ * Composite IDs preserve the source reference when memories are published to spaces or groups.
+ *
+ * Format: {userId}.{memoryId}
+ * Example: "user123.my-recipe"
+ */
+/**
+ * Components of a composite ID
+ */
+export interface CompositeIdComponents {
+    userId: string;
+    memoryId: string;
+}
+/**
+ * Error thrown when composite ID is invalid
+ */
+export declare class InvalidCompositeIdError extends Error {
+    constructor(message: string);
+}
+/**
+ * Generate a composite ID from user ID and memory ID
+ *
+ * @param userId - User ID (must not contain dots)
+ * @param memoryId - Memory ID (must not contain dots)
+ * @returns Composite ID in format {userId}.{memoryId}
+ * @throws {InvalidCompositeIdError} If userId or memoryId contains dots
+ *
+ * @example
+ * generateCompositeId('user123', 'my-recipe')
+ * // Returns: 'user123.my-recipe'
+ */
+export declare function generateCompositeId(userId: string, memoryId: string): string;
+/**
+ * Parse a composite ID into its components
+ *
+ * @param compositeId - Composite ID to parse
+ * @returns Object with userId and memoryId
+ * @throws {InvalidCompositeIdError} If composite ID format is invalid
+ *
+ * @example
+ * parseCompositeId('user123.my-recipe')
+ * // Returns: { userId: 'user123', memoryId: 'my-recipe' }
+ */
+export declare function parseCompositeId(compositeId: string): CompositeIdComponents;
+/**
+ * Check if a string is a composite ID
+ *
+ * @param id - String to check
+ * @returns true if valid composite ID, false otherwise
+ *
+ * @example
+ * isCompositeId('user123.my-recipe') // true
+ * isCompositeId('simple-id') // false
+ * isCompositeId('too.many.dots') // false
+ */
+export declare function isCompositeId(id: string): boolean;
+/**
+ * Validate a composite ID (returns true on valid, throws on invalid)
+ *
+ * @param id - Composite ID to validate
+ * @returns true if valid
+ * @throws {InvalidCompositeIdError} If composite ID is invalid
+ *
+ * @example
+ * validateCompositeId('user123.my-recipe') // Returns true
+ * validateCompositeId('invalid') // Throws InvalidCompositeIdError
+ */
+export declare function validateCompositeId(id: string): true;
+/**
+ * Extract user ID from a composite ID
+ *
+ * @param compositeId - Composite ID
+ * @returns User ID component
+ *
+ * @example
+ * getUserIdFromComposite('user123.my-recipe')
+ * // Returns: 'user123'
+ */
+export declare function getUserIdFromComposite(compositeId: string): string;
+/**
+ * Extract memory ID from a composite ID
+ *
+ * @param compositeId - Composite ID
+ * @returns Memory ID component
+ *
+ * @example
+ * getMemoryIdFromComposite('user123.my-recipe')
+ * // Returns: 'my-recipe'
+ */
+export declare function getMemoryIdFromComposite(compositeId: string): string;
+/**
+ * Check if an ID belongs to a specific user
+ *
+ * @param compositeId - Composite ID to check
+ * @param userId - User ID to match
+ * @returns true if composite ID belongs to user, false otherwise
+ *
+ * @example
+ * belongsToUser('user123.my-recipe', 'user123') // true
+ * belongsToUser('user123.my-recipe', 'user456') // false
+ */
+export declare function belongsToUser(compositeId: string, userId: string): boolean;
+//# sourceMappingURL=composite-ids.d.ts.map

package/dist/collections/core-infrastructure.spec.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Unit tests for Memory Collection Pattern v2 Core Infrastructure
+ *
+ * Tests core functionality of:
+ * - Dot notation collection utilities
+ * - Composite ID utilities
+ * - Tracking array management
+ * - Schema definitions
+ */
+export {};
+//# sourceMappingURL=core-infrastructure.spec.d.ts.map

package/dist/collections/dot-notation.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Dot Notation Collection Utilities
+ *
+ * Provides utilities for working with Memory Collection Pattern v2's
+ * dot notation collection naming scheme.
+ *
+ * Collection Types:
+ * - USERS: Memory_users_{userId} - Private user memories
+ * - SPACES: Memory_spaces_public - All public space memories
+ * - GROUPS: Memory_groups_{groupId} - Group memories
+ */
+/**
+ * Collection type enum for Memory Collection Pattern v2
+ */
+export declare enum CollectionType {
+    USERS = "USERS",
+    SPACES = "SPACES",
+    GROUPS = "GROUPS"
+}
+/**
+ * Metadata about a parsed collection
+ */
+export interface CollectionMetadata {
+    type: CollectionType;
+    id?: string;
+    name: string;
+}
+/**
+ * Error thrown when collection name is invalid
+ */
+export declare class InvalidCollectionNameError extends Error {
+    constructor(message: string);
+}
+/**
+ * Get the Weaviate collection name for a given type and optional ID
+ *
+ * @param type - Collection type (USERS, SPACES, or GROUPS)
+ * @param id - Optional ID (required for USERS and GROUPS, not used for SPACES)
+ * @returns Weaviate collection name
+ *
+ * @example
+ * getCollectionName(CollectionType.USERS, 'user123')
+ * // Returns: 'Memory_users_user123'
+ *
+ * @example
+ * getCollectionName(CollectionType.SPACES)
+ * // Returns: 'Memory_spaces_public'
+ *
+ * @example
+ * getCollectionName(CollectionType.GROUPS, 'group456')
+ * // Returns: 'Memory_groups_group456'
+ */
+export declare function getCollectionName(type: CollectionType, id?: string): string;
+/**
+ * Parse a collection name into its components
+ *
+ * @param name - Weaviate collection name
+ * @returns Collection metadata with type, optional ID, and name
+ *
+ * @example
+ * parseCollectionName('Memory_users_user123')
+ * // Returns: { type: CollectionType.USERS, id: 'user123', name: 'Memory_users_user123' }
+ *
+ * @example
+ * parseCollectionName('Memory_spaces_public')
+ * // Returns: { type: CollectionType.SPACES, id: undefined, name: 'Memory_spaces_public' }
+ *
+ * @example
+ * parseCollectionName('Memory_groups_group456')
+ * // Returns: { type: CollectionType.GROUPS, id: 'group456', name: 'Memory_groups_group456' }
+ */
+export declare function parseCollectionName(name: string): CollectionMetadata;
+/**
+ * Validate a collection name
+ *
+ * @param name - Collection name to validate
+ * @returns true if valid, false otherwise
+ *
+ * @example
+ * validateCollectionName('Memory_users_user123') // true
+ * validateCollectionName('Memory_spaces_public') // true
+ * validateCollectionName('Invalid_name') // false
+ */
+export declare function validateCollectionName(name: string): boolean;
+/**
+ * Check if a collection name is a user collection
+ *
+ * @param name - Collection name to check
+ * @returns true if user collection, false otherwise
+ */
+export declare function isUserCollection(name: string): boolean;
+/**
+ * Check if a collection name is the spaces collection
+ *
+ * @param name - Collection name to check
+ * @returns true if spaces collection, false otherwise
+ */
+export declare function isSpacesCollection(name: string): boolean;
+/**
+ * Check if a collection name is a group collection
+ *
+ * @param name - Collection name to check
+ * @returns true if group collection, false otherwise
+ */
+export declare function isGroupCollection(name: string): boolean;
+//# sourceMappingURL=dot-notation.d.ts.map

package/dist/collections/tracking-arrays.d.ts

@@ -0,0 +1,176 @@
+/**
+ * Tracking Array Management
+ *
+ * Provides utilities for managing space_ids and group_ids tracking arrays
+ * in Memory Collection Pattern v2. These arrays track where memories have
+ * been published.
+ */
+/**
+ * Memory with tracking arrays
+ */
+export interface MemoryWithTracking {
+    space_ids: string[];
+    group_ids: string[];
+    [key: string]: any;
+}
+/**
+ * Published locations summary
+ */
+export interface PublishedLocations {
+    spaces: string[];
+    groups: string[];
+}
+/**
+ * Add a space ID to the space_ids array (immutable)
+ *
+ * @param memory - Memory object with tracking arrays
+ * @param spaceId - Space ID to add
+ * @returns New memory object with updated space_ids
+ *
+ * @example
+ * const memory = { space_ids: ['cooking'], group_ids: [] }
+ * addToSpaceIds(memory, 'recipes')
+ * // Returns: { space_ids: ['cooking', 'recipes'], group_ids: [] }
+ */
+export declare function addToSpaceIds<T extends MemoryWithTracking>(memory: T, spaceId: string): T;
+/**
+ * Remove a space ID from the space_ids array (immutable)
+ *
+ * @param memory - Memory object with tracking arrays
+ * @param spaceId - Space ID to remove
+ * @returns New memory object with updated space_ids
+ *
+ * @example
+ * const memory = { space_ids: ['cooking', 'recipes'], group_ids: [] }
+ * removeFromSpaceIds(memory, 'cooking')
+ * // Returns: { space_ids: ['recipes'], group_ids: [] }
+ */
+export declare function removeFromSpaceIds<T extends MemoryWithTracking>(memory: T, spaceId: string): T;
+/**
+ * Add a group ID to the group_ids array (immutable)
+ *
+ * @param memory - Memory object with tracking arrays
+ * @param groupId - Group ID to add
+ * @returns New memory object with updated group_ids
+ *
+ * @example
+ * const memory = { space_ids: [], group_ids: ['family'] }
+ * addToGroupIds(memory, 'friends')
+ * // Returns: { space_ids: [], group_ids: ['family', 'friends'] }
+ */
+export declare function addToGroupIds<T extends MemoryWithTracking>(memory: T, groupId: string): T;
+/**
+ * Remove a group ID from the group_ids array (immutable)
+ *
+ * @param memory - Memory object with tracking arrays
+ * @param groupId - Group ID to remove
+ * @returns New memory object with updated group_ids
+ *
+ * @example
+ * const memory = { space_ids: [], group_ids: ['family', 'friends'] }
+ * removeFromGroupIds(memory, 'family')
+ * // Returns: { space_ids: [], group_ids: ['friends'] }
+ */
+export declare function removeFromGroupIds<T extends MemoryWithTracking>(memory: T, groupId: string): T;
+/**
+ * Check if memory is published to a specific space
+ *
+ * @param memory - Memory object with tracking arrays
+ * @param spaceId - Space ID to check
+ * @returns true if published to space, false otherwise
+ *
+ * @example
+ * const memory = { space_ids: ['cooking'], group_ids: [] }
+ * isPublishedToSpace(memory, 'cooking') // true
+ * isPublishedToSpace(memory, 'recipes') // false
+ */
+export declare function isPublishedToSpace(memory: MemoryWithTracking, spaceId: string): boolean;
+/**
+ * Check if memory is published to a specific group
+ *
+ * @param memory - Memory object with tracking arrays
+ * @param groupId - Group ID to check
+ * @returns true if published to group, false otherwise
+ *
+ * @example
+ * const memory = { space_ids: [], group_ids: ['family'] }
+ * isPublishedToGroup(memory, 'family') // true
+ * isPublishedToGroup(memory, 'friends') // false
+ */
+export declare function isPublishedToGroup(memory: MemoryWithTracking, groupId: string): boolean;
+/**
+ * Get all published locations for a memory
+ *
+ * @param memory - Memory object with tracking arrays
+ * @returns Object with spaces and groups arrays
+ *
+ * @example
+ * const memory = { space_ids: ['cooking', 'recipes'], group_ids: ['family'] }
+ * getPublishedLocations(memory)
+ * // Returns: { spaces: ['cooking', 'recipes'], groups: ['family'] }
+ */
+export declare function getPublishedLocations(memory: MemoryWithTracking): PublishedLocations;
+/**
+ * Check if memory is published anywhere
+ *
+ * @param memory - Memory object with tracking arrays
+ * @returns true if published to any space or group, false otherwise
+ *
+ * @example
+ * const memory = { space_ids: ['cooking'], group_ids: [] }
+ * isPublished(memory) // true
+ *
+ * const unpublished = { space_ids: [], group_ids: [] }
+ * isPublished(unpublished) // false
+ */
+export declare function isPublished(memory: MemoryWithTracking): boolean;
+/**
+ * Get count of published locations
+ *
+ * @param memory - Memory object with tracking arrays
+ * @returns Total number of spaces and groups memory is published to
+ *
+ * @example
+ * const memory = { space_ids: ['cooking', 'recipes'], group_ids: ['family'] }
+ * getPublishedCount(memory) // 3
+ */
+export declare function getPublishedCount(memory: MemoryWithTracking): number;
+/**
+ * Initialize tracking arrays on a memory object (immutable)
+ *
+ * @param memory - Memory object (may not have tracking arrays)
+ * @returns New memory object with initialized tracking arrays
+ *
+ * @example
+ * const memory = { id: '123', content: 'test' }
+ * initializeTracking(memory)
+ * // Returns: { id: '123', content: 'test', space_ids: [], group_ids: [] }
+ */
+export declare function initializeTracking<T extends Record<string, any>>(memory: T): T & MemoryWithTracking;
+/**
+ * Add multiple space IDs at once (immutable)
+ *
+ * @param memory - Memory object with tracking arrays
+ * @param spaceIds - Array of space IDs to add
+ * @returns New memory object with updated space_ids
+ *
+ * @example
+ * const memory = { space_ids: ['cooking'], group_ids: [] }
+ * addMultipleSpaceIds(memory, ['recipes', 'baking'])
+ * // Returns: { space_ids: ['cooking', 'recipes', 'baking'], group_ids: [] }
+ */
+export declare function addMultipleSpaceIds<T extends MemoryWithTracking>(memory: T, spaceIds: string[]): T;
+/**
+ * Add multiple group IDs at once (immutable)
+ *
+ * @param memory - Memory object with tracking arrays
+ * @param groupIds - Array of group IDs to add
+ * @returns New memory object with updated group_ids
+ *
+ * @example
+ * const memory = { space_ids: [], group_ids: ['family'] }
+ * addMultipleGroupIds(memory, ['friends', 'coworkers'])
+ * // Returns: { space_ids: [], group_ids: ['family', 'friends', 'coworkers'] }
+ */
+export declare function addMultipleGroupIds<T extends MemoryWithTracking>(memory: T, groupIds: string[]): T;
+//# sourceMappingURL=tracking-arrays.d.ts.map

package/dist/constants/content-types.d.ts

@@ -35,6 +35,7 @@ export declare const CONTENT_TYPE_CATEGORIES: {
     readonly organizational: readonly ["bookmark", "form", "location"];
     readonly business: readonly ["invoice", "contract"];
     readonly system: readonly ["system", "action", "audit", "history"];
+    readonly cross_user: readonly ["ghost", "comment"];
 };
 /**
  * Get content type metadata

package/dist/schema/v2-collections-comments.spec.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Tests for comment-related schema fields in v2 collections.
+ *
+ * Verifies that parent_id, thread_root_id, and moderation_flags
+ * are present in both user and published collection schemas.
+ */
+export {};
+//# sourceMappingURL=v2-collections-comments.spec.d.ts.map