neotoma 0.3.0 → 0.3.2

package/docs/releases/v0.3.0/migrations/01_add_source_graph_edges.sql

@@ -1,110 +0,0 @@
--- Migration: Add source-based graph edges for v0.3.0
--- Created: 2025-01-01
--- Description: Replace record-based graph edges with source-based graph edges
-
--- Source-Entity edges table (replaces record_entity_edges)
-CREATE TABLE IF NOT EXISTS source_entity_edges (
-  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-  source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
-  entity_id TEXT NOT NULL REFERENCES entities(id),
-  edge_type TEXT NOT NULL DEFAULT 'EXTRACTED_FROM',
-  interpretation_id UUID REFERENCES interpretations(id),
-  created_at TIMESTAMPTZ DEFAULT NOW()
-);
-
--- Indexes for source_entity_edges
-CREATE INDEX IF NOT EXISTS idx_source_entity_edges_source ON source_entity_edges(source_id);
-CREATE INDEX IF NOT EXISTS idx_source_entity_edges_entity ON source_entity_edges(entity_id);
-CREATE INDEX IF NOT EXISTS idx_source_entity_edges_type ON source_entity_edges(edge_type);
-CREATE INDEX IF NOT EXISTS idx_source_entity_edges_interpretation ON source_entity_edges(interpretation_run_id);
-
--- RLS policies for source_entity_edges
-ALTER TABLE source_entity_edges ENABLE ROW LEVEL SECURITY;
-
-DROP POLICY IF EXISTS "Service role full access - source_entity_edges" ON source_entity_edges;
-CREATE POLICY "Service role full access - source_entity_edges" ON source_entity_edges
-  FOR ALL TO service_role USING (true) WITH CHECK (true);
-
--- Users can read edges for their own sources
-DROP POLICY IF EXISTS "Users read own source entity edges" ON source_entity_edges;
-CREATE POLICY "Users read own source entity edges"
-  ON source_entity_edges
-  FOR SELECT
-  TO authenticated
-  USING (
-    EXISTS (
-      SELECT 1 FROM sources
-      WHERE sources.id = source_entity_edges.source_id
-      AND sources.user_id = auth.uid()
-    )
-  );
-
-COMMENT ON TABLE source_entity_edges IS 'Links sources to entities extracted from them (replaces record_entity_edges)';
-COMMENT ON COLUMN source_entity_edges.interpretation_run_id IS 'Which interpretation run created this edge (for provenance)';
-
--- Source-Event edges table (replaces record_event_edges)
-CREATE TABLE IF NOT EXISTS source_event_edges (
-  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-  source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
-  event_id TEXT NOT NULL,
-  edge_type TEXT NOT NULL DEFAULT 'GENERATED_FROM',
-  created_at TIMESTAMPTZ DEFAULT NOW()
-);
-
--- Indexes for source_event_edges
-CREATE INDEX IF NOT EXISTS idx_source_event_edges_source ON source_event_edges(source_id);
-CREATE INDEX IF NOT EXISTS idx_source_event_edges_event ON source_event_edges(event_id);
-CREATE INDEX IF NOT EXISTS idx_source_event_edges_type ON source_event_edges(edge_type);
-
--- RLS policies for source_event_edges
-ALTER TABLE source_event_edges ENABLE ROW LEVEL SECURITY;
-
-DROP POLICY IF EXISTS "Service role full access - source_event_edges" ON source_event_edges;
-CREATE POLICY "Service role full access - source_event_edges" ON source_event_edges
-  FOR ALL TO service_role USING (true) WITH CHECK (true);
-
--- Users can read edges for their own sources
-DROP POLICY IF EXISTS "Users read own source event edges" ON source_event_edges;
-CREATE POLICY "Users read own source event edges"
-  ON source_event_edges
-  FOR SELECT
-  TO authenticated
-  USING (
-    EXISTS (
-      SELECT 1 FROM sources
-      WHERE sources.id = source_event_edges.source_id
-      AND sources.user_id = auth.uid()
-    )
-  );
-
-COMMENT ON TABLE source_event_edges IS 'Links sources to timeline events generated from them (replaces record_event_edges)';
-
--- Add source_id to timeline_events (making source_record_id optional for migration)
-DO $$ 
-BEGIN
-  IF NOT EXISTS (
-    SELECT 1 FROM information_schema.columns 
-    WHERE table_name = 'timeline_events' AND column_name = 'source_id'
-  ) THEN
-    ALTER TABLE timeline_events ADD COLUMN source_id UUID REFERENCES sources(id);
-  END IF;
-END $$;
-
--- Make source_record_id nullable (for migration period)
-ALTER TABLE timeline_events ALTER COLUMN source_record_id DROP NOT NULL;
-
--- Add index for source_id
-CREATE INDEX IF NOT EXISTS idx_timeline_events_source ON timeline_events(source_id);
-
-COMMENT ON COLUMN timeline_events.source_id IS 'Link to source that generated this event (new architecture)';
-COMMENT ON COLUMN timeline_events.source_record_id IS 'DEPRECATED: Legacy link to record (use source_id instead)';
-
--- Add check constraint to ensure at least one reference exists
-ALTER TABLE timeline_events DROP CONSTRAINT IF EXISTS timeline_events_source_check;
-ALTER TABLE timeline_events ADD CONSTRAINT timeline_events_source_check
-  CHECK (source_record_id IS NOT NULL OR source_id IS NOT NULL);
-
-
-
-
-

package/docs/releases/v0.3.0/migrations/02_drop_legacy_tables.sql

@@ -1,131 +0,0 @@
--- Migration: Drop legacy record-based tables for v0.3.0
--- Created: 2025-01-01
--- Description: Remove legacy records table and associated structures after migration
--- WARNING: This is a destructive migration. Ensure data is migrated before running.
-
--- ============================================================================
--- SAFETY CHECK: Verify migration is complete before running this
--- ============================================================================
-
--- This migration should only be run after:
--- 1. All records have been migrated to sources
--- 2. All graph edges have been migrated
--- 3. All timeline events have been updated
--- 4. Frontend has been updated to use new endpoints
--- 5. Full database backup has been taken
-
--- Recommended verification queries to run BEFORE this migration:
---
--- 1. Check for unmigrated records:
---    SELECT COUNT(*) FROM records WHERE id NOT IN (
---      SELECT provenance->>'migrated_from_record_id' FROM sources
---      WHERE provenance->>'migrated_from_record_id' IS NOT NULL
---    );
---
--- 2. Check for timeline events still using source_record_id:
---    SELECT COUNT(*) FROM timeline_events WHERE source_record_id IS NOT NULL AND source_id IS NULL;
---
--- 3. Check for observations still using source_payload_id:
---    SELECT COUNT(*) FROM observations WHERE source_payload_id IS NOT NULL AND source_id IS NULL;
-
--- ============================================================================
--- DROP FOREIGN KEY CONSTRAINTS
--- ============================================================================
-
--- Drop constraints from timeline_events
-ALTER TABLE timeline_events DROP CONSTRAINT IF EXISTS timeline_events_source_record_id_fkey;
-ALTER TABLE timeline_events DROP CONSTRAINT IF EXISTS timeline_events_source_check;
-
--- Drop constraints from raw_fragments
-ALTER TABLE raw_fragments DROP CONSTRAINT IF EXISTS raw_fragments_record_id_fkey;
-
--- Drop constraints from relationships
-ALTER TABLE relationships DROP CONSTRAINT IF EXISTS relationships_source_record_id_fkey;
-
--- Drop constraints from record_relationships
-ALTER TABLE record_relationships DROP CONSTRAINT IF EXISTS record_relationships_source_id_fkey;
-ALTER TABLE record_relationships DROP CONSTRAINT IF EXISTS record_relationships_target_id_fkey;
-
--- Drop constraints from record_entity_edges
-ALTER TABLE record_entity_edges DROP CONSTRAINT IF EXISTS record_entity_edges_record_id_fkey;
-
--- Drop constraints from record_event_edges
-ALTER TABLE record_event_edges DROP CONSTRAINT IF EXISTS record_event_edges_record_id_fkey;
-
--- ============================================================================
--- DROP DEPRECATED TABLES
--- ============================================================================
-
--- Drop graph edge tables (replaced by source_entity_edges, source_event_edges)
-DROP TABLE IF EXISTS record_entity_edges;
-DROP TABLE IF EXISTS record_event_edges;
-
--- Drop record relationships table
-DROP TABLE IF EXISTS record_relationships;
-
--- Drop main records table
-DROP TABLE IF EXISTS records;
-
--- ============================================================================
--- DROP DEPRECATED COLUMNS
--- ============================================================================
-
--- Drop source_record_id from timeline_events (replaced by source_id)
-ALTER TABLE timeline_events DROP COLUMN IF EXISTS source_record_id;
-
--- Drop record_id from raw_fragments (replaced by source_id)
-ALTER TABLE raw_fragments DROP COLUMN IF EXISTS record_id;
-
--- Drop source_record_id from relationships
-ALTER TABLE relationships DROP COLUMN IF EXISTS source_record_id;
-
--- Drop source_payload_id from observations (replaced by source_id)
-ALTER TABLE observations DROP COLUMN IF EXISTS source_payload_id;
-
--- ============================================================================
--- UPDATE CONSTRAINTS
--- ============================================================================
-
--- Make source_id NOT NULL in timeline_events (now required)
-ALTER TABLE timeline_events ALTER COLUMN source_id SET NOT NULL;
-
--- Make source_id NOT NULL in raw_fragments (now required)
-ALTER TABLE raw_fragments ALTER COLUMN source_id SET NOT NULL;
-
--- ============================================================================
--- CLEANUP COMMENTS
--- ============================================================================
-
-COMMENT ON TABLE timeline_events IS 'Timeline events generated from sources (v0.3.0+)';
-COMMENT ON COLUMN timeline_events.source_id IS 'Source that generated this event';
-
-COMMENT ON TABLE raw_fragments IS 'Unknown fields from interpretation runs (v0.3.0+)';
-COMMENT ON COLUMN raw_fragments.source_id IS 'Source that contained this fragment';
-
--- ============================================================================
--- VERIFICATION QUERIES (Run after migration)
--- ============================================================================
-
--- Verify no orphaned data:
--- 
--- 1. All timeline events have valid source_id:
---    SELECT COUNT(*) FROM timeline_events WHERE source_id IS NULL;
---    -- Should return 0
---
--- 2. All raw_fragments have valid source_id:
---    SELECT COUNT(*) FROM raw_fragments WHERE source_id IS NULL;
---    -- Should return 0
---
--- 3. All observations have valid source_id:
---    SELECT COUNT(*) FROM observations WHERE source_id IS NULL;
---    -- Should return 0
---
--- 4. Verify tables are dropped:
---    SELECT table_name FROM information_schema.tables 
---    WHERE table_name IN ('records', 'record_relationships', 'record_entity_edges', 'record_event_edges');
---    -- Should return 0 rows
-
-
-
-
-

package/docs/releases/v0.3.0/status.md

@@ -1,193 +0,0 @@
-# Release v0.3.0 — Status
-
-**Release**: Reconciliation Release  
-**Release Type**: Internal / Not Marketed  
-**Deployment**: Documentation Only  
-**Status**: `completed`  
-**Date**: 2025-02-05
-
----
-
-## Overview
-
-This is a **documentation-only reconciliation release** that establishes an accurate baseline between documented releases and actual codebase state. No new code was written for this release; instead, it documents what has already been implemented during late 2024 and early 2025.
-
----
-
-## Status Summary
-
-| Category | Status | Notes |
-|----------|--------|-------|
-| **Documentation** | ✅ Complete | Reconciliation release plan created |
-| **Baseline Established** | ✅ Complete | Current state documented |
-| **Archives Created** | ✅ Complete | Aspirational releases archived |
-| **README Updated** | ⏳ Pending | Need to update README.md |
-
----
-
-## What Was Done
-
-### 1. Documentation Created ✅
-
-- [x] Release plan documenting actual current state
-- [x] Manifest describing reconciliation purpose
-- [x] Status file (this document)
-
-### 2. Releases Archived ✅
-
-Moved aspirational releases to `docs/releases/archived/aspirational/`:
-- [x] v0.4.0 (Intelligence + Housekeeping)
-- [x] v0.5.0 (Planning)
-- [x] v0.9.0 (Planning)
-- [x] v1.0.0 (MVP)
-- [x] v2.0.0 (Planning)
-- [x] v2.1.0 (Planning)
-- [x] v2.2.0 (HNSW Vector Search)
-
-### 3. Current State Documented ✅
-
-Documented what's actually implemented:
-- [x] Schema registry system
-- [x] Sources-first architecture
-- [x] Observations architecture
-- [x] Entity system
-- [x] Auto-enhancement
-- [x] MCP integration
-- [x] Frontend
-- [x] CLI
-- [x] Infrastructure
-
----
-
-## Technical Debt Acknowledged
-
-This release acknowledges the following technical debt:
-
-1. **Feature Unit Tracking** - Not all work documented in Feature Units
-2. **Test Coverage** - Integration tests may be outdated
-3. **Documentation Sync** - Some docs may not reflect implementation
-4. **Migration State** - Some migrations may be pending
-5. **Uncommitted Changes** - 262 files need review and cleanup
-
----
-
-## Immediate Next Steps
-
-### 1. Review Uncommitted Changes ⏳
-
-**Status**: Pending  
-**Action Required**: Review 262 uncommitted files
-
-```bash
-git status --short
-```
-
-**Decision Needed**: For each change:
-- Commit if ready for production
-- Discard if experimental/temporary
-- Stash if work-in-progress
-
-### 2. Apply Pending Migrations ⏳
-
-**Status**: Unknown  
-**Action Required**: Verify migration state
-
-```bash
-npm run migrate
-```
-
-**Environments to check**:
-- Development database
-- Production database
-
-### 3. Audit Test Suite ⏳
-
-**Status**: Pending  
-**Action Required**: Run full test suite
-
-```bash
-npm test
-npm run test:integration
-```
-
-**Expected**: Identify broken/outdated tests
-
-### 4. Update README ⏳
-
-**Status**: Pending  
-**Action Required**: Update README.md to reflect:
-- Current accurate release status (v0.3.0 completed)
-- Archived aspirational releases
-- Actual implemented features
-
----
-
-## Near-Term Actions
-
-### 1. Plan v0.4.0 Realistically
-
-Based on v0.3.0 baseline:
-- Define clear scope
-- Follow Feature Unit workflow
-- Set realistic acceptance criteria
-
-### 2. Establish Release Discipline
-
-Going forward:
-- Document all changes in release plans
-- Follow checkpoint process
-- Keep README updated
-- Commit regularly (avoid large uncommitted changesets)
-
-### 3. Address Technical Debt
-
-Prioritize:
-- Test coverage improvements
-- Documentation accuracy
-- Code cleanup
-- Migration verification
-
----
-
-## Success Criteria
-
-| Criterion | Status | Notes |
-|-----------|--------|-------|
-| Reconciliation release plan created | ✅ Complete | Documented actual state |
-| Aspirational releases archived | ✅ Complete | Moved to archived/ |
-| Baseline established | ✅ Complete | v0.3.0 represents current state |
-| Technical debt acknowledged | ✅ Complete | Listed in release plan |
-| Path forward defined | ✅ Complete | Immediate and near-term steps |
-
----
-
-## Release Timeline
-
-- **2025-02-05**: Reconciliation release created
-- **2025-02-05**: Aspirational releases archived
-- **2025-02-05**: Status marked as `completed`
-
----
-
-## Notes
-
-- This is a **documentation-only release** - no code changes
-- Marks the end of "untracked development" period
-- Establishes clean baseline for future release planning
-- All future releases should follow proper Feature Unit workflow
-
----
-
-## Related Documentation
-
-- [`release_plan.md`](./release_plan.md) — Full reconciliation release plan
-- [`manifest.yaml`](./manifest.yaml) — Release metadata
-- `docs/releases/archived/aspirational/` — Archived aspirational releases
-- `README.md` — Project overview (needs update)
-
----
-
-**Status Legend**:
-- `completed`: Release is complete (documentation-only)
-- ✅ Complete: Task finished
-- ⏳ Pending: Task needs to be done

package/docs/reports/AUTO_ENHANCEMENT_TEST_RESULTS.md

@@ -1,51 +0,0 @@
-# Auto-Enhancement Test Results
-
-**Date**: 2026-01-14  
-**Status**: ⚠️ **TESTING IN PROGRESS**
-
-## Test Setup
-
-1. ✅ **Imported sample file**: `samples/task.parquet`
-2. ✅ **Entity type**: `task` (matches existing schema)
-3. ✅ **Unknown fields detected**: 31 fields stored in `raw_fragments`
-4. ⏳ **Auto-enhancement processor**: Waiting for processing
-
-## Import Results
-
-```json
-{
-  "source_id": "232dc440-3f12-410f-a815-1fb5070796e4",
-  "entities": [
-    {
-      "entity_id": "ent_976638491634c15c308f35df",
-      "entity_type": "task",
-      "observation_id": "93fe3520-3ace-4a43-b72a-fa4e87f03f22"
-    },
-    // ... 5 more entities
-  ],
-  "unknown_fields_count": 31
-}
-```
-
-## Expected Behavior
-
-With 6 observations (rows) and 31 unknown fields:
-- ✅ **Row diversity**: 6 observations >= 2 → should pass diversity check
-- ✅ **Frequency**: Each field appears in multiple rows → should pass threshold
-- ✅ **Auto-enhancement**: Should trigger for eligible fields
-
-## Current Status
-
-After waiting 35 seconds for auto-enhancement processor:
-- ❌ **No schema recommendations** found
-- ⏳ **Need to check**: 
-  - Are queue items being created?
-  - Are fragments being queried correctly?
-  - Is eligibility check passing?
-
-## Next Steps
-
-1. Check `auto_enhancement_queue` table for pending items
-2. Check `raw_fragments` table to verify fragments were stored
-3. Verify `fragment_type` matches `entity_type` in queries
-4. Check eligibility criteria (diversity, frequency, confidence)

package/docs/reports/AUTO_ENHANCEMENT_TEST_SUMMARY.md

@@ -1,64 +0,0 @@
-# Auto-Enhancement Test Summary
-
-**Date**: 2026-01-14  
-**Status**: ✅ **BUG FIXED - RESTART REQUIRED**
-
-## Test Results
-
-### 1. Sample File Import ✅
-
-- **File**: `samples/task.parquet`
-- **Entity type**: `task` (matches existing schema)
-- **Result**: ✅ Import successful
-- **Unknown fields**: 31 fields stored in `raw_fragments`
-- **Observations**: 6 observations created
-
-### 2. Auto-Enhancement Status ⚠️
-
-- **Status**: No schemas enhanced yet
-- **Reason**: Bug found and fixed, but server needs restart
-
-## Bug Found and Fixed
-
-### Problem
-
-`calculateFieldConfidence` queried `entity_type` column, but structured data (parquet) stores entity type in `fragment_type` column.
-
-### Fix Applied
-
-Updated queries in `src/services/schema_recommendation.ts`:
-- `calculateFieldConfidence`: Now checks both `fragment_type` and `entity_type`
-- `analyzeRawFragments`: Now includes `fragment_type` in query
-
-### Code Changes
-
-**File**: `src/services/schema_recommendation.ts`
-
-1. **Line 333-338**: Fixed `calculateFieldConfidence` query
-2. **Line 399-404**: Fixed `analyzeRawFragments` query
-
-## Next Steps
-
-1. **Restart MCP server** to load the fix
-2. **Re-import sample file** (or wait for next auto-enhancement cycle)
-3. **Verify auto-enhancement works**:
-   - Check `auto_enhancement_queue` for processed items
-   - Check `schema_recommendations` for auto-applied enhancements
-   - Verify new fields appear in schema
-
-## Expected Behavior After Restart
-
-With 6 observations and 31 unknown fields:
-- ✅ **Row diversity**: 6 observations >= 2 → should pass
-- ✅ **Frequency**: Fields appear in multiple rows → should pass threshold
-- ✅ **Confidence**: Should calculate correctly (bug fixed)
-- ✅ **Auto-enhancement**: Should trigger for eligible fields
-
-## Files Modified
-
-- `src/services/schema_recommendation.ts` - Fixed query mismatch
-
-## Documentation
-
-- `docs/reports/AUTO_ENHANCEMENT_BUG_FIX.md` - Detailed bug analysis
-- `docs/reports/AUTO_ENHANCEMENT_TEST_RESULTS.md` - Test results

package/docs/reports/BIGINT_SERIALIZATION_DEBUG.md

@@ -1,75 +0,0 @@
-# BigInt Serialization Error - Debug Analysis
-
-**Date**: 2026-01-14  
-**Status**: ⚠️ **ONGOING** - Error persists after multiple fixes
-
-## Problem
-
-When storing tasks from parquet file via MCP `store` action:
-```
-Error: Failed to process parquet file: Do not know how to serialize a BigInt
-```
-
-## Investigation Results
-
-### ✅ What Works
-1. **Direct function call**: `readParquetFile()` works when called directly via Node.js
-   - Test: `node -e "readParquetFile('/path/to/tasks.parquet')"` → Success: 16065 rows
-2. **Unit tests**: All 10 parquet_reader tests pass
-3. **BigInt conversion**: `convertBigIntValues()` function works correctly
-4. **Code compilation**: TypeScript builds successfully
-
-### ❌ What Doesn't Work
-1. **MCP store action**: Fails with BigInt serialization error
-2. **Error occurs**: During `readParquetFile()` call within MCP handler
-3. **Error message**: "Do not know how to serialize a BigInt" (standard JavaScript error)
-
-## Root Cause Hypothesis
-
-The error "Do not know how to serialize a BigInt" is thrown by JavaScript's `JSON.stringify()` when it encounters a BigInt value without a replacer function. Since:
-
-1. Direct function call works ✅
-2. Unit tests pass ✅
-3. BigInt conversion is implemented ✅
-
-The error is likely happening in one of these scenarios:
-
-### Hypothesis 1: MCP SDK Error Serialization
-The MCP SDK might be trying to serialize the error object, and the error object contains BigInt values (perhaps from the parquet result or entities array that was partially read).
-
-### Hypothesis 2: Error Object Contains BigInt
-When an error occurs in `readParquetFile`, the error object might contain:
-- Partially read entities with BigInt values
-- Metadata with BigInt values
-- Stack trace or other properties with BigInt values
-
-### Hypothesis 3: Console.log Serialization
-The `console.log()` statements might be trying to serialize objects containing BigInt values (though this is unlikely as console.log typically handles BigInt).
-
-### Hypothesis 4: Progress Callback
-The `options?.onProgress` callback might be trying to serialize data containing BigInt values.
-
-## Fixes Applied
-
-1. ✅ Added `convertBigIntValues()` function with recursive conversion
-2. ✅ Added BigInt replacers to all `JSON.stringify()` calls
-3. ✅ Added deep conversion pass before returning results
-4. ✅ Added error message sanitization
-5. ✅ Added try-catch around conversion with fallback
-6. ✅ Added final serialization check before returning
-7. ✅ Fixed TypeScript compilation errors
-
-## Next Steps
-
-1. **Check MCP server logs**: Enable logging to see where exactly the error occurs
-2. **Add more defensive error handling**: Wrap entire operation in try-catch that sanitizes all BigInt values
-3. **Test with smaller file**: Try with a smaller parquet file to isolate the issue
-4. **Check parquet library**: The `@dsnp/parquetjs` library might be serializing something internally
-
-## Recommendation
-
-Since the direct function call works, the issue is likely in the MCP error handling or serialization. Consider:
-
-1. **Restart MCP server** after rebuild (may have cached old code)
-2. **Add logging** to identify exact location of error
-3. **Test with minimal example** to isolate the issue