neotoma 0.3.0 → 0.3.2

package/scripts/check_raw_fragments.ts

@@ -1,46 +0,0 @@
-import { db } from "../src/db.js";
-
-async function check() {
-  // Check raw_fragments for task
-  const { data: fragments, error } = await db
-    .from('raw_fragments')
-    .select('entity_type, fragment_key, source_id, interpretation_id, user_id')
-    .eq('entity_type', 'task')
-    .limit(10);
-
-  console.log('Sample raw_fragments:');
-  if (fragments) {
-    fragments.forEach(f => {
-      console.log(`  ${f.fragment_key}: entity_type=${f.entity_type}, source_id=${f.source_id}, user_id=${f.user_id}`);
-    });
-  } else {
-    console.log('  No fragments found');
-  }
-  console.log('');
-  console.log('Error:', error);
-  
-  // Count total
-  const { count } = await db
-    .from('raw_fragments')
-    .select('*', { count: 'exact', head: true })
-    .eq('entity_type', 'task');
-  
-  console.log('Total fragments for task:', count);
-  
-  // Check for specific field
-  const { data: taskIdFrags } = await db
-    .from('raw_fragments')
-    .select('*')
-    .eq('entity_type', 'task')
-    .eq('fragment_key', 'task_id')
-    .limit(3);
-  
-  console.log('\nSample task_id fragments:');
-  if (taskIdFrags) {
-    taskIdFrags.forEach(f => {
-      console.log(`  entity_type: ${f.entity_type}, source_id: ${f.source_id}, user_id: ${f.user_id}`);
-    });
-  }
-}
-
-check().catch(console.error);

package/scripts/check_remaining_eligibility.ts

@@ -1,46 +0,0 @@
-import { SchemaRecommendationService } from '../src/services/schema_recommendation.js';
-
-const service = new SchemaRecommendationService();
-
-async function checkEligible() {
-  const fields = [
-    'project_ids',
-    'project_names', 
-    'section_ids',
-    'section_names',
-    'followers_gids',
-    'asana_source_gid',
-    'description_html',
-    'my_tasks_section_ids',
-    'my_tasks_section_names',
-    'outcome_names',
-    'execution_plan_path',
-    'description_html_remote',
-  ];
-
-  console.log('Checking eligibility for remaining fields:\n');
-
-  for (const field of fields) {
-    const confidence = await service.calculateFieldConfidence({
-      fragment_key: field,
-      entity_type: 'task',
-      user_id: '00000000-0000-0000-0000-000000000000',
-    });
-
-    const eligibility = await service.checkAutoEnhancementEligibility({
-      entity_type: 'task',
-      fragment_key: field,
-      user_id: '00000000-0000-0000-0000-000000000000',
-    });
-
-    const status = eligibility.eligible ? '✅ ELIGIBLE' : '❌ NOT ELIGIBLE';
-    console.log(`${field}: ${status}`);
-    console.log(`  Confidence: ${confidence.confidence.toFixed(3)} (threshold: 0.85)`);
-    if (!eligibility.eligible) {
-      console.log(`  Reason: ${eligibility.reasoning}`);
-    }
-    console.log('');
-  }
-}
-
-checkEligible().catch(console.error);

package/scripts/check_remaining_fields.ts

@@ -1,58 +0,0 @@
-import { db } from '../src/db.js';
-
-async function checkRemaining() {
-  // Get all unique fragment_keys for task
-  const { data: fragments } = await db
-    .from('raw_fragments')
-    .select('fragment_key, frequency_count')
-    .eq('entity_type', 'task')
-    .or('user_id.is.null,user_id.eq.00000000-0000-0000-0000-000000000000');
-
-  if (!fragments) {
-    console.log('No fragments found');
-    return;
-  }
-
-  // Group by fragment_key
-  const fieldMap = new Map();
-  for (const frag of fragments) {
-    const key = frag.fragment_key;
-    const freq = frag.frequency_count || 1;
-    fieldMap.set(key, (fieldMap.get(key) || 0) + freq);
-  }
-
-  console.log('All fields in raw_fragments:');
-  console.log('Total unique fields:', fieldMap.size);
-  console.log('');
-  
-  // Schema fields (from the list we saw)
-  const schemaFields = new Set([
-    'tags', 'notes', 'title', 'domain', 'status', 'task_id', 'urgency', 'assignee',
-    'due_date', 'priority', 'created_at', 'project_id', 'recurrence', 'start_date',
-    'updated_at', 'description', 'import_date', 'outcome_ids', 'assignee_gid',
-    'created_date', 'updated_date', 'assignee_name', 'permalink_url', 'completed_date',
-    'follower_names', 'parent_task_id', 'schema_version', 'asana_workspace', 'import_source_file'
-  ]);
-
-  const remaining = Array.from(fieldMap.entries())
-    .filter(([key]) => !schemaFields.has(key))
-    .sort((a, b) => b[1] - a[1]);
-
-  const inSchema = Array.from(fieldMap.entries())
-    .filter(([key]) => schemaFields.has(key))
-    .sort((a, b) => b[1] - a[1]);
-
-  console.log('Fields IN schema (auto-enhanced):');
-  console.log(`Total: ${inSchema.length} fields`);
-  inSchema.forEach(([key, freq]) => {
-    console.log(`  ✅ ${key}: frequency ${freq}`);
-  });
-
-  console.log('\nFields NOT in schema (still in raw_fragments):');
-  console.log(`Total: ${remaining.length} fields`);
-  remaining.forEach(([key, freq]) => {
-    console.log(`  ⚠️  ${key}: frequency ${freq}`);
-  });
-}
-
-checkRemaining().catch(console.error);

package/scripts/check_sendgrid_credits.sh

@@ -1,122 +0,0 @@
-#!/bin/bash
-# Check SendGrid account credits and provide upgrade instructions
-
-set -e
-
-# Load .env
-if [ -f .env ]; then
-  SENDGRID_API_KEY=$(grep "^SENDGRID_API_KEY=" .env | cut -d '=' -f2- | sed 's/^"//;s/"$//' | tr -d '\n' 2>/dev/null || echo "")
-fi
-
-echo "📋 SendGrid Account Credits & Billing"
-echo ""
-
-if [ -z "$SENDGRID_API_KEY" ]; then
-  echo "⚠️  SENDGRID_API_KEY not found in .env"
-  echo "   Run: /sync_env_from_1password"
-  echo "   Or: npm run sync:env"
-  exit 1
-fi
-
-# Check user profile
-echo "📋 Account Information:"
-PROFILE=$(curl -s -X GET "https://api.sendgrid.com/v3/user/profile" \
-  -H "Authorization: Bearer ${SENDGRID_API_KEY}" \
-  -H "Content-Type: application/json" 2>/dev/null || echo '{}')
-
-EMAIL=$(echo "$PROFILE" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('email', 'unknown'))" 2>/dev/null || echo "unknown")
-FIRST_NAME=$(echo "$PROFILE" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('first_name', 'unknown'))" 2>/dev/null || echo "unknown")
-LAST_NAME=$(echo "$PROFILE" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('last_name', 'unknown'))" 2>/dev/null || echo "unknown")
-
-echo "  Email: ${EMAIL}"
-echo "  Name: ${FIRST_NAME} ${LAST_NAME}"
-echo ""
-
-# Check recent email activity to estimate usage
-echo "📋 Recent Email Activity (last 7 days):"
-START_DATE=$(date -u -v-7d +%Y-%m-%d 2>/dev/null || date -u -d "7 days ago" +%Y-%m-%d 2>/dev/null || echo "")
-if [ -n "$START_DATE" ]; then
-  STATS=$(curl -s -X GET "https://api.sendgrid.com/v3/stats?start_date=${START_DATE}&aggregated_by=day" \
-    -H "Authorization: Bearer ${SENDGRID_API_KEY}" \
-    -H "Content-Type: application/json" 2>/dev/null || echo '[]')
-  
-  TOTAL_SENT=$(echo "$STATS" | python3 -c "
-import sys, json
-try:
-    data = json.load(sys.stdin)
-    total = 0
-    if isinstance(data, list):
-        for day in data:
-            stats = day.get('stats', [])
-            for stat in stats:
-                metrics = stat.get('metrics', {})
-                total += metrics.get('delivered', 0)
-    print(total)
-except:
-    print('unknown')
-" 2>/dev/null || echo "unknown")
-  
-  echo "  Total emails sent (last 7 days): ${TOTAL_SENT}"
-  echo "  Average per day: $((TOTAL_SENT / 7))"
-else
-  echo "  ⚠️  Could not calculate date range"
-fi
-echo ""
-
-# Instructions
-echo "📋 How to Check Your Current Plan & Credits:"
-echo ""
-echo "1. Go to SendGrid Dashboard:"
-echo "   https://app.sendgrid.com/"
-echo ""
-echo "2. Navigate to: Settings → Billing"
-echo ""
-echo "3. Check your:"
-echo "   - Current plan (Free, Essentials, Pro, etc.)"
-echo "   - Credit balance"
-echo "   - Monthly email limit"
-echo ""
-
-echo "📋 How to Add More Credits:"
-echo ""
-echo "Option A: Upgrade Your Plan (Recommended)"
-echo "  Steps:"
-echo "  1. Go to: https://app.sendgrid.com/settings/billing"
-echo "  2. Click 'Change Plan' or 'Upgrade'"
-echo "  3. Select your desired plan:"
-echo "     - Free: 100 emails/day (no cost)"
-echo "     - Essentials: \$19.95/month (40,000 emails/month)"
-echo "     - Pro: \$89.95/month (100,000 emails/month)"
-echo "     - Higher tiers available for more volume"
-echo "  4. Add payment information"
-echo "  5. Submit"
-echo ""
-echo "  ⚠️  Note: Plan upgrades take effect immediately"
-echo ""
-
-echo "Option B: Purchase Additional Credits (Paid Plans Only)"
-echo "  Steps:"
-echo "  1. Go to: https://app.sendgrid.com/settings/billing"
-echo "  2. Click 'Purchase Credits' (if available on your plan)"
-echo "  3. Select credit package"
-echo "  4. Complete purchase"
-echo ""
-
-echo "Option C: Wait for Credit Reset"
-echo "  - Free tier: Resets daily at midnight UTC"
-echo "  - Paid plans: Resets monthly (on billing cycle)"
-echo ""
-
-echo "⚠️  About the '451 Maximum credits exceeded' Error:"
-echo "  - This means you've reached your SendGrid account's email sending limit"
-echo "  - SendGrid uses 1 credit per email sent"
-echo "  - Free tier: 100 credits/day"
-echo "  - Paid plans: Based on your monthly limit"
-echo "  - This is NOT a database configuration issue"
-echo ""
-
-echo "📋 Quick Links:"
-echo "  - Dashboard: https://app.sendgrid.com/"
-echo "  - Billing: https://app.sendgrid.com/settings/billing"
-echo "  - Upgrade Plan: https://app.sendgrid.com/settings/billing/plan"
-echo ""

package/scripts/check_sendgrid_email_logs.sh

@@ -1,150 +0,0 @@
-#!/bin/bash
-# Check SendGrid email logs for sent emails
-# Shows recent email activity, delivery status, and detailed logs
-
-set -e
-
-# Load .env
-if [ -f .env ]; then
-  SENDGRID_API_KEY=$(grep "^SENDGRID_API_KEY=" .env | cut -d '=' -f2- | sed 's/^"//;s/"$//' | tr -d '\n' 2>/dev/null || echo "")
-  SENDGRID_SENDER_EMAIL=$(grep "^SENDGRID_SENDER_EMAIL=" .env | cut -d '=' -f2- | sed 's/^"//;s/"$//' | tr -d '\n' 2>/dev/null || echo "")
-fi
-
-echo "📋 SendGrid Email Logs"
-echo ""
-
-if [ -z "$SENDGRID_API_KEY" ]; then
-  echo "❌ SENDGRID_API_KEY not found in .env"
-  echo "   Run: /sync_env_from_1password"
-  echo "   Or: npm run sync:env"
-  exit 1
-fi
-
-# Check email activity (last 7 days)
-echo "=== Email Activity Stats (Last 7 Days) ==="
-echo ""
-
-python3 << 'PYTHON_EOF'
-import urllib.request
-import json
-import os
-import datetime
-
-# Load .env
-env_vars = {}
-if os.path.exists('.env'):
-    with open('.env', 'r') as f:
-        for line in f:
-            line = line.strip()
-            if line and not line.startswith('#') and '=' in line:
-                key, value = line.split('=', 1)
-                value = value.strip('"').strip("'").strip()
-                env_vars[key] = value
-
-api_key = env_vars.get('SENDGRID_API_KEY', '')
-
-if not api_key:
-    print("❌ SENDGRID_API_KEY not found")
-    exit(1)
-
-# Get stats for last 7 days
-end_date = datetime.datetime.now(datetime.timezone.utc).date()
-start_date = end_date - datetime.timedelta(days=7)
-
-url = f"https://api.sendgrid.com/v3/stats?start_date={start_date}&end_date={end_date}&aggregated_by=day"
-req = urllib.request.Request(url)
-req.add_header('Authorization', f'Bearer {api_key}')
-req.add_header('Content-Type', 'application/json')
-
-try:
-    with urllib.request.urlopen(req, timeout=10) as response:
-        data = json.loads(response.read())
-        
-        if isinstance(data, list) and len(data) > 0:
-            total_delivered = 0
-            total_bounces = 0
-            total_drops = 0
-            total_opens = 0
-            total_clicks = 0
-            total_requests = 0
-            
-            print("Daily Activity:")
-            for day_data in data:
-                date = day_data.get('date', 'N/A')
-                stats = day_data.get('stats', [])
-                for stat in stats:
-                    metrics = stat.get('metrics', {})
-                    delivered = metrics.get('delivered', 0)
-                    bounces = metrics.get('bounces', 0)
-                    drops = metrics.get('drops', 0)
-                    opens = metrics.get('opens', 0)
-                    clicks = metrics.get('clicks', 0)
-                    requests = metrics.get('requests', 0)
-                    
-                    if requests > 0 or delivered > 0 or bounces > 0 or drops > 0:
-                        print(f"  {date}:")
-                        print(f"    Requests: {requests}")
-                        print(f"    Delivered: {delivered}")
-                        print(f"    Bounces: {bounces}")
-                        print(f"    Drops: {drops}")
-                        print(f"    Opens: {opens}")
-                        print(f"    Clicks: {clicks}")
-                        print("")
-                    
-                    total_delivered += delivered
-                    total_bounces += bounces
-                    total_drops += drops
-                    total_opens += opens
-                    total_clicks += clicks
-                    total_requests += requests
-            
-            print("Total (Last 7 days):")
-            print(f"  Requests: {total_requests}")
-            print(f"  Delivered: {total_delivered}")
-            print(f"  Bounces: {total_bounces}")
-            print(f"  Drops: {total_drops}")
-            print(f"  Opens: {total_opens}")
-            print(f"  Clicks: {total_clicks}")
-            print("")
-            
-            if total_requests == 0:
-                print("⚠️  No email requests found in the last 7 days")
-                print("   This could mean:")
-                print("   - No emails were sent via SendGrid")
-                print("   - Emails failed before reaching SendGrid (check application logs)")
-                print("   - API access is limited on your plan")
-        else:
-            print("  No email activity data available")
-            print("  (This could mean no emails were sent, or API access is limited)")
-except urllib.error.HTTPError as e:
-    error_body = e.read().decode('utf-8')
-    print(f"  ⚠️  API Error ({e.code}): {error_body}")
-    if e.code == 403:
-        print("  This may require a paid SendGrid plan for API access")
-except Exception as e:
-    print(f"  ⚠️  Error: {e}")
-
-print("")
-print("=== Individual Email Messages ===")
-print("")
-print("  ⚠️  Individual message logs require SendGrid Dashboard access")
-print("  The messages API endpoint requires additional authorization")
-print("")
-print("  Use the SendGrid Dashboard for detailed message logs:")
-print("  https://app.sendgrid.com/activity")
-print("")
-
-print("=== SendGrid Dashboard Links ===")
-print("")
-print("For detailed email logs and individual message tracking:")
-print("  - Email Activity: https://app.sendgrid.com/activity")
-print("    (View individual emails, delivery status, opens, clicks, bounces)")
-print("")
-print("  - Stats Overview: https://app.sendgrid.com/statistics")
-print("    (View aggregated statistics and trends)")
-print("")
-print("  - Billing/Credits: https://app.sendgrid.com/settings/billing")
-print("    (Check your plan, credits, and upgrade options)")
-print("")
-PYTHON_EOF
-

package/scripts/check_snapshot_schema.ts

@@ -1,18 +0,0 @@
-import { db } from '../src/db.js';
-
-async function check() {
-  const { data } = await db
-    .from('entity_snapshots')
-    .select('entity_id, schema_version, snapshot')
-    .eq('entity_id', 'ent_c5042fb1197f4ce6677084ac')
-    .single();
-  
-  console.log('Snapshot metadata schema_version (column):', data?.schema_version);
-  console.log('Snapshot data schema_version field (in JSONB):', data?.snapshot?.schema_version);
-  console.log('');
-  console.log('These are different:');
-  console.log('  - Column schema_version = schema version used for computation (should be 22.0)');
-  console.log('  - Field schema_version = data field from observation (is 1.0)');
-}
-
-check().catch(console.error);

package/scripts/check_sources.ts

@@ -1,34 +0,0 @@
-import { db } from '../src/db.js';
-
-const sourceIds = [
-  'b585e7e6-4a89-42c8-a5fe-beba617c936e',
-  '619590fa-9616-4816-85e0-744639adb429',
-  '93ed92f2-2e09-4c6f-ba27-4c786a516de2'
-];
-
-async function checkSources() {
-  for (const sourceId of sourceIds) {
-    const { data, error } = await db
-      .from('sources')
-      .select('id, content_hash, mime_type, file_size, original_filename, created_at, storage_url')
-      .eq('id', sourceId)
-      .single();
-    
-    if (error) {
-      console.log(`Source ${sourceId}: Error - ${error.message}`);
-    } else if (data) {
-      console.log(`Source ${sourceId}:`);
-      console.log(`  Filename: ${data.original_filename || 'N/A'}`);
-      console.log(`  MIME Type: ${data.mime_type || 'N/A'}`);
-      console.log(`  Size: ${data.file_size ? (data.file_size / 1024).toFixed(2) + ' KB' : 'N/A'}`);
-      console.log(`  Created: ${data.created_at || 'N/A'}`);
-      console.log(`  Content Hash: ${data.content_hash?.substring(0, 16) || 'N/A'}...`);
-      console.log(`  Storage URL: ${data.storage_url || 'N/A'}`);
-      console.log('');
-    } else {
-      console.log(`Source ${sourceId}: Not found`);
-    }
-  }
-}
-
-checkSources().catch(console.error);

package/scripts/check_stale_snapshots.ts

@@ -1,80 +0,0 @@
-import { db } from '../src/db.js';
-
-/**
- * Check for entities with stale snapshots (observation_count: 0 but observations exist)
- */
-async function checkStaleSnapshots() {
-  console.log('Checking for entities with stale snapshots...\n');
-
-  // Get all entity snapshots with observation_count = 0
-  const { data: staleSnapshots, error: snapshotError } = await db
-    .from('entity_snapshots')
-    .select('entity_id, entity_type, observation_count, computed_at')
-    .eq('observation_count', 0)
-    .order('computed_at', { ascending: false });
-
-  if (snapshotError) {
-    console.error('Error fetching snapshots:', snapshotError);
-    return;
-  }
-
-  if (!staleSnapshots || staleSnapshots.length === 0) {
-    console.log('✅ No stale snapshots found (all snapshots have observation_count > 0)');
-    return;
-  }
-
-  console.log(`Found ${staleSnapshots.length} snapshots with observation_count = 0\n`);
-
-  let staleCount = 0;
-  const staleEntities: Array<{
-    entity_id: string;
-    entity_type: string;
-    observation_count_snapshot: number;
-    observation_count_actual: number;
-    computed_at: string;
-  }> = [];
-
-  for (const snapshot of staleSnapshots) {
-    // Check if observations actually exist for this entity
-    const { data: observations, error: obsError } = await db
-      .from('observations')
-      .select('id')
-      .eq('entity_id', snapshot.entity_id);
-
-    if (obsError) {
-      console.error(`Error checking observations for ${snapshot.entity_id}:`, obsError);
-      continue;
-    }
-
-    const actualObsCount = observations?.length || 0;
-
-    if (actualObsCount > 0) {
-      // Stale snapshot detected
-      staleCount++;
-      staleEntities.push({
-        entity_id: snapshot.entity_id,
-        entity_type: snapshot.entity_type,
-        observation_count_snapshot: snapshot.observation_count,
-        observation_count_actual: actualObsCount,
-        computed_at: snapshot.computed_at,
-      });
-
-      console.log(`❌ Stale: ${snapshot.entity_id} (${snapshot.entity_type})`);
-      console.log(`   Snapshot observation_count: ${snapshot.observation_count}`);
-      console.log(`   Actual observations: ${actualObsCount}`);
-      console.log(`   Computed at: ${snapshot.computed_at}\n`);
-    }
-  }
-
-  if (staleCount === 0) {
-    console.log('✅ No stale snapshots found (all entities with observation_count=0 have no observations)');
-  } else {
-    console.log(`\n⚠️  Found ${staleCount} stale snapshots that need recomputation\n`);
-    console.log('Stale entities:');
-    console.log(JSON.stringify(staleEntities, null, 2));
-  }
-
-  return staleEntities;
-}
-
-checkStaleSnapshots().catch(console.error);

package/scripts/cleanup-auto-test-schemas.ts

@@ -1,80 +0,0 @@
-/**
- * Cleanup Auto Test Schemas
- *
- * Removes all schemas with entity_type starting with "auto_test_" from schema_registry.
- * These are test artifacts from integration tests that should have been cleaned up.
- * Uses local SQLite database.
- *
- * Usage:
- *   npx tsx scripts/cleanup-auto-test-schemas.ts [--confirm]
- *
- * Options:
- *   --confirm    Skip confirmation prompt
- */
-
-import { db } from "../src/db.js";
-import readline from "readline";
-
-async function cleanupAutoTestSchemas(confirm = false) {
-  // Find all auto_test schemas
-  const { data: schemas, error: fetchError } = await db
-    .from("schema_registry")
-    .select("id, entity_type, schema_version, scope, user_id")
-    .ilike("entity_type", "auto_test_%");
-
-  if (fetchError) {
-    console.error("Error fetching schemas:", fetchError);
-    process.exit(1);
-  }
-
-  if (!schemas || schemas.length === 0) {
-    console.log("✅ No auto_test schemas found. Database is clean.");
-    return;
-  }
-
-  console.log(`\nFound ${schemas.length} auto_test schema(s) to delete:`);
-  schemas.forEach((schema) => {
-    console.log(`  - ${schema.entity_type} (v${schema.schema_version}, ${schema.scope || "global"})`);
-  });
-
-  if (!confirm) {
-    const rl = readline.createInterface({
-      input: process.stdin,
-      output: process.stdout,
-    });
-
-    const answer = await new Promise<string>((resolve) => {
-      rl.question("\nDelete these schemas? (yes/no): ", resolve);
-    });
-
-    rl.close();
-
-    if (answer.toLowerCase() !== "yes" && answer.toLowerCase() !== "y") {
-      console.log("Cancelled.");
-      return;
-    }
-  }
-
-  // Delete all auto_test schemas
-  const schemaIds = schemas.map((s) => s.id);
-  const { error: deleteError } = await db
-    .from("schema_registry")
-    .delete()
-    .in("id", schemaIds);
-
-  if (deleteError) {
-    console.error("Error deleting schemas:", deleteError);
-    process.exit(1);
-  }
-
-  console.log(`\n✅ Successfully deleted ${schemas.length} auto_test schema(s).`);
-}
-
-// Parse command line arguments
-const args = process.argv.slice(2);
-const confirm = args.includes("--confirm");
-
-cleanupAutoTestSchemas(confirm).catch((error) => {
-  console.error("Unexpected error:", error);
-  process.exit(1);
-});