@stackmemoryai/stackmemory 0.3.17 → 0.3.19

package/scripts/sync-and-clean-tasks.js

@@ -0,0 +1,258 @@
+#!/usr/bin/env node
+
+/**
+ * Sync Linear tasks with local storage and clean up duplicates
+ */
+
+import 'dotenv/config';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const API_KEY = process.env.LINEAR_OAUTH_TOKEN || process.env.LINEAR_API_KEY;
+if (!API_KEY) {
+  console.error('❌ LINEAR_OAUTH_TOKEN or LINEAR_API_KEY environment variable not set');
+  console.log('Please set LINEAR_OAUTH_TOKEN or LINEAR_API_KEY in your .env file or export it in your shell');
+  process.exit(1);
+}
+
+async function fetchAllLinearTasks() {
+  console.log('📥 Fetching all tasks from Linear...');
+  
+  const query = `
+    query GetAllIssues($after: String) {
+      issues(first: 100, after: $after, includeArchived: false) {
+        nodes {
+          id
+          identifier
+          title
+          description
+          state {
+            id
+            name
+            type
+          }
+          priority
+          estimate
+          createdAt
+          updatedAt
+        }
+        pageInfo {
+          hasNextPage
+          endCursor
+        }
+      }
+    }
+  `;
+  
+  let allIssues = [];
+  let hasNextPage = true;
+  let cursor = null;
+  
+  while (hasNextPage) {
+    const response = await fetch('https://api.linear.app/graphql', {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${API_KEY}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        query,
+        variables: { after: cursor }
+      })
+    });
+    
+    const result = await response.json();
+    
+    if (result.errors) {
+      throw new Error(result.errors[0].message);
+    }
+    
+    allIssues = allIssues.concat(result.data.issues.nodes);
+    hasNextPage = result.data.issues.pageInfo.hasNextPage;
+    cursor = result.data.issues.pageInfo.endCursor;
+  }
+  
+  console.log(`  Found ${allIssues.length} Linear tasks\n`);
+  return allIssues;
+}
+
+function readLocalTasks() {
+  const tasksFile = path.join(process.cwd(), '.stackmemory', 'tasks.jsonl');
+  
+  if (!fs.existsSync(tasksFile)) {
+    return [];
+  }
+  
+  const lines = fs.readFileSync(tasksFile, 'utf8').split('\n').filter(Boolean);
+  return lines.map(line => {
+    try {
+      return JSON.parse(line);
+    } catch {
+      return null;
+    }
+  }).filter(Boolean);
+}
+
+function writeLocalTasks(tasks) {
+  const tasksFile = path.join(process.cwd(), '.stackmemory', 'tasks.jsonl');
+  const backupFile = `${tasksFile}.backup-${Date.now()}`;
+  
+  // Backup current file
+  if (fs.existsSync(tasksFile)) {
+    fs.copyFileSync(tasksFile, backupFile);
+    console.log(`  Backed up to: ${path.basename(backupFile)}`);
+  }
+  
+  // Write new tasks
+  const content = tasks.map(t => JSON.stringify(t)).join('\n');
+  fs.writeFileSync(tasksFile, content + '\n');
+}
+
+async function syncAndCleanTasks() {
+  console.log('🔄 Syncing and cleaning local tasks with Linear...\n');
+  
+  try {
+    // Fetch Linear tasks
+    const linearTasks = await fetchAllLinearTasks();
+    const linearMap = new Map(linearTasks.map(t => [t.identifier, t]));
+    
+    // Read local tasks
+    console.log('📂 Reading local tasks...');
+    const localTasks = readLocalTasks();
+    console.log(`  Found ${localTasks.length} local tasks\n`);
+    
+    // Build new clean task list from Linear
+    const cleanTasks = [];
+    const taskIdMap = new Map(); // To track duplicates
+    
+    for (const linearTask of linearTasks) {
+      // Skip if we already have this task ID
+      if (taskIdMap.has(linearTask.identifier)) {
+        console.log(`  Skipping duplicate: ${linearTask.identifier}`);
+        continue;
+      }
+      
+      // Find matching local task for metadata
+      const localTask = localTasks.find(t => 
+        t.linearId === linearTask.identifier ||
+        t.taskId === linearTask.identifier ||
+        t.id === linearTask.identifier
+      );
+      
+      // Create clean task entry
+      const cleanTask = {
+        id: linearTask.identifier,
+        taskId: linearTask.identifier,
+        linearId: linearTask.identifier,
+        title: linearTask.title,
+        description: linearTask.description || '',
+        status: mapLinearStatus(linearTask.state),
+        priority: linearTask.priority || 4,
+        createdAt: linearTask.createdAt,
+        updatedAt: linearTask.updatedAt,
+        linearState: linearTask.state.name,
+        linearStateType: linearTask.state.type,
+        // Preserve local metadata if it exists
+        ...(localTask ? {
+          localContext: localTask.localContext,
+          gitBranch: localTask.gitBranch,
+          files: localTask.files,
+          completedAt: localTask.completedAt
+        } : {})
+      };
+      
+      cleanTasks.push(cleanTask);
+      taskIdMap.set(linearTask.identifier, true);
+    }
+    
+    console.log('📊 Sync Results:');
+    console.log(`  Linear tasks: ${linearTasks.length}`);
+    console.log(`  Local tasks (before): ${localTasks.length}`);
+    console.log(`  Clean tasks (after): ${cleanTasks.length}`);
+    console.log(`  Removed duplicates: ${localTasks.length - cleanTasks.length}\n`);
+    
+    // Show what's being removed
+    const removedTasks = localTasks.filter(t => 
+      !linearMap.has(t.linearId) && 
+      !linearMap.has(t.taskId) &&
+      !linearMap.has(t.id)
+    );
+    
+    if (removedTasks.length > 0) {
+      console.log('🗑️  Removing local-only tasks (not in Linear):');
+      removedTasks.slice(0, 10).forEach(t => {
+        console.log(`  - ${t.taskId || t.id}: ${(t.title || '').substring(0, 50)}...`);
+      });
+      if (removedTasks.length > 10) {
+        console.log(`  ... and ${removedTasks.length - 10} more\n`);
+      }
+    }
+    
+    // Write clean tasks
+    console.log('💾 Writing clean task list...');
+    writeLocalTasks(cleanTasks);
+    
+    // Update Linear mappings
+    const mappingsFile = path.join(process.cwd(), '.stackmemory', 'linear-mappings.json');
+    const mappings = {};
+    
+    for (const task of cleanTasks) {
+      mappings[task.linearId] = {
+        linearId: task.linearId,
+        localId: task.id,
+        title: task.title,
+        state: task.linearState,
+        lastSync: new Date().toISOString()
+      };
+    }
+    
+    fs.writeFileSync(mappingsFile, JSON.stringify(mappings, null, 2));
+    console.log('  Updated linear-mappings.json\n');
+    
+    // Clean up old backup files
+    console.log('🧹 Cleaning up old backup files...');
+    const backupFiles = fs.readdirSync(path.join(process.cwd(), '.stackmemory'))
+      .filter(f => f.startsWith('tasks.jsonl.backup-'))
+      .sort()
+      .reverse();
+    
+    // Keep only the 3 most recent backups
+    const toDelete = backupFiles.slice(3);
+    toDelete.forEach(file => {
+      fs.unlinkSync(path.join(process.cwd(), '.stackmemory', file));
+      console.log(`  Deleted: ${file}`);
+    });
+    
+    if (toDelete.length === 0) {
+      console.log('  No old backups to delete');
+    }
+    
+    console.log('\n✅ Sync and cleanup complete!');
+    console.log(`  Active tasks: ${cleanTasks.filter(t => t.linearStateType !== 'completed' && t.linearStateType !== 'canceled').length}`);
+    console.log(`  Completed tasks: ${cleanTasks.filter(t => t.linearStateType === 'completed').length}`);
+    console.log(`  Total synced tasks: ${cleanTasks.length}`);
+    
+  } catch (error) {
+    console.error('❌ Sync failed:', error.message);
+    process.exit(1);
+  }
+}
+
+function mapLinearStatus(state) {
+  switch (state.type) {
+    case 'completed': return 'completed';
+    case 'started': return 'in_progress';
+    case 'canceled': return 'cancelled';
+    case 'backlog': return 'backlog';
+    default: return 'todo';
+  }
+}
+
+// Run if called directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  syncAndCleanTasks().catch(console.error);
+}
+
+export { syncAndCleanTasks };

package/scripts/sync-frames-from-railway.js

@@ -0,0 +1,228 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+import pg from 'pg';
+import Database from 'better-sqlite3';
+import { mkdirSync } from 'fs';
+import { dirname, join } from 'path';
+import { homedir } from 'os';
+
+const { Client } = pg;
+
+// Railway PostgreSQL connection
+// Use environment variable or fallback to internal URL
+const RAILWAY_DATABASE_URL = process.env.RAILWAY_DATABASE_URL || 
+  process.env.DATABASE_URL ||
+  'postgresql://postgres:YTSFXqPzFhghOcefgwPvJyWOBTYHbYxd@postgres.railway.internal:5432/railway';
+
+// Local SQLite database path
+const dbPath = join(homedir(), '.stackmemory', 'context.db');
+mkdirSync(dirname(dbPath), { recursive: true });
+
+async function syncFramesFromRailway() {
+  console.log('🔄 Starting sync from Railway database...\n');
+  
+  // Connect to Railway PostgreSQL
+  const pgClient = new Client({
+    connectionString: RAILWAY_DATABASE_URL,
+  });
+  
+  try {
+    await pgClient.connect();
+    console.log('✅ Connected to Railway PostgreSQL database');
+    
+    // Connect to local SQLite
+    const sqliteDb = new Database(dbPath);
+    console.log('✅ Connected to local SQLite database\n');
+    
+    // Check if frames table exists in PostgreSQL
+    const tableCheckQuery = `
+      SELECT EXISTS (
+        SELECT FROM information_schema.tables 
+        WHERE table_name = 'frames'
+      );
+    `;
+    
+    const tableExists = await pgClient.query(tableCheckQuery);
+    
+    if (!tableExists.rows[0].exists) {
+      console.log('⚠️  No frames table found in Railway database');
+      console.log('   The Railway deployment may not have created frames yet.\n');
+      
+      // Check for other relevant tables
+      const tablesQuery = `
+        SELECT table_name 
+        FROM information_schema.tables 
+        WHERE table_schema = 'public'
+        ORDER BY table_name;
+      `;
+      
+      const tables = await pgClient.query(tablesQuery);
+      console.log('📊 Available tables in Railway database:');
+      tables.rows.forEach(row => {
+        console.log(`   - ${row.table_name}`);
+      });
+      
+      return;
+    }
+    
+    // Fetch frames from Railway
+    const framesQuery = `
+      SELECT 
+        frame_id,
+        run_id,
+        project_id,
+        parent_frame_id,
+        depth,
+        type,
+        name,
+        state,
+        inputs,
+        outputs,
+        digest_text,
+        digest_json,
+        created_at,
+        closed_at
+      FROM frames
+      ORDER BY created_at DESC
+      LIMIT 1000;
+    `;
+    
+    const framesResult = await pgClient.query(framesQuery);
+    console.log(`📥 Found ${framesResult.rows.length} frames in Railway database\n`);
+    
+    if (framesResult.rows.length === 0) {
+      console.log('ℹ️  No frames to sync. The Railway database is empty.');
+      return;
+    }
+    
+    // Prepare SQLite insert statement
+    const insertStmt = sqliteDb.prepare(`
+      INSERT OR REPLACE INTO frames (
+        frame_id, run_id, project_id, parent_frame_id, depth,
+        type, name, state, inputs, outputs, digest_text, digest_json,
+        created_at, closed_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    
+    // Begin transaction for bulk insert
+    const insertMany = sqliteDb.transaction((frames) => {
+      for (const frame of frames) {
+        insertStmt.run(
+          frame.frame_id,
+          frame.run_id,
+          frame.project_id,
+          frame.parent_frame_id,
+          frame.depth,
+          frame.type,
+          frame.name,
+          frame.state,
+          typeof frame.inputs === 'object' ? JSON.stringify(frame.inputs) : frame.inputs,
+          typeof frame.outputs === 'object' ? JSON.stringify(frame.outputs) : frame.outputs,
+          frame.digest_text,
+          typeof frame.digest_json === 'object' ? JSON.stringify(frame.digest_json) : frame.digest_json,
+          frame.created_at ? new Date(frame.created_at).getTime() : Date.now(),
+          frame.closed_at ? new Date(frame.closed_at).getTime() : null
+        );
+      }
+    });
+    
+    // Execute bulk insert
+    insertMany(framesResult.rows);
+    console.log(`✅ Synced ${framesResult.rows.length} frames to local database\n`);
+    
+    // Also sync events if they exist
+    const eventsCheckQuery = `
+      SELECT EXISTS (
+        SELECT FROM information_schema.tables 
+        WHERE table_name = 'events'
+      );
+    `;
+    
+    const eventsExist = await pgClient.query(eventsCheckQuery);
+    
+    if (eventsExist.rows[0].exists) {
+      const eventsQuery = `
+        SELECT 
+          event_id,
+          run_id,
+          frame_id,
+          seq,
+          event_type,
+          payload,
+          ts
+        FROM events
+        ORDER BY ts DESC
+        LIMIT 5000;
+      `;
+      
+      const eventsResult = await pgClient.query(eventsQuery);
+      
+      if (eventsResult.rows.length > 0) {
+        console.log(`📥 Found ${eventsResult.rows.length} events in Railway database`);
+        
+        const eventInsertStmt = sqliteDb.prepare(`
+          INSERT OR REPLACE INTO events (
+            event_id, run_id, frame_id, seq, event_type, payload, ts
+          ) VALUES (?, ?, ?, ?, ?, ?, ?)
+        `);
+        
+        const insertEvents = sqliteDb.transaction((events) => {
+          for (const event of events) {
+            eventInsertStmt.run(
+              event.event_id,
+              event.run_id,
+              event.frame_id,
+              event.seq,
+              event.event_type,
+              typeof event.payload === 'object' ? JSON.stringify(event.payload) : event.payload,
+              event.ts ? new Date(event.ts).getTime() : Date.now()
+            );
+          }
+        });
+        
+        insertEvents(eventsResult.rows);
+        console.log(`✅ Synced ${eventsResult.rows.length} events to local database\n`);
+      }
+    }
+    
+    // Verify the sync
+    const frameCount = sqliteDb.prepare('SELECT COUNT(*) as count FROM frames').get();
+    const eventCount = sqliteDb.prepare('SELECT COUNT(*) as count FROM events').get();
+    
+    console.log('📊 Local database statistics:');
+    console.log(`   - Frames: ${frameCount.count}`);
+    console.log(`   - Events: ${eventCount.count}`);
+    
+    // Show recent frames
+    const recentFrames = sqliteDb.prepare(`
+      SELECT frame_id, name, type, state, datetime(created_at/1000, 'unixepoch') as created
+      FROM frames
+      ORDER BY created_at DESC
+      LIMIT 5
+    `).all();
+    
+    if (recentFrames.length > 0) {
+      console.log('\n🕐 Recent frames:');
+      recentFrames.forEach(frame => {
+        console.log(`   - ${frame.name} (${frame.type}) - ${frame.state} - ${frame.created}`);
+      });
+    }
+    
+    sqliteDb.close();
+    
+  } catch (error) {
+    console.error('❌ Error syncing frames:', error.message);
+    
+    // If connection failed due to internal network, try external URL
+    if (error.message.includes('ENOTFOUND') || error.message.includes('postgres.railway.internal')) {
+      console.log('\n🔄 Retrying with external Railway database URL...');
+      console.log('   Note: You may need to get the external DATABASE_URL from Railway dashboard.');
+      console.log('   Run: railway variables --json | jq -r .DATABASE_URL');
+    }
+  } finally {
+    await pgClient.end();
+  }
+}
+
+// Run the sync
+syncFramesFromRailway().catch(console.error);