@aikeytake/social-automation 2.0.0

package/src/query.js

@@ -0,0 +1,316 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+class DataQuery {
+  constructor() {
+    this.dataDir = path.join(__dirname, '../data');
+  }
+
+  getTodayFolder() {
+    const today = new Date().toISOString().split('T')[0];
+    return path.join(this.dataDir, today);
+  }
+
+  getDateFolder(dateStr) {
+    return path.join(this.dataDir, dateStr);
+  }
+
+  loadTrending(dateStr = null) {
+    const folder = dateStr ? this.getDateFolder(dateStr) : this.getTodayFolder();
+    const filePath = path.join(folder, 'trending.json');
+
+    if (!fs.existsSync(filePath)) {
+      console.error(`❌ No trending data found for ${dateStr || 'today'}`);
+      return null;
+    }
+
+    const content = fs.readFileSync(filePath, 'utf-8');
+    return JSON.parse(content);
+  }
+
+  loadAll(dateStr = null) {
+    const folder = dateStr ? this.getDateFolder(dateStr) : this.getTodayFolder();
+    const filePath = path.join(folder, 'all.json');
+
+    if (!fs.existsSync(filePath)) {
+      console.error(`❌ No data found for ${dateStr || 'today'}`);
+      return null;
+    }
+
+    const content = fs.readFileSync(filePath, 'utf-8');
+    return JSON.parse(content);
+  }
+
+  // Query: Get trending items
+  getTrending(limit = 20, dateStr = null) {
+    const data = this.loadTrending(dateStr);
+    if (!data) return [];
+
+    return data.items.slice(0, limit);
+  }
+
+  // Query: Get by topic
+  getByTopic(topic, dateStr = null) {
+    const data = this.loadAll(dateStr);
+    if (!data) return [];
+
+    const lowerTopic = topic.toLowerCase();
+    return data.items.filter(item => {
+      const title = (item.title || '').toLowerCase();
+      const summary = (item.summary || item.content || '').toLowerCase();
+      const keywords = (item.keywords || item.tags || []).map(k => k.toLowerCase());
+
+      return title.includes(lowerTopic) ||
+             summary.includes(lowerTopic) ||
+             keywords.some(k => k.includes(lowerTopic));
+    });
+  }
+
+  // Query: Get fresh items (last N hours)
+  getFresh(hours = 24, dateStr = null) {
+    const data = this.loadAll(dateStr);
+    if (!data) return [];
+
+    const cutoffTime = Date.now() - (hours * 60 * 60 * 1000);
+
+    return data.items.filter(item => {
+      const itemTime = new Date(item.pubDate || item.created_at || item.scraped_at).getTime();
+      return itemTime > cutoffTime;
+    });
+  }
+
+  // Query: Search
+  search(query, dateStr = null) {
+    const data = this.loadAll(dateStr);
+    if (!data) return [];
+
+    const lowerQuery = query.toLowerCase();
+    return data.items.filter(item => {
+      const title = (item.title || '').toLowerCase();
+      const content = (item.content || item.summary || '').toLowerCase();
+
+      return title.includes(lowerQuery) || content.includes(lowerQuery);
+    });
+  }
+
+  // Query: Get by source
+  getBySource(source, dateStr = null) {
+    const folder = dateStr ? this.getDateFolder(dateStr) : this.getTodayFolder();
+    const filePath = path.join(folder, `${source}.json`);
+
+    if (!fs.existsSync(filePath)) {
+      console.error(`❌ No data found for source: ${source}`);
+      return [];
+    }
+
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const data = JSON.parse(content);
+    return data.items || [];
+  }
+
+  // Query: Compare two days
+  compareDays(date1, date2) {
+    const data1 = this.loadTrending(date1);
+    const data2 = this.loadTrending(date2);
+
+    if (!data1 || !data2) {
+      console.error('❌ Could not load data for comparison');
+      return null;
+    }
+
+    const titles1 = new Set(data1.items.map(i => i.title));
+    const titles2 = new Set(data2.items.map(i => i.title));
+
+    const newToday = [...titles2].filter(t => !titles1.has(t));
+    const goneToday = [...titles1].filter(t => !titles2.has(t));
+
+    return {
+      new: newToday,
+      gone: goneToday,
+      common: [...titles1].filter(t => titles2.has(t))
+    };
+  }
+
+  // Display: Show trending items
+  displayTrending(items, limit = 10) {
+    console.log('\n📊 Trending Items:\n');
+    items.slice(0, limit).forEach((item, index) => {
+      console.log(`${index + 1}. ${item.title || 'No title'}`);
+      console.log(`   Score: ${item.score || item.combined_score || 'N/A'}`);
+      console.log(`   Sources: ${(item.sources || [item.source]).join(', ')}`);
+      console.log(`   URL: ${item.url || item.link || 'N/A'}`);
+      console.log();
+    });
+  }
+
+  // Display: Show by topic
+  displayByTopic(items, topic) {
+    console.log(`\n🔍 Items about "${topic}":\n`);
+    items.forEach((item, index) => {
+      console.log(`${index + 1}. ${item.title || 'No title'}`);
+      console.log(`   Source: ${item.source || 'N/A'}`);
+      console.log(`   URL: ${item.url || item.link || 'N/A'}`);
+      console.log();
+    });
+  }
+
+  // Display: Show search results
+  displaySearchResults(items, query) {
+    console.log(`\n🔎 Search results for "${query}":\n`);
+    items.forEach((item, index) => {
+      console.log(`${index + 1}. ${item.title || 'No title'}`);
+      console.log(`   Source: ${item.source || 'N/A'}`);
+      if (item.summary) {
+        console.log(`   Summary: ${item.summary.substring(0, 100)}...`);
+      }
+      console.log();
+    });
+  }
+
+  // Display: Show comparison
+  displayComparison(comparison, date1, date2) {
+    console.log(`\n📅 Comparison: ${date1} vs ${date2}\n`);
+
+    if (comparison.new.length > 0) {
+      console.log('✨ New today:');
+      comparison.new.slice(0, 5).forEach(title => {
+        console.log(`  + ${title}`);
+      });
+      console.log();
+    }
+
+    if (comparison.gone.length > 0) {
+      console.log('📉 Gone today:');
+      comparison.gone.slice(0, 5).forEach(title => {
+        console.log(`  - ${title}`);
+      });
+      console.log();
+    }
+
+    console.log(`🔄 Common: ${comparison.common.length} items`);
+  }
+}
+
+// CLI interface
+async function main() {
+  const query = new DataQuery();
+  const args = process.argv.slice(2);
+
+  if (args.length === 0) {
+    console.log(`
+Usage: npm run query [command] [options]
+
+Commands:
+  trending           Show today's trending items
+  trending [date]    Show trending for specific date (YYYY-MM-DD)
+  topic [name]       Get items by topic
+  fresh [hours]      Get items from last N hours (default: 24)
+  search [query]     Search content
+  source [name]      Get items by source (reddit, hackernews, rss)
+  compare [d1] [d2]  Compare two days
+
+Options:
+  --limit=N          Limit results (default: 10)
+  --date=YYYY-MM-DD  Use specific date
+
+Examples:
+  npm run query trending
+  npm run query trending --limit=5
+  npm run query topic GPT
+  npm run query fresh 6
+  npm run query search "openai"
+  npm run query source reddit
+  npm run query compare 2025-03-05 2025-03-06
+    `);
+    return;
+  }
+
+  const command = args[0];
+  const limit = parseInt(args.find(a => a.startsWith('--limit='))?.split('=')[1]) || 10;
+  const dateArg = args.find(a => a.startsWith('--date='))?.split('=')[1];
+
+  switch (command) {
+    case 'trending': {
+      const items = query.getTrending(limit, dateArg);
+      query.displayTrending(items, limit);
+      break;
+    }
+
+    case 'topic': {
+      const topic = args[1];
+      if (!topic) {
+        console.error('❌ Please provide a topic name');
+        return;
+      }
+      const items = query.getByTopic(topic, dateArg);
+      query.displayByTopic(items, topic);
+      break;
+    }
+
+    case 'fresh': {
+      const hours = parseInt(args[1]) || 24;
+      const items = query.getFresh(hours, dateArg);
+      console.log(`\n⏰ Items from last ${hours} hours:\n`);
+      items.slice(0, limit).forEach((item, index) => {
+        console.log(`${index + 1}. ${item.title || 'No title'}`);
+        console.log(`   Age: ${item.age_hours || 'N/A'} hours`);
+        console.log(`   Source: ${item.source || 'N/A'}`);
+        console.log();
+      });
+      break;
+    }
+
+    case 'search': {
+      const searchQuery = args[1];
+      if (!searchQuery) {
+        console.error('❌ Please provide a search query');
+        return;
+      }
+      const items = query.search(searchQuery, dateArg);
+      query.displaySearchResults(items, searchQuery);
+      break;
+    }
+
+    case 'source': {
+      const source = args[1];
+      if (!source) {
+        console.error('❌ Please provide a source name (reddit, hackernews, rss)');
+        return;
+      }
+      const items = query.getBySource(source, dateArg);
+      console.log(`\n📰 Items from ${source}:\n`);
+      items.slice(0, limit).forEach((item, index) => {
+        console.log(`${index + 1}. ${item.title || 'No title'}`);
+        console.log(`   Score: ${item.metadata?.score || item.engagement?.upvotes || 'N/A'}`);
+        console.log(`   URL: ${item.url || item.link || 'N/A'}`);
+        console.log();
+      });
+      break;
+    }
+
+    case 'compare': {
+      const date1 = args[1];
+      const date2 = args[2];
+      if (!date1 || !date2) {
+        console.error('❌ Please provide two dates to compare (YYYY-MM-DD YYYY-MM-DD)');
+        return;
+      }
+      const comparison = query.compareDays(date1, date2);
+      if (comparison) {
+        query.displayComparison(comparison, date1, date2);
+      }
+      break;
+    }
+
+    default:
+      console.error(`❌ Unknown command: ${command}`);
+      console.log('Run "npm run query" to see available commands');
+  }
+}
+
+main().catch(console.error);
+
+export default DataQuery;

package/src/utils/logger.js

@@ -0,0 +1,74 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const LOGS_DIR = path.join(__dirname, '../../logs');
+
+class Logger {
+  constructor(component) {
+    this.component = component;
+    this.ensureLogsDir();
+  }
+
+  ensureLogsDir() {
+    if (!fs.existsSync(LOGS_DIR)) {
+      fs.mkdirSync(LOGS_DIR, { recursive: true });
+    }
+  }
+
+  getLogFilePath(type) {
+    const date = new Date().toISOString().split('T')[0];
+    return path.join(LOGS_DIR, `${date}-${type}.log`);
+  }
+
+  formatMessage(level, message, data = null) {
+    const timestamp = new Date().toISOString();
+    const baseMsg = `[${timestamp}] [${level}] [${this.component}] ${message}`;
+    if (data) {
+      return `${baseMsg}\n${JSON.stringify(data, null, 2)}`;
+    }
+    return baseMsg;
+  }
+
+  writeToFile(type, message) {
+    const filePath = this.getLogFilePath(type);
+    fs.appendFileSync(filePath, message + '\n');
+  }
+
+  info(message, data) {
+    const msg = this.formatMessage('INFO', message, data);
+    console.log(msg);
+    this.writeToFile('info', msg);
+  }
+
+  error(message, data) {
+    const msg = this.formatMessage('ERROR', message, data);
+    console.error(msg);
+    this.writeToFile('error', msg);
+  }
+
+  warn(message, data) {
+    const msg = this.formatMessage('WARN', message, data);
+    console.warn(msg);
+    this.writeToFile('warn', msg);
+  }
+
+  debug(message, data) {
+    const msg = this.formatMessage('DEBUG', message, data);
+    if (process.env.DEBUG === 'true') {
+      console.debug(msg);
+    }
+    this.writeToFile('debug', msg);
+  }
+
+  success(message, data) {
+    const msg = this.formatMessage('SUCCESS', message, data);
+    console.log(`\x1b[32m${msg}\x1b[0m`);
+    this.writeToFile('success', msg);
+  }
+}
+
+export default function createLogger(component) {
+  return new Logger(component);
+}

package/src/utils/storage.js

@@ -0,0 +1,134 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const DATA_DIR = path.join(__dirname, '../../data');
+
+class Storage {
+  constructor() {
+    this.ensureDataDirs();
+  }
+
+  ensureDataDirs() {
+    const dirs = ['queue', 'drafts', 'published'];
+    dirs.forEach(dir => {
+      const dirPath = path.join(DATA_DIR, dir);
+      if (!fs.existsSync(dirPath)) {
+        fs.mkdirSync(dirPath, { recursive: true });
+      }
+    });
+  }
+
+  getFilePath(type, filename) {
+    return path.join(DATA_DIR, type, filename);
+  }
+
+  readAll(type) {
+    const dirPath = path.join(DATA_DIR, type);
+    const files = fs.readdirSync(dirPath).filter(f => f.endsWith('.json'));
+    return files.map(file => {
+      const content = fs.readFileSync(path.join(dirPath, file), 'utf-8');
+      return JSON.parse(content);
+    });
+  }
+
+  read(type, id) {
+    const filePath = this.getFilePath(type, `${id}.json`);
+    if (fs.existsSync(filePath)) {
+      const content = fs.readFileSync(filePath, 'utf-8');
+      return JSON.parse(content);
+    }
+    return null;
+  }
+
+  write(type, id, data) {
+    const filePath = this.getFilePath(type, `${id}.json`);
+    fs.writeFileSync(filePath, JSON.stringify(data, null, 2));
+    return true;
+  }
+
+  delete(type, id) {
+    const filePath = this.getFilePath(type, `${id}.json`);
+    if (fs.existsSync(filePath)) {
+      fs.unlinkSync(filePath);
+      return true;
+    }
+    return false;
+  }
+
+  exists(type, id) {
+    const filePath = this.getFilePath(type, `${id}.json`);
+    return fs.existsSync(filePath);
+  }
+
+  moveTo(fromType, toType, id) {
+    const data = this.read(fromType, id);
+    if (data) {
+      this.write(toType, id, data);
+      this.delete(fromType, id);
+      return true;
+    }
+    return false;
+  }
+
+  addToQueue(item) {
+    const id = this.generateId();
+    item.id = id;
+    item.queuedAt = new Date().toISOString();
+    item.status = 'queued';
+    this.write('queue', id, item);
+    return id;
+  }
+
+  getQueueItems(status = null) {
+    const items = this.readAll('queue');
+    if (status) {
+      return items.filter(item => item.status === status);
+    }
+    return items.sort((a, b) => new Date(a.queuedAt) - new Date(b.queuedAt));
+  }
+
+  updateQueueStatus(id, status) {
+    const item = this.read('queue', id);
+    if (item) {
+      item.status = status;
+      item.updatedAt = new Date().toISOString();
+      this.write('queue', id, item);
+      return true;
+    }
+    return false;
+  }
+
+  generateId() {
+    return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  isDuplicate(content) {
+    // Check if content already exists in queue or drafts
+    const normalizedContent = this.normalizeContent(content);
+    const queueItems = this.readAll('queue');
+    const draftItems = this.readAll('drafts');
+    const publishedItems = this.readAll('published');
+
+    const allItems = [...queueItems, ...draftItems, ...publishedItems];
+
+    for (const item of allItems) {
+      if (this.normalizeContent(item.originalContent || item.content) === normalizedContent) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  normalizeContent(content) {
+    return content
+      .toLowerCase()
+      .replace(/\s+/g, ' ')
+      .replace(/[^\w\s]/g, '')
+      .trim()
+      .substring(0, 200);
+  }
+}
+
+export default new Storage();

package/src/writing-agents/QUICK-REFERENCE.md

@@ -0,0 +1,111 @@
+# Writing Agents - Quick Reference Guide
+
+## Quick Comparison: Original vs Improved
+
+| Aspect | Original | Improved |
+|--------|----------|----------|
+| **Writer guidance** | Basic requirements | Examples + anti-patterns |
+| **Critic rubrics** | Basic descriptions | 5-level score rubrics |
+| **Evaluation** | Generic questions | Specific questions per criterion |
+| **Common issues** | Not documented | Listed with point deductions |
+| **Tone guidance** | "Be objective" | Good vs bad examples |
+
+## Key Files
+
+| File | Purpose |
+|------|---------|
+| `prompt-templates.js` | Original prompts (currently in use) |
+| `prompt-templates-improved.js` | Improved prompts (ready for testing) |
+| `WRITING-SKILLS-IMPROVEMENTS.md` | Full documentation |
+| `QUICK-REFERENCE.md` | This file |
+
+## Quality Criteria At a Glance
+
+| Criterion | Weight | Key Question |
+|-----------|--------|--------------|
+| **Accuracy** | 20% | Claims supported by sources? |
+| **Clarity** | 20% | Easy to understand? |
+| **Value** | 25% | Actionable insights? |
+| **Completeness** | 15% | Key points covered? |
+| **Voice** | 10% | Objective tone? |
+| **Citations** | 10% | Sources credited? |
+
+## Score Ranges
+
+| Score | Meaning | Action |
+|-------|---------|--------|
+| 9-10 | Excellent | Ready |
+| 7-8 | Good | Minor tweaks |
+| 5-6 | Fair | Needs revision |
+| 3-4 | Poor | Major revision |
+| 1-2 | Fail | Start over |
+
+## Thresholds
+
+- **Quality threshold:** 8.0/10 (satisfactory)
+- **Max iterations:** 3
+- **Target iterations:** 1-2
+
+## Common Issues Quick Reference
+
+| Issue | Impact | Criterion |
+|-------|--------|-----------|
+| Claim not in sources | -2 | Accuracy |
+- Promotional language | -3 | Voice |
+- Only summarizes | -3 | Value |
+- Unexplained jargon | -1 | Clarity |
+- Missing citations | -2 | Citations |
+- Prescriptive (you must) | -2 | Voice |
+
+## Testing Checklist
+
+```
+□ Generate newsletter with improved prompts
+□ Compare quality vs baseline
+□ Check critic scoring consistency
+□ Verify no promotional language
+□ Confirm all claims cited
+□ Validate recommendation actionability
+□ Measure iterations to threshold
+```
+
+## Migration Steps
+
+1. Test improved prompts in parallel
+2. Compare 5-10 newsletters side by side
+3. Validate quality improvement
+4. Update environment if needed
+5. Deploy improved prompts
+6. Monitor metrics for 2 weeks
+
+## Environment Variables (No Changes)
+
+```bash
+WRITING_AGENTS_ENABLED=true
+WRITING_AGENTS_MAX_ITERATIONS=3
+WRITING_AGENTS_QUALITY_THRESHOLD=8
+WRITING_AGENTS_MODEL=claude-3-5-sonnet-20241022
+```
+
+## CLI Commands (No Changes)
+
+```bash
+# Generate with existing prompts
+npm run newsletter:generate:enhanced 2026-03-13
+
+# To use improved prompts: Update import in agents
+# Change from: './prompt-templates.js'
+# Change to:   './prompt-templates-improved.js'
+```
+
+## Contacts
+
+For issues or questions about the improved writing skills:
+- Documentation: `WRITING-SKILLS-IMPROVEMENTS.md`
+- Implementation: `agents/writer-agent.js`, `agents/critic-agent.js`
+- Original design: `../../WRITING-AGENTS-IMPLEMENTATION.md`
+
+---
+
+**Last Updated:** 2026-03-13
+**Status:** Improved prompts ready for testing