@aikeytake/social-automation 2.0.0

package/src/cli.js

@@ -0,0 +1,197 @@
+import dotenv from 'dotenv';
+import createLogger from './utils/logger.js';
+import storage from './utils/storage.js';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+dotenv.config();
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const logger = createLogger('CLI');
+
+async function showQueue() {
+  const items = storage.getQueueItems();
+
+  if (items.length === 0) {
+    console.log('\n📭 Queue is empty\n');
+    return;
+  }
+
+  console.log(`\n📥 Queue (${items.length} items):\n`);
+
+  items.forEach((item, index) => {
+    console.log(`${index + 1}. ${item.title}`);
+    console.log(`   Source: ${item.sourceName} | Score: ${item.processing?.relevanceScore || 0}`);
+    console.log(`   Status: ${item.status}`);
+    console.log(`   Added: ${new Date(item.queuedAt).toLocaleString()}`);
+    console.log();
+  });
+}
+
+async function showDrafts() {
+  const drafts = storage.readAll('drafts');
+
+  if (drafts.length === 0) {
+    console.log('\n📝 No drafts available\n');
+    return;
+  }
+
+  console.log(`\n📝 Drafts (${drafts.length} items):\n`);
+
+  drafts.forEach((draft, index) => {
+    console.log(`${index + 1}. ${draft.title}`);
+    console.log(`   Source: ${draft.sourceName} | Tags: ${draft.tags?.join(', ') || 'N/A'}`);
+    console.log(`   Summary: ${draft.summary?.substring(0, 80)}...`);
+    console.log(`   Created: ${new Date(draft.processedAt).toLocaleString()}`);
+    console.log();
+  });
+}
+
+async function showDraft(id) {
+  const draft = storage.read('drafts', id);
+
+  if (!draft) {
+    console.log(`\n❌ Draft ${id} not found\n`);
+    return;
+  }
+
+  console.log('\n' + '='.repeat(80));
+  console.log(`📝 Draft: ${draft.title}`);
+  console.log('='.repeat(80));
+  console.log(`\n📊 Source: ${draft.sourceName}`);
+  console.log(`🔗 Link: ${draft.link}`);
+  console.log(`📅 Published: ${new Date(draft.pubDate).toLocaleString()}`);
+  console.log(`🏷️  Tags: ${draft.tags?.join(', ') || 'N/A'}`);
+
+  console.log(`\n💡 Summary:`);
+  console.log(`   ${draft.summary}`);
+
+  console.log(`\n📌 Suggested Titles:`);
+  draft.suggestedTitles?.forEach((title, i) => {
+    console.log(`   ${i + 1}. ${title}`);
+  });
+
+  console.log(`\n✍️  Rewritten Content:`);
+  console.log('─'.repeat(80));
+  console.log(draft.rewrittenContent);
+  console.log('─'.repeat(80));
+
+  console.log(`\n📄 Original Content:`);
+  console.log('─'.repeat(80));
+  console.log(draft.content?.substring(0, 500) + '...');
+  console.log('─'.repeat(80));
+
+  console.log();
+}
+
+async function approveDraft(id) {
+  const draft = storage.read('drafts', id);
+
+  if (!draft) {
+    console.log(`\n❌ Draft ${id} not found\n`);
+    return;
+  }
+
+  // Move to published
+  draft.status = 'approved';
+  draft.approvedAt = new Date().toISOString();
+  storage.write('published', id, draft);
+  storage.delete('drafts', id);
+
+  console.log(`\n✅ Draft "${draft.title.substring(0, 50)}..." approved for publishing\n`);
+}
+
+async function deleteItem(type, id) {
+  if (storage.exists(type, id)) {
+    storage.delete(type, id);
+    console.log(`\n🗑️  Item ${id} deleted from ${type}\n`);
+  } else {
+    console.log(`\n❌ Item ${id} not found in ${type}\n`);
+  }
+}
+
+async function showPublished() {
+  const items = storage.readAll('published');
+
+  if (items.length === 0) {
+    console.log('\n✅ No published items yet\n');
+    return;
+  }
+
+  console.log(`\n✅ Published (${items.length} items):\n`);
+
+  items.forEach((item, index) => {
+    console.log(`${index + 1}. ${item.title}`);
+    console.log(`   Published: ${new Date(item.approvedAt).toLocaleString()}`);
+    console.log(`   Status: ${item.status}`);
+    console.log();
+  });
+}
+
+async function main() {
+  const command = process.argv[2];
+  const arg = process.argv[3];
+
+  switch (command) {
+    case 'queue':
+      await showQueue();
+      break;
+
+    case 'drafts':
+      await showDrafts();
+      break;
+
+    case 'draft':
+      if (!arg) {
+        console.log('\n❌ Please provide a draft ID\n');
+        process.exit(1);
+      }
+      await showDraft(arg);
+      break;
+
+    case 'approve':
+      if (!arg) {
+        console.log('\n❌ Please provide a draft ID to approve\n');
+        process.exit(1);
+      }
+      await approveDraft(arg);
+      break;
+
+    case 'delete':
+      if (!arg) {
+        console.log('\n❌ Please provide item type (queue/drafts/published) and ID\n');
+        process.exit(1);
+      }
+      const type = process.argv[3];
+      const id = process.argv[4];
+      await deleteItem(type, id);
+      break;
+
+    case 'published':
+      await showPublished();
+      break;
+
+    default:
+      console.log(`
+Usage: node src/cli.js [command] [arguments]
+
+Commands:
+  queue              - Show all queued items
+  drafts             - Show all drafts
+  draft <id>         - Show full draft content
+  approve <id>       - Approve draft for publishing
+  delete <type> <id> - Delete item (queue/drafts/published)
+  published          - Show all published items
+
+Examples:
+  node src/cli.js queue
+  node src/cli.js drafts
+  node src/cli.js draft 1234567890-abc
+  node src/cli.js approve 1234567890-abc
+  node src/cli.js delete drafts 1234567890-abc
+      `);
+  }
+}
+
+main().catch(console.error);

package/src/fetchers/api.js

@@ -0,0 +1,232 @@
+import axios from 'axios';
+import crypto from 'crypto';
+import createLogger from '../utils/logger.js';
+
+const logger = createLogger('APIFetcher');
+
+// --- Auth ---
+
+async function resolveAuthHeaders(auth) {
+  if (!auth) return {};
+
+  if (auth.type === 'bearer') {
+    const token = process.env[auth.tokenEnv];
+    if (!token) { logger.warn(`Env var ${auth.tokenEnv} not set`); return {}; }
+    return { Authorization: `Bearer ${token}` };
+  }
+
+  if (auth.type === 'oauth2_client_credentials') {
+    const clientId = process.env[auth.clientIdEnv];
+    const clientSecret = process.env[auth.clientSecretEnv];
+    if (!clientId || !clientSecret) {
+      logger.warn(`Env vars ${auth.clientIdEnv} or ${auth.clientSecretEnv} not set`);
+      return {};
+    }
+    const res = await axios.post(auth.tokenUrl, {
+      client_id: clientId,
+      client_secret: clientSecret,
+      grant_type: 'client_credentials',
+    });
+    return { Authorization: `Bearer ${res.data.access_token}` };
+  }
+
+  return {};
+}
+
+// --- Computed variables (runtime values injected into request params/variables) ---
+// Supported types:
+//   { "type": "daysAgo", "days": 7 } → ISO timestamp N days in the past
+
+function resolveComputedVariables(computedVars) {
+  if (!computedVars) return {};
+  const resolved = {};
+  for (const [key, spec] of Object.entries(computedVars)) {
+    if (spec.type === 'daysAgo') {
+      resolved[key] = new Date(Date.now() - spec.days * 86400000).toISOString();
+    }
+  }
+  return resolved;
+}
+
+// --- Value resolution ---
+// Mapping spec types:
+//   "fieldName"                               → direct field lookup
+//   "https://example.com/{field}"             → template string
+//   ["field1", "field2"]                      → join with \n\n
+//   { "path": "a.b.c" }                       → deep dot-notation path
+//   { "path": "arr", "map": "node.name" }     → map over array, extract subpath
+//   { "path": "arr", "map": "node.name", "index": 0 } → take one element
+//   { "field": "x", "split": ",", "index": 0 } → split string, take element
+//   { "field": "x", "split": "," }            → split string into array
+//   null                                      → skip (keep default)
+
+function getPath(obj, dotPath) {
+  return dotPath.split('.').reduce((acc, key) => acc?.[key], obj);
+}
+
+function resolveValue(item, spec) {
+  if (spec === null || spec === undefined) return undefined;
+
+  if (typeof spec === 'string') {
+    if (spec.includes('{')) {
+      return spec.replace(/\{(\w+)\}/g, (_, key) => item[key] ?? '');
+    }
+    const val = item[spec];
+    return val !== undefined ? val : undefined;
+  }
+
+  if (Array.isArray(spec)) {
+    const parts = spec.map(f => item[f]).filter(v => v !== null && v !== undefined && v !== '');
+    return parts.length ? parts.join('\n\n') : undefined;
+  }
+
+  if (typeof spec === 'object') {
+    if (spec.path !== undefined) {
+      const val = getPath(item, spec.path);
+      if (spec.map !== undefined) {
+        if (!Array.isArray(val)) return undefined;
+        const mapped = val.map(v => getPath(v, spec.map)).filter(Boolean);
+        return spec.index !== undefined ? mapped[spec.index] : mapped;
+      }
+      return val !== undefined ? val : undefined;
+    }
+
+    if (spec.field !== undefined) {
+      const val = item[spec.field];
+      if (val === null || val === undefined) return spec.index !== undefined ? undefined : [];
+      const parts = String(val).split(spec.split || ',').map(s => s.trim()).filter(Boolean);
+      return spec.index !== undefined ? (parts[spec.index] ?? undefined) : parts;
+    }
+  }
+
+  return undefined;
+}
+
+function setPath(obj, dotPath, value) {
+  const keys = dotPath.split('.');
+  let cur = obj;
+  for (let i = 0; i < keys.length - 1; i++) {
+    if (!cur[keys[i]]) cur[keys[i]] = {};
+    cur = cur[keys[i]];
+  }
+  cur[keys[keys.length - 1]] = value;
+}
+
+// --- Map one raw API item to the standard pipeline shape ---
+
+function mapItem(raw, source) {
+  const item = {
+    source: source.id,
+    sourceName: source.name,
+    scraped_at: new Date().toISOString(),
+    age_hours: 0,
+    tags: [],
+    engagement: { upvotes: 0, comments: 0 },
+    metadata: { score: 0 },
+  };
+
+  for (const [outputPath, spec] of Object.entries(source.mapping)) {
+    const value = resolveValue(raw, spec);
+    if (value !== undefined) setPath(item, outputPath, value);
+  }
+
+  // Keep link and url in sync
+  if (item.link && !item.url) item.url = item.link;
+  if (item.url && !item.link) item.link = item.url;
+
+  // Stable ID from link
+  item.id = crypto.createHash('md5')
+    .update(item.link || item.title || String(Math.random()))
+    .digest('hex');
+
+  // Normalise pubDate to ISO
+  if (item.pubDate && !String(item.pubDate).includes('T')) {
+    item.pubDate = new Date(item.pubDate).toISOString();
+  }
+  if (!item.pubDate) item.pubDate = new Date().toISOString();
+
+  // Source weight for trending score normalisation (default 1)
+  item.metadata.weight = source.weight ?? 1;
+
+  // Store weighted score for display
+  if (!item.metadata.score) {
+    item.metadata.score = Math.round((item.engagement.upvotes || 0) * item.metadata.weight);
+  }
+
+  return item;
+}
+
+// --- Main entry point ---
+
+export default async function apiFetch(source) {
+  const { request, response, filter, auth } = source;
+
+  let authHeaders;
+  try {
+    authHeaders = await resolveAuthHeaders(auth);
+  } catch (err) {
+    logger.error(`[${source.id}] Auth failed: ${err.message}`);
+    return [];
+  }
+
+  const headers = {
+    'User-Agent': 'social-automation-scraper/1.0',
+    ...authHeaders,
+    ...(request.headers || {}),
+  };
+
+  const computedVars = resolveComputedVariables(request.computedVariables);
+
+  let rawItems;
+  try {
+    let data;
+
+    if (request.graphql) {
+      const variables = { ...request.graphql.variables, ...computedVars };
+      const res = await axios.post(
+        request.url,
+        { query: request.graphql.query, variables },
+        { headers, timeout: 30000 }
+      );
+      data = res.data;
+    } else if (request.method === 'POST') {
+      const res = await axios.post(request.url, request.body || {}, {
+        headers,
+        params: { ...request.params, ...computedVars },
+        timeout: 15000,
+      });
+      data = res.data;
+    } else {
+      const res = await axios.get(request.url, {
+        headers,
+        params: { ...request.params, ...computedVars },
+        timeout: 15000,
+      });
+      data = res.data;
+    }
+
+    rawItems = response?.itemsPath ? getPath(data, response.itemsPath) : data;
+    if (!Array.isArray(rawItems)) rawItems = [];
+
+    if (response?.itemUnwrap) {
+      rawItems = rawItems.map(i => i[response.itemUnwrap]).filter(Boolean);
+    }
+  } catch (err) {
+    logger.error(`[${source.id}] Request failed: ${err.message}`);
+    return [];
+  }
+
+  // Apply optional filter
+  let filtered = rawItems;
+  if (filter) {
+    filtered = rawItems.filter(item => {
+      const val = getPath(item, filter.field) ?? 0;
+      return (filter.min === undefined || val >= filter.min) &&
+             (filter.max === undefined || val <= filter.max);
+    });
+  }
+
+  const items = filtered.map(raw => mapItem(raw, source));
+  logger.success(`[${source.id}] Fetched ${items.length} items`);
+  return items;
+}

package/src/fetchers/hackernews.js

@@ -0,0 +1,86 @@
+import axios from 'axios';
+import createLogger from '../utils/logger.js';
+
+const logger = createLogger('HackerNewsFetcher');
+
+export default async function hnFetch(config) {
+  const hnConfig = config.trendingSources?.hackernews;
+  if (!hnConfig?.enabled) {
+    return [];
+  }
+
+  const keywords = hnConfig.keywords || [];
+  const minPoints = hnConfig.minPoints || 50;
+  const maxAge = 48; // hours
+  const limit = hnConfig.limit || 30;
+  const cutoff = new Date(Date.now() - maxAge * 60 * 60 * 1000);
+
+  logger.info('Fetching top stories from Hacker News...');
+
+  const allItems = [];
+
+  try {
+    // Get top story IDs
+    const { data: topIds } = await axios.get(
+      'https://hacker-news.firebaseio.com/v0/topstories.json'
+    );
+
+    // Fetch story details
+    for (const id of topIds.slice(0, limit * 2)) {
+      try {
+        const { data } = await axios.get(
+          `https://hacker-news.firebaseio.com/v0/item/${id}.json`
+        );
+
+        const created = new Date(data.time * 1000);
+
+        // Skip old stories
+        if (created < cutoff) continue;
+
+        // Skip low-score stories
+        if ((data.score || 0) < minPoints) continue;
+
+        // Check keyword relevance
+        const searchText = (data.title + ' ' + (data.text || '')).toLowerCase();
+        const hasKeyword = keywords.some(kw =>
+          searchText.includes(kw.toLowerCase())
+        );
+
+        if (!hasKeyword) continue;
+
+        allItems.push({
+          id: `hn_${id}`,
+          source: 'hackernews',
+          sourceName: 'Hacker News',
+          title: data.title,
+          content: data.text || '',
+          summary: (data.text || '').substring(0, 200),
+          url: data.url || `https://news.ycombinator.com/item?id=${id}`,
+          hn_url: `https://news.ycombinator.com/item?id=${id}`,
+          author: data.by,
+          posted_at: new Date(data.time * 1000).toISOString(),
+          scraped_at: new Date().toISOString(),
+          age_hours: Math.floor((Date.now() - created.getTime()) / (1000 * 60 * 60)),
+          engagement: {
+            points: data.score || 0,
+            comments: data.descendants || 0
+          },
+          metadata: {
+            score: data.score || 0,
+            type: data.type
+          }
+        });
+      } catch (error) {
+        logger.debug(`Failed to fetch story ${id}`);
+      }
+
+      if (allItems.length >= limit) break;
+    }
+
+    logger.success(`Fetched ${allItems.length} stories from Hacker News`);
+  } catch (error) {
+    logger.error(`Error fetching HN stories: ${error.message}`);
+  }
+
+  return allItems;
+}