@lobu/cli 6.0.1 → 7.0.0

package/dist/connectors/reddit.ts

@@ -0,0 +1,600 @@
+/**
+ * Reddit Connector (V1 runtime)
+ *
+ * Fetches posts and comments from Reddit subreddits or search queries.
+ * Supports both authenticated (OAuth) and unauthenticated (public JSON API) modes.
+ */
+
+import {
+  type ActionContext,
+  type ActionResult,
+  type ConnectorDefinition,
+  ConnectorRuntime,
+  calculateEngagementScore,
+  type EventEnvelope,
+  type SyncContext,
+  type SyncResult,
+} from '@lobu/connector-sdk';
+
+// ---------------------------------------------------------------------------
+// Reddit API types
+// ---------------------------------------------------------------------------
+
+interface RedditPost {
+  name: string;
+  id: string;
+  title: string;
+  selftext: string;
+  author: string;
+  permalink: string;
+  url: string;
+  created_utc: number;
+  score: number;
+  ups: number;
+  num_comments: number;
+  upvote_ratio: number;
+  is_self: boolean;
+  domain: string;
+  subreddit: string;
+  crosspost_parent?: string;
+  thumbnail?: string;
+}
+
+interface RedditComment {
+  name: string;
+  id: string;
+  body: string;
+  author: string;
+  permalink: string;
+  created_utc: number;
+  score: number;
+  ups: number;
+  parent_id: string;
+  link_id: string;
+  subreddit: string;
+}
+
+interface RedditListingResponse {
+  data: {
+    children: Array<{
+      kind: string;
+      data: RedditPost & RedditComment;
+    }>;
+    after: string | null;
+  };
+}
+
+interface RedditCheckpoint {
+  last_timestamp?: string;
+  pagination_token?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Connector
+// ---------------------------------------------------------------------------
+
+export default class RedditConnector extends ConnectorRuntime {
+  readonly definition: ConnectorDefinition = {
+    key: 'reddit',
+    name: 'Reddit',
+    description: 'Fetches posts and comments from Reddit subreddits or search queries.',
+    version: '1.0.0',
+    authSchema: {
+      methods: [
+        {
+          type: 'oauth',
+          provider: 'reddit',
+          requiredScopes: ['identity', 'read', 'history'],
+          setupInstructions:
+            'Create a Reddit app at https://www.reddit.com/prefs/apps — choose "web app" as the type. Set the redirect URI to {{redirect_uri}}, then copy the client ID and secret below.',
+        },
+        {
+          type: 'none',
+        },
+      ],
+    },
+    feeds: {
+      posts: {
+        key: 'posts',
+        name: 'Posts',
+        description: 'Fetch posts from subreddits or search queries.',
+        displayNameTemplate: 'r/{subreddit} posts',
+        configSchema: {
+          type: 'object',
+          properties: {
+            subreddit: {
+              type: 'string',
+              description: 'Subreddit name without r/ prefix (e.g., "programming").',
+            },
+            search_terms: {
+              type: 'string',
+              description: 'Search terms to query across Reddit.',
+            },
+            lookback_days: {
+              type: 'integer',
+              minimum: 1,
+              maximum: 730,
+              default: 365,
+              description: 'Number of days to look back for historical data.',
+            },
+          },
+        },
+        eventKinds: {
+          post: {
+            description: 'A Reddit post (self-post or link)',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                subreddit: { type: 'string' },
+                score: { type: 'number', description: 'Reddit score (upvotes - downvotes)' },
+                num_comments: { type: 'number' },
+                upvote_ratio: { type: 'number' },
+                is_self: {
+                  type: 'boolean',
+                  description: 'True for text posts, false for link/media posts',
+                },
+                domain: {
+                  type: 'string',
+                  description: 'Content domain (e.g., "i.redd.it", "self.programming")',
+                },
+                thumbnail: { type: 'string', format: 'uri', description: 'Preview thumbnail URL' },
+                media_url: {
+                  type: 'string',
+                  format: 'uri',
+                  description: 'Linked content URL for non-self posts (image, video, article)',
+                },
+              },
+            },
+          },
+        },
+      },
+      comments: {
+        key: 'comments',
+        name: 'Comments',
+        description: 'Fetch comments from subreddits.',
+        displayNameTemplate: 'r/{subreddit} comments',
+        configSchema: {
+          type: 'object',
+          properties: {
+            subreddit: {
+              type: 'string',
+              description: 'Subreddit name without r/ prefix (e.g., "programming").',
+            },
+            lookback_days: {
+              type: 'integer',
+              minimum: 1,
+              maximum: 730,
+              default: 365,
+              description: 'Number of days to look back for historical data.',
+            },
+          },
+        },
+        eventKinds: {
+          comment: {
+            description: 'A Reddit comment',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                subreddit: { type: 'string' },
+                score: { type: 'number', description: 'Reddit score (upvotes - downvotes)' },
+              },
+            },
+          },
+        },
+      },
+      user_activity: {
+        key: 'user_activity',
+        name: 'User activity',
+        description:
+          "Fetch a Reddit user's posts and comments interleaved. Defaults to the connected user.",
+        displayNameTemplate: 'u/{username} activity',
+        configSchema: {
+          type: 'object',
+          properties: {
+            username: {
+              type: 'string',
+              description:
+                "Reddit username without u/ prefix. Leave empty to use the connected account's identity.",
+            },
+            lookback_days: {
+              type: 'integer',
+              minimum: 1,
+              maximum: 730,
+              default: 365,
+              description: 'Number of days to look back for historical activity.',
+            },
+          },
+        },
+        eventKinds: {
+          post: {
+            description: 'A Reddit post authored by the user',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                subreddit: { type: 'string' },
+                score: { type: 'number', description: 'Reddit score (upvotes - downvotes)' },
+                num_comments: { type: 'number' },
+                upvote_ratio: { type: 'number' },
+                is_self: { type: 'boolean' },
+                domain: { type: 'string' },
+              },
+            },
+          },
+          comment: {
+            description: 'A Reddit comment authored by the user',
+            metadataSchema: {
+              type: 'object',
+              properties: {
+                subreddit: { type: 'string' },
+                score: { type: 'number', description: 'Reddit score (upvotes - downvotes)' },
+              },
+            },
+          },
+        },
+      },
+    },
+    optionsSchema: {
+      type: 'object',
+      properties: {
+        subreddit: {
+          type: 'string',
+          description: 'Subreddit name without r/ prefix (e.g., "programming").',
+        },
+        search_terms: {
+          type: 'string',
+          description: 'Search terms to query across Reddit.',
+        },
+        lookback_days: {
+          type: 'integer',
+          minimum: 1,
+          maximum: 730,
+          default: 365,
+          description: 'Number of days to look back for historical data.',
+        },
+      },
+    },
+  };
+
+  private readonly MAX_PAGES = 10;
+  private readonly RATE_LIMIT_MS = 1000;
+  private readonly USER_AGENT = 'Lobu-Connector/1.0.0';
+
+  // -------------------------------------------------------------------------
+  // sync
+  // -------------------------------------------------------------------------
+
+  async sync(ctx: SyncContext): Promise<SyncResult> {
+    const subreddit = ctx.config.subreddit as string | undefined;
+    const searchTerms = ctx.config.search_terms as string | undefined;
+    const isUserFeed = ctx.feedKey === 'user_activity';
+    const contentType = isUserFeed ? 'overview' : ctx.feedKey === 'comments' ? 'comment' : 'post';
+    const lookbackDays = (ctx.config.lookback_days as number) ?? 365;
+
+    // Resolve access token: user OAuth > app-only OAuth > unauthenticated
+    const userAccessToken = ctx.credentials?.accessToken ?? null;
+    let accessToken: string | undefined = userAccessToken ?? undefined;
+    if (!accessToken) {
+      accessToken = await this.getAppOnlyToken(ctx);
+    }
+    const baseUrl = accessToken ? 'https://oauth.reddit.com' : 'https://www.reddit.com';
+
+    let username: string | undefined;
+    if (isUserFeed) {
+      if (!userAccessToken) {
+        throw new Error(
+          'user_activity feed requires user OAuth. Connect Reddit with read+history scopes.'
+        );
+      }
+      username = await this.resolveUsername(ctx, userAccessToken);
+    }
+
+    const cutoffDate = new Date();
+    cutoffDate.setDate(cutoffDate.getDate() - lookbackDays);
+
+    const events: EventEnvelope[] = [];
+    let after: string | null = null;
+    let page = 0;
+    let reachedCutoff = false;
+
+    while (page < this.MAX_PAGES && !reachedCutoff) {
+      const url = this.buildFetchUrl({
+        baseUrl,
+        subreddit,
+        searchTerms,
+        username,
+        contentType,
+        after,
+        isOAuth: !!accessToken,
+      });
+
+      const headers: Record<string, string> = {
+        'User-Agent': this.USER_AGENT,
+      };
+      if (accessToken) {
+        headers.Authorization = `Bearer ${accessToken}`;
+      }
+
+      const response = await fetch(url, { headers });
+      if (!response.ok) {
+        const status = response.status;
+        if (status === 429) {
+          throw new Error('Reddit rate limit exceeded. Please wait before retrying.');
+        }
+        if (status === 404) {
+          throw new Error('Subreddit or resource not found. Please check the subreddit name.');
+        }
+        if (status === 403) {
+          throw new Error('Access forbidden. The subreddit may be private or banned.');
+        }
+        throw new Error(`Reddit API error (${status}): ${await response.text()}`);
+      }
+
+      const listing = (await response.json()) as RedditListingResponse;
+      const children = listing.data.children;
+
+      if (children.length === 0) break;
+
+      for (const child of children) {
+        const itemData = child.data;
+        const itemDate = new Date(itemData.created_utc * 1000);
+
+        if (itemDate < cutoffDate) {
+          reachedCutoff = true;
+          break;
+        }
+
+        // Filter deleted/removed items
+        if (itemData.author === '[deleted]') continue;
+
+        // Use actual Reddit API kind (t3=post, t1=comment) instead of config
+        const isPost = child.kind === 't3';
+        const isComment = child.kind === 't1';
+
+        if (isPost) {
+          const post = itemData as RedditPost;
+          if (post.crosspost_parent) continue;
+          if (post.selftext === '[removed]' || post.selftext === '[deleted]') continue;
+
+          events.push(this.transformPost(post));
+        } else if (isComment) {
+          const comment = itemData as RedditComment;
+          if (!comment.body || comment.body === '[removed]' || comment.body === '[deleted]')
+            continue;
+
+          events.push(this.transformComment(comment));
+        }
+      }
+
+      after = listing.data.after;
+      if (!after) break;
+
+      page++;
+
+      if (page < this.MAX_PAGES && !reachedCutoff) {
+        await this.sleep(this.RATE_LIMIT_MS);
+      }
+    }
+
+    const checkpoint: RedditCheckpoint = {
+      last_timestamp: new Date().toISOString(),
+      pagination_token: after ?? undefined,
+    };
+
+    return {
+      events,
+      checkpoint: checkpoint as Record<string, unknown>,
+      metadata: {
+        items_found: events.length,
+      },
+    };
+  }
+
+  // -------------------------------------------------------------------------
+  // execute
+  // -------------------------------------------------------------------------
+
+  async execute(_ctx: ActionContext): Promise<ActionResult> {
+    return { success: false, error: 'Actions not supported' };
+  }
+
+  // -------------------------------------------------------------------------
+  // App-only OAuth
+  // -------------------------------------------------------------------------
+
+  private appOnlyToken: string | null = null;
+
+  private async getAppOnlyToken(ctx: SyncContext): Promise<string | undefined> {
+    if (this.appOnlyToken) return this.appOnlyToken;
+
+    const clientId = (ctx.config as Record<string, unknown>).REDDIT_CLIENT_ID as string | undefined;
+    const clientSecret = (ctx.config as Record<string, unknown>).REDDIT_CLIENT_SECRET as
+      | string
+      | undefined;
+    if (!clientId || !clientSecret) return undefined;
+
+    const userAgent =
+      ((ctx.config as Record<string, unknown>).REDDIT_USER_AGENT as string) || this.USER_AGENT;
+    const response = await fetch('https://www.reddit.com/api/v1/access_token', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        Authorization: `Basic ${btoa(`${clientId}:${clientSecret}`)}`,
+        'User-Agent': userAgent,
+      },
+      body: 'grant_type=client_credentials',
+    });
+
+    if (!response.ok) {
+      console.error(
+        `Reddit app-only auth failed (${response.status}), falling back to unauthenticated`
+      );
+      return undefined;
+    }
+
+    const data = (await response.json()) as { access_token?: string };
+    if (data.access_token) {
+      this.appOnlyToken = data.access_token;
+      return data.access_token;
+    }
+    return undefined;
+  }
+
+  // -------------------------------------------------------------------------
+  // Username resolution
+  // -------------------------------------------------------------------------
+
+  private async resolveUsername(ctx: SyncContext, userAccessToken: string): Promise<string> {
+    const configured = (ctx.config.username as string | undefined)?.trim();
+    if (configured) return configured.replace(/^u\//, '');
+
+    const response = await fetch('https://oauth.reddit.com/api/v1/me', {
+      headers: {
+        Authorization: `Bearer ${userAccessToken}`,
+        'User-Agent': this.USER_AGENT,
+      },
+    });
+    if (!response.ok) {
+      throw new Error(
+        `Failed to resolve Reddit username via /api/v1/me (${response.status}). ` +
+          'Set "username" in feed config or re-authenticate.'
+      );
+    }
+    const data = (await response.json()) as { name?: string };
+    if (!data.name) {
+      throw new Error('Reddit /api/v1/me returned no username.');
+    }
+    return data.name;
+  }
+
+  // -------------------------------------------------------------------------
+  // URL building
+  // -------------------------------------------------------------------------
+
+  private buildFetchUrl(params: {
+    baseUrl: string;
+    subreddit?: string;
+    searchTerms?: string;
+    username?: string;
+    contentType: string;
+    after: string | null;
+    isOAuth: boolean;
+  }): string {
+    const { baseUrl, subreddit, searchTerms, username, contentType, after, isOAuth } = params;
+    const jsonSuffix = isOAuth ? '' : '.json';
+    const afterParam = after ? `&after=${after}` : '';
+
+    if (contentType === 'overview') {
+      if (!username) {
+        throw new Error('user_activity feed requires a resolved username.');
+      }
+      return `${baseUrl}/user/${encodeURIComponent(username)}/overview${jsonSuffix}?limit=100&sort=new${afterParam}`;
+    }
+
+    if (contentType === 'comment') {
+      // Comments from a subreddit
+      if (subreddit) {
+        return `${baseUrl}/r/${subreddit}/comments${jsonSuffix}?limit=100${afterParam}`;
+      }
+      // Comments aren't searchable via Reddit search API, fall back to r/all
+      return `${baseUrl}/r/all/comments${jsonSuffix}?limit=100${afterParam}`;
+    }
+
+    // Posts mode
+    if (searchTerms) {
+      const query = encodeURIComponent(searchTerms);
+      if (subreddit) {
+        return `${baseUrl}/r/${subreddit}/search${jsonSuffix}?q=${query}&restrict_sr=on&sort=relevance&t=year&limit=100${afterParam}`;
+      }
+      return `${baseUrl}/search${jsonSuffix}?q=${query}&sort=relevance&t=year&limit=100${afterParam}`;
+    }
+
+    // Subreddit listing
+    if (subreddit) {
+      return `${baseUrl}/r/${subreddit}/new${jsonSuffix}?t=year&limit=100${afterParam}`;
+    }
+
+    // Fallback to r/all
+    return `${baseUrl}/r/all/new${jsonSuffix}?t=year&limit=100${afterParam}`;
+  }
+
+  // -------------------------------------------------------------------------
+  // Transform helpers
+  // -------------------------------------------------------------------------
+
+  private transformPost(post: RedditPost): EventEnvelope {
+    const engagementScore = calculateEngagementScore('reddit', {
+      score: post.score,
+      reply_count: post.num_comments,
+      upvotes: post.ups,
+    });
+
+    // For non-self posts, the Reddit `url` field points to the linked content (image, article, etc.)
+    const mediaUrl = !post.is_self ? post.url : undefined;
+    const thumbnail =
+      post.thumbnail && !['self', 'default', 'nsfw', 'spoiler', ''].includes(post.thumbnail)
+        ? post.thumbnail
+        : undefined;
+
+    return {
+      origin_id: `reddit_post_${post.name}`,
+      title: post.title,
+      payload_text: (post.selftext ?? '').trim(),
+      author_name: post.author,
+      source_url: `https://reddit.com${post.permalink}`,
+      occurred_at: new Date(post.created_utc * 1000),
+      origin_type: 'post',
+      score: engagementScore,
+      metadata: {
+        subreddit: post.subreddit,
+        score: post.score,
+        num_comments: post.num_comments,
+        upvote_ratio: post.upvote_ratio,
+        is_self: post.is_self,
+        domain: post.domain,
+        ...(thumbnail && { thumbnail }),
+        ...(mediaUrl && { media_url: mediaUrl }),
+      },
+    };
+  }
+
+  private transformComment(comment: RedditComment): EventEnvelope {
+    const engagementScore = calculateEngagementScore('reddit', {
+      score: comment.score,
+      upvotes: comment.ups,
+    });
+
+    let parentExternalId: string | undefined;
+    if (comment.parent_id) {
+      if (comment.parent_id.startsWith('t1_')) {
+        // Parent is another comment
+        parentExternalId = `reddit_comment_${comment.parent_id}`;
+      } else if (comment.parent_id.startsWith('t3_')) {
+        // Parent is a post
+        parentExternalId = `reddit_post_${comment.parent_id}`;
+      }
+    }
+
+    return {
+      origin_id: `reddit_comment_${comment.name}`,
+      payload_text: comment.body ?? '',
+      author_name: comment.author,
+      source_url: `https://reddit.com${comment.permalink}`,
+      occurred_at: new Date(comment.created_utc * 1000),
+      origin_type: 'comment',
+      score: engagementScore,
+      origin_parent_id: parentExternalId,
+      metadata: {
+        subreddit: comment.subreddit,
+        score: comment.score,
+      },
+    };
+  }
+
+  // -------------------------------------------------------------------------
+  // Utilities
+  // -------------------------------------------------------------------------
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}