crawd 0.8.0

package/src/lib/chat/youtube/client.ts

@@ -0,0 +1,131 @@
+import { Masterchat, stringify } from '@stu43005/masterchat'
+import { BaseChatClient } from '../base'
+import { generateShortId } from '../types'
+import type { ChatPlatform, ChatMessage } from '../types'
+
+/**
+ * YouTube Live Chat client using masterchat (no API credentials needed).
+ */
+export class YouTubeChatClient extends BaseChatClient {
+  readonly platform: ChatPlatform = 'youtube'
+
+  private mc: Masterchat | null = null
+  private _connected = false
+  private videoId: string
+
+  // Deduplication: sliding window of recent IDs
+  private recentIds: string[] = []
+  private readonly maxRecentIds = 500
+
+  constructor(videoIdOrUrl: string) {
+    super()
+    this.videoId = this.extractVideoId(videoIdOrUrl)
+  }
+
+  async connect(): Promise<void> {
+    this.mc = await Masterchat.init(this.videoId)
+
+    this.mc.on('chat', (chat) => {
+      const prefixedId = `yt:${chat.id}`
+      if (this.recentIds.includes(prefixedId)) return
+      this.trackId(prefixedId)
+
+      const msg: ChatMessage = {
+        id: prefixedId,
+        shortId: generateShortId(),
+        platform: 'youtube',
+        username: chat.authorName ?? 'Anonymous',
+        message: chat.message ? stringify(chat.message) : '',
+        timestamp: chat.timestamp?.getTime() ?? Date.now(),
+        metadata: {
+          channelId: chat.authorChannelId,
+          isModerator: chat.isModerator,
+          isMember: !!chat.membership,
+          avatarUrl: chat.authorPhoto,
+        }
+      }
+
+      this.emit('message', msg)
+    })
+
+    // Handle super chats via actions event
+    this.mc.on('actions', (actions) => {
+      for (const action of actions) {
+        if (action.type !== 'addSuperChatItemAction') continue
+
+        const prefixedId = `yt:${action.id}`
+        if (this.recentIds.includes(prefixedId)) continue
+        this.trackId(prefixedId)
+
+        const msg: ChatMessage = {
+          id: prefixedId,
+          shortId: generateShortId(),
+          platform: 'youtube',
+          username: action.authorName ?? 'Anonymous',
+          message: action.message ? stringify(action.message) : '',
+          timestamp: action.timestamp?.getTime() ?? Date.now(),
+          metadata: {
+            channelId: action.authorChannelId,
+            isModerator: action.isModerator,
+            isMember: !!action.membership,
+            avatarUrl: action.authorPhoto,
+            superChat: {
+              amountDisplayString: `${action.currency}${action.amount}`,
+              backgroundColor: action.color ?? 'blue'
+            }
+          }
+        }
+
+        this.emit('message', msg)
+      }
+    })
+
+    this.mc.on('error', (err) => {
+      console.error('[YouTube] Chat error:', err)
+      this.emit('error', err)
+    })
+
+    this.mc.on('end', () => {
+      this._connected = false
+      this.emit('disconnected')
+    })
+
+    this.mc.listen()
+    this._connected = true
+    this.emit('connected')
+  }
+
+  disconnect(): void {
+    this.mc?.stop()
+    this.mc = null
+    this._connected = false
+  }
+
+  isConnected(): boolean {
+    return this._connected
+  }
+
+  /** Update video ID for new stream without recreating client */
+  setVideoId(videoIdOrUrl: string): void {
+    this.videoId = this.extractVideoId(videoIdOrUrl)
+  }
+
+  private extractVideoId(input: string): string {
+    const patterns = [
+      /(?:v=|youtu\.be\/|\/live\/)([a-zA-Z0-9_-]{11})/,
+      /^([a-zA-Z0-9_-]{11})$/
+    ]
+    for (const pattern of patterns) {
+      const match = input.match(pattern)
+      if (match) return match[1]
+    }
+    return input
+  }
+
+  private trackId(id: string): void {
+    this.recentIds.push(id)
+    if (this.recentIds.length > this.maxRecentIds) {
+      this.recentIds.shift()
+    }
+  }
+}

package/src/lib/pumpfun/live/client.ts

@@ -0,0 +1,69 @@
+import type {
+	PumpfunCurrentlyLiveResponse,
+	PumpfunJoinLivestreamResponse,
+	PumpfunLivestreamConnection,
+} from "./types";
+import { PumpfunLiveStreamSocketUrl } from "./types";
+
+export class Pumpfun {
+	public async getLivestream(
+		mint: string,
+	): Promise<PumpfunLivestreamConnection> {
+		const url = `https://livestream-api.pump.fun/livestream/join`;
+
+		try {
+			const [joinResponse, currentlyLiveData] = await Promise.all([
+				fetch(url, {
+					method: "POST",
+					headers: {
+						"Content-Type": "application/json",
+					},
+					body: JSON.stringify({
+						mintId: mint,
+						viewer: true,
+						hidden: true,
+					}),
+				}),
+				this.getLivestreams(),
+			]);
+
+			if (!joinResponse.ok) {
+				throw new Error(
+					`Failed to fetch livestreams: ${joinResponse.status} ${joinResponse.statusText}`,
+				);
+			}
+
+			const data = (await joinResponse.json()) as PumpfunJoinLivestreamResponse;
+
+			const livestreamData = currentlyLiveData.find(
+				(stream) => stream.mint === mint,
+			);
+
+			return {
+				token: data.token,
+				websocketUrl: PumpfunLiveStreamSocketUrl,
+				livestreamData,
+			};
+		} catch (error) {
+			throw error;
+		}
+	}
+
+	public async getLivestreams(): Promise<PumpfunCurrentlyLiveResponse> {
+		const url = `https://frontend-api-v3.pump.fun/coins/currently-live?offset=0&limit=1000&sort=currently_live&order=DESC&includeNsfw=true`;
+
+		try {
+			const response = await fetch(url);
+
+			if (!response.ok) {
+				throw new Error(
+					`Failed to fetch livestreams: ${response.status} ${response.statusText}`,
+				);
+			}
+
+			return (await response.json()) as PumpfunCurrentlyLiveResponse;
+		} catch (error) {
+			throw error;
+		}
+	}
+}

package/src/lib/pumpfun/live/index.ts

@@ -0,0 +1,3 @@
+import { Pumpfun } from "./client";
+
+export const pumpfun = new Pumpfun();

package/src/lib/pumpfun/live/types.ts

@@ -0,0 +1,38 @@
+export const PumpfunLiveStreamSocketUrl =
+	"wss://pump-prod-tg2x8veh.livekit.cloud/rtc";
+
+export type PumpfunLivestreamConnection = {
+	websocketUrl: string;
+	token: string;
+	livestreamData?: PumpfunLivestream;
+};
+
+export type PumpfunLivestream = {
+	mint: string;
+	name: string;
+	symbol: string;
+	description: string;
+	image_uri: string;
+	metadata_uri: string;
+	twitter: string | null;
+	telegram: string | null;
+	bonding_curve: string;
+	creator: string;
+	created_timestamp: number;
+	raydium_pool: string;
+	num_participants: number;
+	complete: boolean;
+	thumbnail: string;
+	market_id: string;
+	usd_market_cap: number;
+	is_currently_live: boolean;
+	reply_count: number;
+	market_cap: number; // usd
+};
+
+export type PumpfunCurrentlyLiveResponse = PumpfunLivestream[];
+
+export type PumpfunJoinLivestreamResponse = {
+	token: string;
+	role: string;
+};

package/src/lib/pumpfun/v2/client.ts

@@ -0,0 +1,139 @@
+import {
+    Candle,
+    CandlesOptions,
+    CreatorFee,
+    CreatorFeesOptions,
+    FollowerInfo,
+    JoinLivestreamResponse,
+    Livestream,
+    LivestreamClipsOptions,
+    LivestreamClipsResponse,
+    MarketTradesResponse,
+    TotalFees,
+    UserProfile,
+    WalletBalances,
+    WalletBalancesOptions,
+} from "./types";
+
+export class PumpFunClient {
+    private readonly frontendApiBase = "https://frontend-api-v3.pump.fun";
+    private readonly swapApiBase = "https://swap-api.pump.fun";
+    private readonly livestreamApiBase = "https://livestream-api.pump.fun";
+
+    private async fetch<T>(url: string): Promise<T> {
+        const response = await fetch(url);
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        return response.json();
+    }
+
+    async getUserProfile(username: string): Promise<UserProfile> {
+        return this.fetch<UserProfile>(`${this.frontendApiBase}/users/${username}`);
+    }
+
+    async getFollowers(walletAddress: string): Promise<FollowerInfo[]> {
+        return this.fetch<FollowerInfo[]>(`${this.frontendApiBase}/following/followers/${walletAddress}`);
+    }
+
+    async getFollowing(walletAddress: string): Promise<FollowerInfo[]> {
+        return this.fetch<FollowerInfo[]>(`${this.frontendApiBase}/following/${walletAddress}`);
+    }
+
+    async getWalletBalances(walletAddress: string, options: WalletBalancesOptions = {}): Promise<WalletBalances> {
+        const params = new URLSearchParams();
+        if (options.includePnl !== undefined) {
+            params.append("includePnl", String(options.includePnl));
+        }
+        if (options.page !== undefined) {
+            params.append("page", String(options.page));
+        }
+        if (options.limit !== undefined) {
+            params.append("limit", String(options.limit));
+        }
+        const queryString = params.toString();
+        const url = `${this.swapApiBase}/v1/wallet/${walletAddress}/balances${queryString ? `?${queryString}` : ""}`;
+        return this.fetch<WalletBalances>(url);
+    }
+
+    async getCreatorFees(walletAddress: string, options: CreatorFeesOptions = {}): Promise<CreatorFee[]> {
+        const params = new URLSearchParams();
+        if (options.interval !== undefined) {
+            params.append("interval", options.interval);
+        }
+        if (options.limit !== undefined) {
+            params.append("limit", String(options.limit));
+        }
+        const queryString = params.toString();
+        const url = `${this.swapApiBase}/v1/creators/${walletAddress}/fees${queryString ? `?${queryString}` : ""}`;
+        return this.fetch<CreatorFee[]>(url);
+    }
+
+    async getTotalFees(walletAddress: string): Promise<TotalFees> {
+        return this.fetch<TotalFees>(`${this.swapApiBase}/v1/creators/${walletAddress}/fees/total`);
+    }
+
+    async getCoin(mint: string): Promise<Livestream> {
+        return this.fetch<Livestream>(`${this.frontendApiBase}/coins-v2/${mint}`);
+    }
+
+    async getCurrentLivestreams(): Promise<Livestream[]> {
+        return this.fetch<Livestream[]>(`${this.frontendApiBase}/coins-v2/currently-live`);
+    }
+
+    async getLivestreamClips(mint: string, options: LivestreamClipsOptions = {}): Promise<LivestreamClipsResponse> {
+        const params = new URLSearchParams();
+        if (options.limit !== undefined) {
+            params.append("limit", String(options.limit));
+        }
+        if (options.clipType !== undefined) {
+            params.append("clipType", options.clipType);
+        }
+        const queryString = params.toString();
+        const url = `${this.livestreamApiBase}/clips/${mint}${queryString ? `?${queryString}` : ""}`;
+        return this.fetch<LivestreamClipsResponse>(url);
+    }
+
+    async getMarketActivity(mint: string): Promise<MarketTradesResponse> {
+        return this.fetch<MarketTradesResponse>(`${this.swapApiBase}/v2/coins/${mint}/trades`);
+    }
+
+    async getCandles(mint: string, options: CandlesOptions = {}): Promise<Candle[]> {
+        const params = new URLSearchParams();
+        if (options.interval !== undefined) {
+            params.append("interval", options.interval);
+        }
+        if (options.limit !== undefined) {
+            params.append("limit", String(options.limit));
+        }
+        if (options.currency !== undefined) {
+            params.append("currency", options.currency);
+        }
+        if (options.createdTs !== undefined) {
+            params.append("createdTs", String(options.createdTs));
+        }
+        const qs = params.toString();
+        const url = `${this.swapApiBase}/v2/coins/${mint}/candles${qs ? `?${qs}` : ""}`;
+        return this.fetch<Candle[]>(url);
+    }
+
+    async joinLivestream(mint: string): Promise<JoinLivestreamResponse> {
+        const response = await fetch("https://livestream-api.pump.fun/livestream/join", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+                mintId: mint,
+                viewer: true,
+                hidden: true,
+            }),
+        });
+
+        if (!response.ok) {
+            throw new Error(`Failed to join the livestream: ${response.status} ${response.statusText}`);
+        }
+
+        return (await response.json()) as JoinLivestreamResponse;
+    }
+}

package/src/lib/pumpfun/v2/index.ts

@@ -0,0 +1,5 @@
+import { PumpFunClient } from "./client";
+
+export * from "./types";
+
+export const pumpfun = new PumpFunClient();

package/src/lib/pumpfun/v2/socket/client.ts

@@ -0,0 +1,60 @@
+import { io, Socket } from "socket.io-client";
+import { PumpfunMessage } from "./types";
+
+export class PumpfunSocketClient {
+    private readonly socket: Socket;
+
+    constructor(authToken: string | null) {
+        this.socket = io("wss://livechat.pump.fun", {
+            path: "/socket.io/",
+            transports: ["websocket"],
+            autoConnect: false,
+            withCredentials: true,
+            extraHeaders: {
+                Host: "livechat.pump.fun",
+                Origin: "https://pump.fun",
+            },
+        });
+    }
+
+    connect(): void {
+        this.socket.connect();
+    }
+
+    disconnect(): void {
+        this.socket.disconnect();
+    }
+
+    async joinRoom(roomId: string): Promise<void> {
+        return new Promise((resolve) => {
+            this.socket.emit("joinRoom", { roomId, username: "" });
+            setTimeout(resolve, 200);
+        });
+    }
+
+    async sendMessage(roomId: string, message: string): Promise<void> {
+        this.socket.emit("sendMessage2", { roomId, message, username: "" });
+    }
+
+    async getMessagesHistory({
+        roomId,
+        limit,
+    }: {
+        roomId: string;
+        limit?: number;
+    }): Promise<PumpfunMessage[]> {
+        return new Promise((resolve) => {
+            this.socket.emit(
+                "getMessageHistory",
+                { roomId: roomId, before: null, limit: limit ?? 20 },
+                (messages: PumpfunMessage[]) => {
+                    resolve(messages);
+                },
+            );
+        });
+    }
+
+    onMessage(callback: (message: PumpfunMessage) => void): void {
+        this.socket.on("newMessage", callback);
+    }
+}

package/src/lib/pumpfun/v2/socket/index.ts

@@ -0,0 +1,6 @@
+import dotenv from "dotenv";
+import { PumpfunSocketClient } from "./client";
+
+dotenv.config();
+
+export const pumpfunSocketClient = new PumpfunSocketClient(process.env.PUMPFUN_AUTH_TOKEN!);

package/src/lib/pumpfun/v2/socket/types.ts

@@ -0,0 +1,7 @@
+export interface PumpfunMessage {
+    id: string;
+    roomId: string;
+    message: string;
+    userAddress: string;
+    username: string;
+}

package/src/lib/pumpfun/v2/types.ts

@@ -0,0 +1,234 @@
+export interface UserProfile {
+    address: string;
+    likes_received: number;
+    unread_notifs_count: number;
+    mentions_received: number;
+    username: string;
+    profile_image: string;
+    last_username_update_timestamp: number;
+    following: number;
+    followers: number;
+    bio: string;
+    x_username: string | null;
+    x_id: string | null;
+}
+
+export interface FollowerInfo {
+    username: string;
+    profile_image: string;
+    address: string;
+    timestamp: number;
+    followers: number;
+}
+
+export interface TokenCoin {
+    mint: string;
+    name: string;
+    symbol: string;
+    image_uri: string;
+    creator: string;
+    market_cap: number;
+    usd_market_cap: number;
+}
+
+export interface PnlInfo {
+    realized_pnl: {
+        percent: string;
+        usd: string;
+    };
+    unrealized_pnl: {
+        percent: string;
+        usd: string;
+    };
+    total_usd_in: string;
+    total_usd_out: string;
+    total_pnl: {
+        percent: string;
+        usd: string;
+    };
+}
+
+export interface HoldingToken {
+    mint: string;
+    balance: string;
+    price: string;
+    fetchedAt: string;
+    balanceUSD: string;
+    coin: TokenCoin;
+    pnl: PnlInfo;
+}
+
+export interface WalletBalances {
+    items: HoldingToken[];
+    total: number;
+    nextCursor: string;
+    nativeBalance: number;
+    itemCount: number;
+    totalBalanceUSD: number;
+    lastUpdated: string;
+}
+
+export interface CreatorFee {
+    bucket: string;
+    creatorFee: string;
+    creatorFeeSOL: string;
+    numTrades: number;
+    cumulativeCreatorFee: string;
+    cumulativeCreatorFeeSOL: string;
+}
+
+export interface TotalFees {
+    totalFees: string;
+    totalFeesSOL: string;
+}
+
+export interface Livestream {
+    mint: string;
+    name: string;
+    symbol: string;
+    description: string;
+    image_uri: string;
+    metadata_uri: string;
+    twitter: string | null;
+    telegram: string | null;
+    bonding_curve: string;
+    associated_bonding_curve: string;
+    creator: string;
+    created_timestamp: number;
+    raydium_pool: string | null;
+    complete: boolean;
+    virtual_sol_reserves: number;
+    virtual_token_reserves: number;
+    hidden: boolean | null;
+    total_supply: number;
+    website: string;
+    show_name: boolean;
+    last_trade_timestamp: number;
+    king_of_the_hill_timestamp: number;
+    market_cap: number;
+    nsfw: boolean;
+    market_id: string | null;
+    inverted: boolean;
+    real_sol_reserves: number;
+    real_token_reserves: number;
+    livestream_ban_expiry: number;
+    last_reply: number;
+    reply_count: number;
+    is_banned: boolean;
+    is_currently_live: boolean;
+    initialized: boolean;
+    video_uri: string | null;
+    updated_at: string | null;
+    pump_swap_pool: string;
+    ath_market_cap: number;
+    ath_market_cap_timestamp: number;
+    banner_uri: string;
+    hide_banner: boolean;
+    livestream_downrank_score: number;
+    program: string | null;
+    platform: string | null;
+    thumbnail: string;
+    thumbnail_updated_at: number;
+    num_participants: number;
+    downrank_score: number;
+    usd_market_cap: number;
+}
+
+export interface LivestreamClip {
+    roomName: string;
+    clipId: string;
+    sessionId: string;
+    startTime: string;
+    endTime: string;
+    duration: number;
+    playlistUrl: string;
+    mp4S3Key: string;
+    mp4SizeBytes: number;
+    mp4CreatedAt: string;
+    thumbnailUrl: string;
+    thumbnailS3Key: string;
+    playlistS3Key: string;
+    hidden: boolean;
+    clipType: string;
+    createdAt: string;
+}
+
+export interface LivestreamClipsResponse {
+    clips: LivestreamClip[];
+}
+
+export interface MarketActivityPeriod {
+    numTxs: number;
+    volumeUSD: number;
+    numUsers: number;
+    numBuys: number;
+    numSells: number;
+    buyVolumeUSD: number;
+    sellVolumeUSD: number;
+    numBuyers: number;
+    numSellers: number;
+    priceChangePercent: number;
+}
+
+export interface MarketActivity {
+    "5m": MarketActivityPeriod | null;
+    "1h": MarketActivityPeriod | null;
+    "6h": MarketActivityPeriod | null;
+    "24h": MarketActivityPeriod | null;
+}
+
+export interface WalletBalancesOptions {
+    includePnl?: boolean;
+    page?: number;
+    limit?: number;
+}
+
+export interface CreatorFeesOptions {
+    interval?: string;
+    limit?: number;
+}
+
+export interface LivestreamClipsOptions {
+    limit?: number;
+    clipType?: string;
+}
+
+export interface MarketTrade {
+    slotIndexId: string;
+    tx: string;
+    timestamp: string;
+    userAddress: string;
+    type: string;
+    program: string;
+    priceUsd: string;
+    priceSol: string;
+    amountUsd: string;
+    amountSol: string;
+    baseAmount: string;
+    quoteAmount: string;
+}
+
+export interface MarketTradesResponse {
+    trades: MarketTrade[];
+}
+
+export interface Candle {
+    timestamp: number;
+    open: string;
+    high: string;
+    low: string;
+    close: string;
+    volume: string;
+}
+
+export interface CandlesOptions {
+    interval?: string;
+    limit?: number;
+    currency?: string;
+    createdTs?: number;
+}
+
+export type JoinLivestreamResponse = {
+    token: string;
+    role: string;
+};