@apitap/core 1.0.0

package/src/read/decoders/grokipedia.ts

@@ -0,0 +1,246 @@
+// src/read/decoders/grokipedia.ts
+import type { Decoder, ReadResult } from '../types.js';
+import { safeFetch } from '../../discovery/fetch.js';
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Grokipedia decoder — xAI's open knowledge base (6M+ articles)
+ *
+ * API endpoints (all public, no auth):
+ *   /api/page?slug=X&includeContent=true  — Full article with citations
+ *   /api/full-text-search?query=X&limit=N — Search with relevance scoring
+ *   /api/stats                            — Site-wide stats
+ *   /api/typeahead?query=X                — Autocomplete
+ *   /api/list-pages?limit=N               — Browse articles
+ *   /api/top-contributors?limit=N         — Top editors
+ *   /api/list-edit-requests?limit=N       — Recent edits
+ */
+
+const GROKIPEDIA_API = 'https://grokipedia.com/api';
+
+export const grokipediaDecoder: Decoder = {
+  name: 'grokipedia',
+  patterns: [
+    /grokipedia\.com\/wiki\/([^#?]+)/,
+    /grokipedia\.com\/article\/([^#?]+)/,
+    /grokipedia\.com\/search\?/,
+    /grokipedia\.com\/?$/,
+    /grokipedia\.com\/?(?:\?|#|$)/,
+  ],
+
+  async decode(url: string, options: { skipSsrf?: boolean; maxBytes?: number; [key: string]: any } = {}): Promise<ReadResult | null> {
+    try {
+      const apiBase = options._apiBaseUrl || GROKIPEDIA_API;
+
+      // Search URL: /search?q=query
+      const searchMatch = url.match(/grokipedia\.com\/search\?.*q=([^&#]+)/);
+      if (searchMatch) {
+        return decodeSearch(apiBase, decodeURIComponent(searchMatch[1]), url, options);
+      }
+
+      // Article URL: /wiki/Slug or /article/Slug
+      const articleMatch = url.match(/grokipedia\.com\/(?:wiki|article)\/([^#?]+)/);
+      if (articleMatch) {
+        return decodeArticle(apiBase, articleMatch[1], url, options);
+      }
+
+      // Homepage: return stats + trending/recent
+      if (/grokipedia\.com\/?(?:\?|#|$)/.test(url)) {
+        return decodeHomepage(apiBase, url, options);
+      }
+
+      return null;
+    } catch {
+      return null;
+    }
+  },
+};
+
+async function decodeArticle(
+  apiBase: string,
+  slug: string,
+  url: string,
+  options: { skipSsrf?: boolean; maxBytes?: number; [key: string]: any },
+): Promise<ReadResult | null> {
+  const apiUrl = `${apiBase}/page?slug=${encodeURIComponent(slug)}&includeContent=true`;
+  // Grokipedia articles can be very large (743KB+ for Elon Musk) — raise body limit to 2MB
+  const result = await safeFetch(apiUrl, { skipSsrf: options.skipSsrf, maxBodySize: 2 * 1024 * 1024 });
+  if (!result || result.status !== 200) return null;
+
+  let data: any;
+  try {
+    data = JSON.parse(result.body);
+  } catch {
+    return null;
+  }
+
+  const page = data?.page;
+  if (!page) return null;
+
+  const title = page.title || decodeURIComponent(slug).replace(/_/g, ' ');
+  const content = page.content || page.description || '';
+  const citations = page.citations || [];
+  const images = page.images || [];
+  const metadata = page.metadata || {};
+  const stats = page.stats || {};
+
+  // Truncate content if maxBytes specified
+  const maxChars = options.maxBytes ? options.maxBytes : 20000;
+  const truncatedContent = content.length > maxChars
+    ? content.slice(0, maxChars) + `\n\n[Truncated — full article is ${content.length} chars. ${citations.length} citations available.]`
+    : content;
+
+  // Build citations section (top 10)
+  const topCitations = citations.slice(0, 10);
+  const citationBlock = topCitations.length > 0
+    ? '\n\n## Sources\n' + topCitations.map((c: any, i: number) =>
+        `${i + 1}. [${c.title || 'Source'}](${c.url})`
+      ).join('\n')
+    : '';
+
+  // Build stats line
+  const statsLine = stats.totalViews
+    ? `\n\nViews: ${Number(stats.totalViews).toLocaleString()} | Quality: ${stats.qualityScore || 'N/A'} | Language: ${metadata.language || 'en'}`
+    : '';
+
+  const resultImages = images.slice(0, 5).map((img: any) => ({
+    alt: img.caption || title,
+    src: img.url || '',
+  }));
+
+  const resultLinks: Array<{ text: string; href: string }> = [
+    { text: 'Full article', href: `https://grokipedia.com/wiki/${slug}` },
+  ];
+
+  // Add citation links
+  topCitations.forEach((c: any) => {
+    if (c.url) {
+      resultLinks.push({ text: c.title || 'Source', href: c.url });
+    }
+  });
+
+  return {
+    url,
+    title,
+    author: metadata.lastEditor || null,
+    description: page.description || null,
+    content: truncatedContent + citationBlock + statsLine,
+    links: resultLinks,
+    images: resultImages,
+    metadata: {
+      type: 'article',
+      publishedAt: metadata.lastModified ? new Date(metadata.lastModified * 1000).toISOString() : null,
+      source: 'grokipedia-api',
+      canonical: `https://grokipedia.com/wiki/${slug}`,
+      siteName: 'Grokipedia',
+    },
+    cost: { tokens: estimateTokens(truncatedContent + citationBlock + statsLine) },
+  };
+}
+
+async function decodeSearch(
+  apiBase: string,
+  query: string,
+  url: string,
+  options: { skipSsrf?: boolean; [key: string]: any },
+): Promise<ReadResult | null> {
+  const apiUrl = `${apiBase}/full-text-search?query=${encodeURIComponent(query)}&limit=10`;
+  const result = await safeFetch(apiUrl, { skipSsrf: options.skipSsrf });
+  if (!result || result.status !== 200) return null;
+
+  let data: any;
+  try {
+    data = JSON.parse(result.body);
+  } catch {
+    return null;
+  }
+
+  const results = data?.results || [];
+  if (results.length === 0) return null;
+
+  const content = results.map((r: any, i: number) => {
+    const views = r.viewCount ? ` (${Number(r.viewCount).toLocaleString()} views)` : '';
+    const snippet = (r.snippet || '').replace(/<\/?em>/g, '**').replace(/\n/g, ' ').trim();
+    return `${i + 1}. **[${r.title}](https://grokipedia.com/wiki/${r.slug})**${views}\n   ${snippet}`;
+  }).join('\n\n');
+
+  const links = results.map((r: any) => ({
+    text: r.title || r.slug,
+    href: `https://grokipedia.com/wiki/${r.slug}`,
+  }));
+
+  return {
+    url,
+    title: `Grokipedia search: "${query}"`,
+    author: null,
+    description: `${results.length} results for "${query}"`,
+    content,
+    links,
+    images: [],
+    metadata: {
+      type: 'search-results',
+      publishedAt: null,
+      source: 'grokipedia-api',
+      canonical: null,
+      siteName: 'Grokipedia',
+    },
+    cost: { tokens: estimateTokens(content) },
+  };
+}
+
+async function decodeHomepage(
+  apiBase: string,
+  url: string,
+  options: { skipSsrf?: boolean; [key: string]: any },
+): Promise<ReadResult | null> {
+  // Fetch stats
+  const statsResult = await safeFetch(`${apiBase}/stats`, { skipSsrf: options.skipSsrf });
+  let statsData: any = {};
+  if (statsResult?.status === 200) {
+    try { statsData = JSON.parse(statsResult.body); } catch {}
+  }
+
+  // Fetch recent edits
+  const editsResult = await safeFetch(`${apiBase}/list-edit-requests?limit=5`, { skipSsrf: options.skipSsrf });
+  let editsData: any = {};
+  if (editsResult?.status === 200) {
+    try { editsData = JSON.parse(editsResult.body); } catch {}
+  }
+
+  const totalPages = Number(statsData.totalPages || 0).toLocaleString();
+  const indexGB = (Number(statsData.indexSizeBytes || 0) / (1024 ** 3)).toFixed(1);
+
+  let content = `# Grokipedia\n\nAn open source, comprehensive collection of all knowledge.\n\n`;
+  content += `**${totalPages} articles** | **${indexGB} GB index**\n\n`;
+
+  const edits = editsData.editRequests || [];
+  if (edits.length > 0) {
+    content += `## Recent Activity\n`;
+    for (const edit of edits) {
+      const article = edit.slug?.replace(/_/g, ' ') || 'Unknown';
+      const editor = edit.userId || 'Anonymous';
+      content += `- **${article}** — edited by ${editor} (${edit.type?.replace('EDIT_REQUEST_TYPE_', '').toLowerCase().replace(/_/g, ' ')})\n`;
+    }
+  }
+
+  return {
+    url,
+    title: 'Grokipedia',
+    author: null,
+    description: `Open knowledge base with ${totalPages} articles`,
+    content,
+    links: [],
+    images: [],
+    metadata: {
+      type: 'website',
+      publishedAt: null,
+      source: 'grokipedia-api',
+      canonical: 'https://grokipedia.com',
+      siteName: 'Grokipedia',
+    },
+    cost: { tokens: estimateTokens(content) },
+  };
+}

package/src/read/decoders/hackernews.ts

@@ -0,0 +1,198 @@
+// src/read/decoders/hackernews.ts
+import type { Decoder, ReadResult } from '../types.js';
+import { safeFetch } from '../../discovery/fetch.js';
+
+const DEFAULT_API_BASE = 'https://hacker-news.firebaseio.com';
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+export const hackernewsDecoder: Decoder = {
+  name: 'hackernews',
+  patterns: [
+    /news\.ycombinator\.com\/item\?id=\d+/,
+    /news\.ycombinator\.com\/?(?:\?|$)/,
+  ],
+
+  async decode(url: string, options: { skipSsrf?: boolean; [key: string]: any } = {}): Promise<ReadResult | null> {
+    try {
+      const apiBase = options._apiBaseUrl || DEFAULT_API_BASE;
+      const fetchOpts = { skipSsrf: options.skipSsrf };
+
+      // Check if this is an item page or front page
+      const itemMatch = url.match(/item\?id=(\d+)/);
+
+      if (itemMatch) {
+        return decodeItem(url, itemMatch[1], apiBase, fetchOpts);
+      }
+
+      return decodeFrontPage(url, apiBase, fetchOpts);
+    } catch {
+      return null;
+    }
+  },
+};
+
+async function decodeItem(
+  url: string,
+  id: string,
+  apiBase: string,
+  fetchOpts: { skipSsrf?: boolean },
+): Promise<ReadResult | null> {
+  try {
+    const result = await safeFetch(`${apiBase}/v0/item/${id}.json`, fetchOpts);
+    if (!result || result.status !== 200) return null;
+
+    let item: any;
+    try {
+      item = JSON.parse(result.body);
+    } catch {
+      return null;
+    }
+
+    if (!item) return null;
+
+    const title = item.title || null;
+    const author = item.by || null;
+    const score = item.score ?? 0;
+    const itemUrl = item.url || null;
+    const text = item.text || '';
+
+    // Fetch top 10 comments
+    const kids = item.kids || [];
+    const commentIds = kids.slice(0, 10);
+    const comments = await fetchComments(commentIds, apiBase, fetchOpts);
+
+    const commentText = comments
+      .map((c: any) => `${c.by || '[deleted]'}: ${c.text || '[deleted]'}`)
+      .join('\n\n');
+
+    const contentParts: string[] = [];
+    if (text) contentParts.push(text);
+    contentParts.push(`Score: ${score} | ${kids.length} comments`);
+    if (commentText) contentParts.push(`---\n${commentText}`);
+    const content = contentParts.join('\n\n');
+
+    const links: Array<{ text: string; href: string }> = [];
+    if (itemUrl) {
+      links.push({ text: title || 'Link', href: itemUrl });
+    }
+
+    return {
+      url,
+      title,
+      author,
+      description: `HN ${item.type || 'story'} by ${author} (${score} points)`,
+      content,
+      links,
+      images: [],
+      metadata: {
+        type: item.type || 'story',
+        publishedAt: item.time ? new Date(item.time * 1000).toISOString() : null,
+        source: 'hackernews-firebase',
+        canonical: `https://news.ycombinator.com/item?id=${id}`,
+        siteName: 'Hacker News',
+      },
+      cost: { tokens: estimateTokens(content) },
+    };
+  } catch {
+    return null;
+  }
+}
+
+async function decodeFrontPage(
+  url: string,
+  apiBase: string,
+  fetchOpts: { skipSsrf?: boolean },
+): Promise<ReadResult | null> {
+  try {
+    const result = await safeFetch(`${apiBase}/v0/topstories.json`, fetchOpts);
+    if (!result || result.status !== 200) return null;
+
+    let storyIds: number[];
+    try {
+      storyIds = JSON.parse(result.body);
+    } catch {
+      return null;
+    }
+
+    if (!Array.isArray(storyIds)) return null;
+
+    // Fetch first 10 stories
+    const topIds = storyIds.slice(0, 10);
+    const stories = await fetchStories(topIds, apiBase, fetchOpts);
+
+    const content = stories
+      .map((s: any, i: number) => `${i + 1}. ${s.title || '[untitled]'} (${s.score ?? 0} pts, ${(s.descendants ?? 0)} comments) by ${s.by || '[deleted]'}`)
+      .join('\n');
+
+    const links = stories
+      .filter((s: any) => s.url)
+      .map((s: any) => ({ text: s.title || 'Link', href: s.url }));
+
+    return {
+      url,
+      title: 'Hacker News — Top Stories',
+      author: null,
+      description: `Top ${stories.length} stories`,
+      content,
+      links,
+      images: [],
+      metadata: {
+        type: 'listing',
+        publishedAt: null,
+        source: 'hackernews-firebase',
+        canonical: 'https://news.ycombinator.com/',
+        siteName: 'Hacker News',
+      },
+      cost: { tokens: estimateTokens(content) },
+    };
+  } catch {
+    return null;
+  }
+}
+
+async function fetchComments(
+  ids: number[],
+  apiBase: string,
+  fetchOpts: { skipSsrf?: boolean },
+): Promise<any[]> {
+  const comments: any[] = [];
+  for (const id of ids) {
+    try {
+      const result = await safeFetch(`${apiBase}/v0/item/${id}.json`, fetchOpts);
+      if (result && result.status === 200) {
+        const comment = JSON.parse(result.body);
+        if (comment && !comment.deleted) {
+          comments.push(comment);
+        }
+      }
+    } catch {
+      // skip failed comments
+    }
+  }
+  return comments;
+}
+
+async function fetchStories(
+  ids: number[],
+  apiBase: string,
+  fetchOpts: { skipSsrf?: boolean },
+): Promise<any[]> {
+  const stories: any[] = [];
+  for (const id of ids) {
+    try {
+      const result = await safeFetch(`${apiBase}/v0/item/${id}.json`, fetchOpts);
+      if (result && result.status === 200) {
+        const story = JSON.parse(result.body);
+        if (story) {
+          stories.push(story);
+        }
+      }
+    } catch {
+      // skip failed stories
+    }
+  }
+  return stories;
+}

package/src/read/decoders/index.ts

@@ -0,0 +1,15 @@
+// src/read/decoders/index.ts
+import type { Decoder } from '../types.js';
+import { redditDecoder } from './reddit.js';
+import { youtubeDecoder } from './youtube.js';
+import { wikipediaDecoder } from './wikipedia.js';
+import { hackernewsDecoder } from './hackernews.js';
+import { grokipediaDecoder } from './grokipedia.js';
+import { twitterDecoder } from './twitter.js';
+import { deepwikiDecoder } from './deepwiki.js';
+
+const decoders: Decoder[] = [redditDecoder, youtubeDecoder, wikipediaDecoder, hackernewsDecoder, grokipediaDecoder, twitterDecoder, deepwikiDecoder];
+
+export function findDecoder(url: string): Decoder | null {
+  return decoders.find(d => d.patterns.some(p => p.test(url))) ?? null;
+}

package/src/read/decoders/reddit.ts

@@ -0,0 +1,158 @@
+// src/read/decoders/reddit.ts
+import type { Decoder, ReadResult } from '../types.js';
+import { safeFetch } from '../../discovery/fetch.js';
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+export const redditDecoder: Decoder = {
+  name: 'reddit',
+  patterns: [
+    /reddit\.com\/r\/[^/]+\/comments\//,
+    /reddit\.com\/r\/[^/]+\/?$/,
+    /reddit\.com\/r\/[^/]+\/?(?:\?|$)/,
+    /reddit\.com\/user\/[^/]+/,
+  ],
+
+  async decode(url: string, options: { skipSsrf?: boolean; [key: string]: any } = {}): Promise<ReadResult | null> {
+    try {
+      // Append .json to the URL to get JSON response
+      const jsonUrl = url.replace(/\/?(\?|$)/, '.json$1');
+
+      const result = await safeFetch(jsonUrl, { skipSsrf: options.skipSsrf });
+      if (!result || result.status !== 200) return null;
+
+      let data: any;
+      try {
+        data = JSON.parse(result.body);
+      } catch {
+        return null;
+      }
+
+      // Post page: response is an array [post, comments]
+      if (Array.isArray(data) && data.length >= 1) {
+        return decodePostPage(url, data);
+      }
+
+      // Subreddit/user listing: response has data.children
+      if (data && data.data && Array.isArray(data.data.children)) {
+        return decodeListingPage(url, data);
+      }
+
+      return null;
+    } catch {
+      return null;
+    }
+  },
+};
+
+function decodePostPage(url: string, data: any[]): ReadResult | null {
+  try {
+    const postData = data[0]?.data?.children?.[0]?.data;
+    if (!postData) return null;
+
+    const title = postData.title || null;
+    const author = postData.author || null;
+    const selftext = postData.selftext || '';
+    const score = postData.score ?? 0;
+    const subreddit = postData.subreddit || '';
+
+    // Extract comments
+    const commentChildren = data[1]?.data?.children || [];
+    const comments = commentChildren
+      .filter((c: any) => c.kind === 't1' && c.data)
+      .slice(0, 25)
+      .map((c: any) => ({
+        author: c.data.author || '[deleted]',
+        body: c.data.body || '',
+        score: c.data.score ?? 0,
+      }));
+
+    const commentText = comments
+      .map((c: any) => `${c.author} (${c.score} pts): ${c.body}`)
+      .join('\n\n');
+
+    const content = selftext
+      ? `${selftext}\n\n---\nScore: ${score} | ${comments.length} comments\n\n${commentText}`
+      : `Score: ${score} | ${comments.length} comments\n\n${commentText}`;
+
+    const links: Array<{ text: string; href: string }> = [];
+    if (postData.url && postData.url !== postData.permalink) {
+      links.push({ text: 'Link', href: postData.url });
+    }
+
+    return {
+      url,
+      title,
+      author,
+      description: `r/${subreddit} post by u/${author} (${score} points)`,
+      content,
+      links,
+      images: [],
+      metadata: {
+        type: 'discussion',
+        publishedAt: postData.created_utc ? new Date(postData.created_utc * 1000).toISOString() : null,
+        source: 'reddit-json',
+        canonical: postData.permalink ? `https://www.reddit.com${postData.permalink}` : null,
+        siteName: 'Reddit',
+      },
+      cost: { tokens: estimateTokens(content) },
+    };
+  } catch {
+    return null;
+  }
+}
+
+function decodeListingPage(url: string, data: any): ReadResult | null {
+  try {
+    const children = data.data.children || [];
+    const posts = children
+      .filter((c: any) => c.data)
+      .slice(0, 25)
+      .map((c: any) => ({
+        title: c.data.title || c.data.link_title || '',
+        author: c.data.author || '[deleted]',
+        score: c.data.score ?? 0,
+        numComments: c.data.num_comments ?? 0,
+        permalink: c.data.permalink || '',
+        subreddit: c.data.subreddit || '',
+      }));
+
+    const content = posts
+      .map((p: any, i: number) => `${i + 1}. ${p.title} (${p.score} pts, ${p.numComments} comments) by u/${p.author}`)
+      .join('\n');
+
+    const links = posts
+      .filter((p: any) => p.permalink)
+      .map((p: any) => ({ text: p.title, href: `https://www.reddit.com${p.permalink}` }));
+
+    // Try to determine subreddit name from URL
+    const subMatch = url.match(/\/r\/([^/]+)/);
+    const subreddit = subMatch ? subMatch[1] : null;
+    const userMatch = url.match(/\/user\/([^/]+)/);
+    const user = userMatch ? userMatch[1] : null;
+
+    const title = subreddit ? `r/${subreddit}` : user ? `u/${user}` : 'Reddit listing';
+
+    return {
+      url,
+      title,
+      author: null,
+      description: `${posts.length} posts`,
+      content,
+      links,
+      images: [],
+      metadata: {
+        type: 'listing',
+        publishedAt: null,
+        source: 'reddit-json',
+        canonical: null,
+        siteName: 'Reddit',
+      },
+      cost: { tokens: estimateTokens(content) },
+    };
+  } catch {
+    return null;
+  }
+}