awess 0.1.0 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "awess",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "MCP server for searching curated awesome lists and their contents",
   "type": "module",
   "main": "src/index.ts",
@@ -34,7 +34,6 @@
     "@modelcontextprotocol/sdk": "^1.0.0",
     "mdast-util-from-markdown": "^2.0.2",
     "minisearch": "^7.2.0",
-    "octokit": "^5.0.5",
     "zod": "^4.3.5"
   }
 }

package/src/diff.ts

@@ -0,0 +1,42 @@
+import type { Item, DiffResult } from "./types";
+
+export function diffItems(oldItems: Item[], newItems: Item[]): DiffResult {
+  const oldByUrl = new Map(oldItems.map(i => [i.url, i]));
+  const newByUrl = new Map(newItems.map(i => [i.url, i]));
+
+  const added: Item[] = [];
+  const removed: Item[] = [];
+  const unchanged: Item[] = [];
+  const updated: Item[] = [];
+
+  // Check new items against old
+  for (const newItem of newItems) {
+    const oldItem = oldByUrl.get(newItem.url);
+    if (!oldItem) {
+      added.push(newItem);
+    } else if (oldItem.name === newItem.name && oldItem.description === newItem.description) {
+      // Unchanged - preserve enrichment data from old item
+      unchanged.push({
+        ...newItem,
+        github: oldItem.github,
+        lastEnriched: oldItem.lastEnriched,
+      });
+    } else {
+      // Updated - preserve enrichment data, use new metadata
+      updated.push({
+        ...newItem,
+        github: oldItem.github,
+        lastEnriched: oldItem.lastEnriched,
+      });
+    }
+  }
+
+  // Find removed items
+  for (const oldItem of oldItems) {
+    if (!newByUrl.has(oldItem.url)) {
+      removed.push(oldItem);
+    }
+  }
+
+  return { added, removed, unchanged, updated };
+}

package/src/enricher.ts

@@ -1,10 +1,29 @@
 // src/enricher.ts
-import { Octokit } from "octokit";
 import type { Item } from "./types";
 
 export function extractGitHubRepo(url: string): string | null {
   const match = url.match(/github\.com\/([^\/]+\/[^\/]+)/);
-  return match ? match[1].replace(/\.git$/, "") : null;
+  if (!match) return null;
+
+  // Clean up repo name: remove .git, #readme, query params, etc.
+  let repo = match[1]
+    .replace(/\.git$/, "")
+    .replace(/#.*$/, "")
+    .replace(/\?.*$/, "");
+
+  // Skip non-repo paths like "topics/awesome", "sponsors/foo"
+  const invalidPrefixes = ["topics", "sponsors", "orgs", "settings", "marketplace"];
+  if (invalidPrefixes.some(p => repo.startsWith(p + "/"))) {
+    return null;
+  }
+
+  return repo;
+}
+
+// Sleep helper with jitter
+function sleep(ms: number, jitter = 0.2): Promise<void> {
+  const jitterMs = ms * jitter * (Math.random() - 0.5) * 2;
+  return new Promise(resolve => setTimeout(resolve, ms + jitterMs));
 }
 
 export async function batchEnrichItems(items: Item[]): Promise<Item[]> {
@@ -14,8 +33,6 @@ export async function batchEnrichItems(items: Item[]): Promise<Item[]> {
     return items;
   }
 
-  const octokit = new Octokit({ auth: token });
-
   // Extract unique GitHub repos
   const repoMap = new Map<string, Item[]>();
   for (const item of items) {
@@ -29,13 +46,26 @@ export async function batchEnrichItems(items: Item[]): Promise<Item[]> {
   const repos = Array.from(repoMap.keys());
   console.log(`Enriching ${repos.length} unique repos...`);
 
-  // Batch query using GraphQL (100 at a time)
-  const batchSize = 100;
+  // Tuned for GitHub's secondary rate limits
+  const batchSize = 50;        // Smaller batches = lower query cost
+  const baseDelayMs = 500;     // Base delay between batches
+  let currentDelay = baseDelayMs;
+  let consecutiveErrors = 0;
+
   for (let i = 0; i < repos.length; i += batchSize) {
     const batch = repos.slice(i, i + batchSize);
+    const batchNum = Math.floor(i / batchSize) + 1;
+    const totalBatches = Math.ceil(repos.length / batchSize);
+
+    // Add delay between batches (except first)
+    if (i > 0) {
+      await sleep(currentDelay);
+    }
 
+    // Include rateLimit in query to monitor usage
     const query = `
       query {
+        rateLimit { cost remaining resetAt }
         ${batch.map((repo, idx) => {
           const [owner, name] = repo.split("/");
           return `repo${idx}: repository(owner: "${owner}", name: "${name}") {
@@ -48,8 +78,55 @@ export async function batchEnrichItems(items: Item[]): Promise<Item[]> {
     `;
 
     try {
-      const result: any = await octokit.graphql(query);
+      // Use fetch instead of octokit.graphql to handle partial results
+      const response = await fetch("https://api.github.com/graphql", {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${token}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ query }),
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}`);
+      }
+
+      const json: any = await response.json();
+
+      // Check for complete failure (errors but no data)
+      if (json.errors && !json.data) {
+        throw new Error(json.errors[0]?.message || "GraphQL query failed");
+      }
+
+      // Log any partial errors (repos that don't exist, etc.)
+      if (json.errors) {
+        const failedRepos = json.errors
+          .filter((e: any) => e.path?.[0]?.startsWith("repo"))
+          .map((e: any) => batch[parseInt(e.path[0].slice(4))])
+          .filter(Boolean);
+        if (failedRepos.length > 0) {
+          console.log(`  [${batchNum}/${totalBatches}] ${failedRepos.length} repos not found`);
+        }
+      }
+
+      const result = json.data;
+
+      // Log rate limit status periodically
+      const rl = result.rateLimit;
+      if (batchNum % 10 === 0 || rl?.remaining < 100) {
+        console.log(`  [${batchNum}/${totalBatches}] Rate limit: ${rl?.remaining} remaining, cost: ${rl?.cost}`);
+      } else {
+        process.stdout.write(`  [${batchNum}/${totalBatches}]\r`);
+      }
 
+      // If running low on points, slow down
+      if (rl?.remaining < 500) {
+        currentDelay = Math.min(currentDelay * 1.5, 5000);
+        console.log(`  ⚠️ Low rate limit (${rl.remaining}), increasing delay to ${currentDelay}ms`);
+      }
+
+      // Process results - now handles partial data correctly
       for (let j = 0; j < batch.length; j++) {
         const data = result[`repo${j}`];
         if (data) {
@@ -64,10 +141,115 @@ export async function batchEnrichItems(items: Item[]): Promise<Item[]> {
           }
         }
       }
+
+      // Reset on success
+      consecutiveErrors = 0;
+      currentDelay = Math.max(currentDelay * 0.9, baseDelayMs); // Gradually speed up
+
     } catch (error: any) {
-      console.error(`Error enriching batch ${i}-${i + batchSize}:`, error.message);
+      consecutiveErrors++;
+
+      // Check for secondary rate limit
+      if (error.message?.includes("SecondaryRateLimit") || error.message?.includes("403")) {
+        // Exponential backoff with jitter
+        const backoffMs = Math.min(baseDelayMs * Math.pow(2, consecutiveErrors), 60000);
+        console.log(`  ⚠️ Secondary rate limit hit, backing off for ${backoffMs}ms...`);
+        await sleep(backoffMs, 0.3);
+        currentDelay = backoffMs; // Keep the higher delay
+        i -= batchSize; // Retry this batch
+        continue;
+      }
+
+      // Log other errors but continue
+      console.error(`  Error batch ${batchNum}: ${error.message?.slice(0, 100)}`);
+
+      // If too many consecutive errors, slow down
+      if (consecutiveErrors >= 3) {
+        currentDelay = Math.min(currentDelay * 2, 10000);
+        console.log(`  Multiple errors, slowing to ${currentDelay}ms`);
+      }
     }
   }
 
+  console.log(""); // Clear the \r line
   return items;
 }
+
+export async function batchQueryListRepos(
+  repos: string[]
+): Promise<Map<string, { pushedAt: string } | null>> {
+  const results = new Map<string, { pushedAt: string } | null>();
+
+  const token = process.env.GITHUB_TOKEN;
+  if (!token) {
+    console.warn("No GITHUB_TOKEN - skipping list repo query");
+    return results;
+  }
+
+  const BATCH_SIZE = 50;
+
+  for (let i = 0; i < repos.length; i += BATCH_SIZE) {
+    const batch = repos.slice(i, i + BATCH_SIZE);
+    const batchNum = Math.floor(i / BATCH_SIZE) + 1;
+    const totalBatches = Math.ceil(repos.length / BATCH_SIZE);
+    process.stdout.write(`  [${batchNum}/${totalBatches}]\r`);
+
+    // Build GraphQL query for this batch
+    const repoQueries = batch.map((repo, idx) => {
+      const [owner, name] = repo.split("/");
+      return `repo${idx}: repository(owner: "${owner}", name: "${name}") {
+        pushedAt
+      }`;
+    });
+
+    const query = `query { ${repoQueries.join("\n")} }`;
+
+    try {
+      const response = await fetch("https://api.github.com/graphql", {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${token}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ query }),
+      });
+
+      if (!response.ok) {
+        console.error(`  Error batch ${batchNum}: HTTP ${response.status}`);
+        for (const repo of batch) {
+          results.set(repo, null);
+        }
+        continue;
+      }
+
+      const json = await response.json();
+
+      if (json.errors && !json.data) {
+        console.error(`  Error batch ${batchNum}:`, json.errors[0]?.message);
+        // Mark all repos in batch as null
+        for (const repo of batch) {
+          results.set(repo, null);
+        }
+        continue;
+      }
+
+      // Extract results
+      batch.forEach((repo, idx) => {
+        const data = json.data?.[`repo${idx}`];
+        if (data?.pushedAt) {
+          results.set(repo, { pushedAt: data.pushedAt });
+        } else {
+          results.set(repo, null);
+        }
+      });
+    } catch (error: any) {
+      console.error(`  Error batch ${batchNum}:`, error.message);
+      for (const repo of batch) {
+        results.set(repo, null);
+      }
+    }
+  }
+
+  console.log(); // newline after progress
+  return results;
+}

package/src/index.ts

@@ -14,14 +14,25 @@ interface AwesomeList {
   source: string;
 }
 
-// Data URL - GitHub raw (no file size limits, 5 min cache)
-const DATA_URL = "https://raw.githubusercontent.com/arimxyer/ass/main/data";
+// CDN URLs - jsDelivr primary (faster, global CDN), GitHub raw fallback
+const JSDELIVR_URL = "https://cdn.jsdelivr.net/gh/arimxyer/ass@main/data";
+const GITHUB_RAW_URL = "https://raw.githubusercontent.com/arimxyer/ass/main/data";
 
-// Load data from GitHub, fallback to local for development
+// Load data from CDN, fallback to local for development
 async function loadData<T>(filename: string): Promise<T> {
-  // Try remote first
+  // Try jsDelivr first (faster global CDN)
   try {
-    const res = await fetch(`${DATA_URL}/${filename}`);
+    const res = await fetch(`${JSDELIVR_URL}/${filename}`);
+    if (res.ok) {
+      return res.json();
+    }
+  } catch {
+    // jsDelivr failed, try GitHub raw
+  }
+
+  // Try GitHub raw as fallback
+  try {
+    const res = await fetch(`${GITHUB_RAW_URL}/${filename}`);
     if (res.ok) {
       return res.json();
     }
@@ -34,6 +45,39 @@ async function loadData<T>(filename: string): Promise<T> {
   return Bun.file(localPath).json();
 }
 
+// Load gzipped data from CDN, fallback to local
+async function loadGzippedData<T>(filename: string): Promise<T> {
+  // Try jsDelivr first
+  try {
+    const res = await fetch(`${JSDELIVR_URL}/${filename}`);
+    if (res.ok) {
+      const compressed = new Uint8Array(await res.arrayBuffer());
+      const decompressed = Bun.gunzipSync(compressed);
+      return JSON.parse(new TextDecoder().decode(decompressed));
+    }
+  } catch {
+    // jsDelivr failed, try GitHub raw
+  }
+
+  // Try GitHub raw as fallback
+  try {
+    const res = await fetch(`${GITHUB_RAW_URL}/${filename}`);
+    if (res.ok) {
+      const compressed = new Uint8Array(await res.arrayBuffer());
+      const decompressed = Bun.gunzipSync(compressed);
+      return JSON.parse(new TextDecoder().decode(decompressed));
+    }
+  } catch {
+    // Remote failed, try local
+  }
+
+  // Fallback to local file
+  const localPath = new URL(`../data/${filename}`, import.meta.url);
+  const compressed = new Uint8Array(await Bun.file(localPath).arrayBuffer());
+  const decompressed = Bun.gunzipSync(compressed);
+  return JSON.parse(new TextDecoder().decode(decompressed));
+}
+
 // Load curated data
 const lists: AwesomeList[] = await loadData("lists.json");
 
@@ -225,7 +269,7 @@ server.tool(
 let itemsIndex: ItemsIndex | null = null;
 
 try {
-  itemsIndex = await loadData<ItemsIndex>("items.json");
+  itemsIndex = await loadGzippedData<ItemsIndex>("items.json.gz");
   console.error(`Loaded ${itemsIndex?.itemCount} items from ${itemsIndex?.listCount} lists`);
 } catch {
   console.error("No items.json found - get_items will be unavailable");

package/src/types.ts

@@ -26,3 +26,10 @@ export interface ItemsIndex {
   itemCount: number;
   lists: Record<string, ListEntry>;
 }
+
+export interface DiffResult {
+  added: Item[];
+  removed: Item[];
+  unchanged: Item[];
+  updated: Item[];
+}