@rmdes/indiekit-endpoint-github 1.1.0 → 1.2.0

package/lib/controllers/starred.js

@@ -18,7 +18,7 @@ import {
 export const starredController = {
   /**
    * GET /api/starred/all — Public JSON API
-   * Returns all cached starred repos for Eleventy build
+   * Returns all cached starred repos with list metadata
    */
   async all(request, response, next) {
     try {
@@ -26,7 +26,7 @@ export const starredController = {
       const db = application.getGithubDb();
 
       if (!db) {
-        return response.json({ stars: [], totalCount: 0, lastSync: null });
+        return response.json({ stars: [], totalCount: 0, lastSync: null, listMeta: [] });
       }
 
       const collection = db.collection("github_stars");
@@ -40,6 +40,7 @@ export const starredController = {
         stars,
         totalCount,
         lastSync: syncState.lastIncrementalSync || syncState.lastFullSync || null,
+        listMeta: syncState.lists || [],
       });
     } catch (error) {
       next(error);

package/lib/github-graphql.js

@@ -36,6 +36,49 @@ const STARRED_QUERY = `
   }
 `;
 
+const LISTS_QUERY = `
+  query($cursor: String) {
+    viewer {
+      lists(first: 100, after: $cursor) {
+        totalCount
+        nodes {
+          name
+          slug
+          description
+          isPrivate
+          items(first: 100) {
+            totalCount
+            nodes {
+              ... on Repository {
+                nameWithOwner
+              }
+            }
+            pageInfo { endCursor hasNextPage }
+          }
+        }
+        pageInfo { endCursor hasNextPage }
+      }
+    }
+  }
+`;
+
+const LIST_ITEMS_QUERY = `
+  query($slug: String!, $cursor: String) {
+    viewer {
+      list(slug: $slug) {
+        items(first: 100, after: $cursor) {
+          nodes {
+            ... on Repository {
+              nameWithOwner
+            }
+          }
+          pageInfo { endCursor hasNextPage }
+        }
+      }
+    }
+  }
+`;
+
 /**
  * Format a single starred repo edge from GraphQL response
  * @param {object} edge - GraphQL edge with starredAt + node
@@ -129,3 +172,117 @@ export async function fetchAllStarred(token, options = {}) {
 
   return { stars: allStars, totalCount };
 }
+
+/**
+ * Fetch all GitHub Lists and their repository memberships
+ * @param {string} token - GitHub personal access token (REQUIRED)
+ * @returns {Promise<Array<{name: string, slug: string, description: string, repoFullNames: string[]}>>}
+ */
+export async function fetchAllLists(token) {
+  if (!token) {
+    throw new Error("GitHub token is required for GraphQL API");
+  }
+
+  const allLists = [];
+  let cursor = null;
+  let hasNextPage = true;
+
+  // Fetch all lists (outer pagination)
+  while (hasNextPage) {
+    const response = await fetch(GITHUB_GRAPHQL_URL, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        query: LISTS_QUERY,
+        variables: { cursor },
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`GraphQL request failed: ${response.status} ${response.statusText}`);
+    }
+
+    const body = await response.json();
+
+    if (body.errors) {
+      throw new Error(`GraphQL errors: ${body.errors.map((e) => e.message).join(", ")}`);
+    }
+
+    const listsData = body.data.viewer.lists;
+
+    for (const node of listsData.nodes) {
+      // Skip private lists
+      if (node.isPrivate) continue;
+
+      const repoFullNames = (node.items.nodes || [])
+        .map((n) => n.nameWithOwner)
+        .filter(Boolean);
+
+      const list = {
+        name: node.name,
+        slug: node.slug,
+        description: node.description || "",
+        repoFullNames,
+      };
+
+      // If this list has more items than the first page, paginate them
+      if (node.items.pageInfo.hasNextPage) {
+        let itemsCursor = node.items.pageInfo.endCursor;
+        let itemsHasNext = true;
+
+        while (itemsHasNext) {
+          await new Promise((resolve) => setTimeout(resolve, 200));
+
+          const itemsResponse = await fetch(GITHUB_GRAPHQL_URL, {
+            method: "POST",
+            headers: {
+              Authorization: `Bearer ${token}`,
+              "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+              query: LIST_ITEMS_QUERY,
+              variables: { slug: node.slug, cursor: itemsCursor },
+            }),
+          });
+
+          if (!itemsResponse.ok) {
+            console.error(`[GitHub Lists] Failed to paginate items for list "${node.name}": ${itemsResponse.status}`);
+            break;
+          }
+
+          const itemsBody = await itemsResponse.json();
+
+          if (itemsBody.errors) {
+            console.error(`[GitHub Lists] GraphQL errors for list "${node.name}": ${itemsBody.errors.map((e) => e.message).join(", ")}`);
+            break;
+          }
+
+          const itemsData = itemsBody.data.viewer.list.items;
+          for (const itemNode of itemsData.nodes) {
+            if (itemNode.nameWithOwner) {
+              list.repoFullNames.push(itemNode.nameWithOwner);
+            }
+          }
+
+          itemsCursor = itemsData.pageInfo.endCursor;
+          itemsHasNext = itemsData.pageInfo.hasNextPage;
+        }
+      }
+
+      allLists.push(list);
+    }
+
+    cursor = listsData.pageInfo.endCursor;
+    hasNextPage = listsData.pageInfo.hasNextPage;
+
+    if (hasNextPage) {
+      await new Promise((resolve) => setTimeout(resolve, 200));
+    }
+  }
+
+  console.log(`[GitHub Lists] Fetched ${allLists.length} public lists`);
+  return allLists;
+}

package/lib/starred-cache.js

@@ -12,6 +12,7 @@ export async function ensureIndexes(collection) {
   await collection.createIndex({ fullName: 1 }, { unique: true });
   await collection.createIndex({ starredAt: -1 });
   await collection.createIndex({ language: 1 });
+  await collection.createIndex({ lists: 1 });
 }
 
 /**
@@ -65,6 +66,48 @@ export async function removeUnstarred(collection, currentFullNames) {
   return result.deletedCount;
 }
 
+/**
+ * Annotate starred repos with their GitHub List memberships
+ * Builds a fullName→slugs map in memory, then uses ordered bulkWrite
+ * to minimize the race window where concurrent reads see empty lists
+ * @param {import("mongodb").Db} db
+ * @param {Array<{slug: string, repoFullNames: string[]}>} lists
+ * @returns {Promise<{listsProcessed: number, reposAnnotated: number}>}
+ */
+export async function annotateWithLists(db, lists) {
+  const collection = db.collection("github_stars");
+
+  // Build fullName → slugs map in memory
+  const repoToLists = new Map();
+  for (const list of lists) {
+    for (const fullName of list.repoFullNames) {
+      if (!repoToLists.has(fullName)) repoToLists.set(fullName, []);
+      repoToLists.get(fullName).push(list.slug);
+    }
+  }
+
+  // Single ordered bulkWrite: reset all, then set per-repo lists
+  const ops = [
+    { updateMany: { filter: {}, update: { $set: { lists: [] } } } },
+    ...[...repoToLists.entries()].map(([fullName, slugs]) => ({
+      updateOne: {
+        filter: { fullName },
+        update: { $set: { lists: slugs } },
+      },
+    })),
+  ];
+
+  // Process in batches of 1000 to avoid MongoDB limits
+  let reposAnnotated = 0;
+  for (let i = 0; i < ops.length; i += 1000) {
+    const batch = ops.slice(i, i + 1000);
+    const result = await collection.bulkWrite(batch, { ordered: true });
+    reposAnnotated += result.modifiedCount;
+  }
+
+  return { listsProcessed: lists.length, reposAnnotated };
+}
+
 /**
  * Get all starred repos sorted by starredAt descending
  * @param {import("mongodb").Collection} collection

package/lib/starred-sync.js

@@ -4,11 +4,12 @@
  * @module starred-sync
  */
 
-import { fetchAllStarred } from "./github-graphql.js";
+import { fetchAllStarred, fetchAllLists } from "./github-graphql.js";
 import {
   ensureIndexes,
   syncStars,
   removeUnstarred,
+  annotateWithLists,
   getStarCount,
   getSyncState,
   updateSyncState,
@@ -35,6 +36,37 @@ export function getStarredSyncStatus() {
   return { ...syncStatus };
 }
 
+/**
+ * Fetch GitHub Lists and annotate starred repos
+ * @param {import("mongodb").Db} db
+ * @param {object} options - Plugin options (needs token)
+ */
+async function syncLists(db, options) {
+  try {
+    const lists = await fetchAllLists(options.token);
+
+    if (lists.length > 0) {
+      const result = await annotateWithLists(db, lists);
+      console.log(
+        `[GitHub Stars] Lists sync: ${result.listsProcessed} lists, ${result.reposAnnotated} repos annotated`,
+      );
+    }
+
+    // Store list metadata in sync state
+    const listMeta = lists.map((l) => ({
+      name: l.name,
+      slug: l.slug,
+      description: l.description,
+      count: l.repoFullNames.length,
+    }));
+
+    await updateSyncState(db, { lists: listMeta });
+  } catch (error) {
+    console.error("[GitHub Stars] Lists sync failed:", error.message);
+    // Non-fatal — starred data is still usable without lists
+  }
+}
+
 /**
  * Start the background sync scheduler
  * @param {object} Indiekit - Indiekit instance
@@ -68,16 +100,22 @@ export function startStarredSync(Indiekit, options) {
         console.log(`[GitHub Stars] Cache has ${count} stars, running incremental sync...`);
         await runIncrementalSync(db, options);
       }
+
+      // Sync lists after initial star sync
+      await syncLists(db, options);
     } catch (error) {
       console.error("[GitHub Stars] Initial sync error:", error.message);
     }
   }, 15_000);
 
-  // Incremental sync every 6 hours
-  incrementalTimer = setInterval(() => {
-    runIncrementalSync(db, options).catch((err) => {
+  // Incremental sync every 6 hours (also re-annotate lists for new stars)
+  incrementalTimer = setInterval(async () => {
+    try {
+      await runIncrementalSync(db, options);
+      await syncLists(db, options);
+    } catch (err) {
       console.error("[GitHub Stars] Incremental sync error:", err.message);
-    });
+    }
   }, SIX_HOURS);
 
   // Full sync every 7 days
@@ -189,6 +227,9 @@ export async function runFullSync(db, options) {
       totalStars,
     });
 
+    // Sync lists during full sync
+    await syncLists(db, options);
+
     console.log(
       `[GitHub Stars] Full sync complete: ${result.upserted} new, ${result.modified} updated, ${removed} removed, ${totalStars} total (GitHub reports ${totalCount})`,
     );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-github",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "GitHub activity endpoint for Indiekit. Display commits, stars, contributions, and featured repositories.",
   "keywords": [
     "indiekit",