@rmdes/indiekit-endpoint-github 1.0.7 → 1.1.0

package/index.js

@@ -9,6 +9,7 @@ import { contributionsController } from "./lib/controllers/contributions.js";
 import { dashboardController } from "./lib/controllers/dashboard.js";
 import { featuredController } from "./lib/controllers/featured.js";
 import { starsController } from "./lib/controllers/stars.js";
+import { starredController } from "./lib/controllers/starred.js";
 
 // Module-level routers (matching Indiekit's endpoint pattern)
 const protectedRouter = express.Router();
@@ -77,6 +78,7 @@ export default class GitHubEndpoint {
     protectedRouter.get("/contributions", contributionsController.get);
     protectedRouter.get("/activity", activityController.get);
     protectedRouter.get("/featured", featuredController.get);
+    protectedRouter.get("/starred/sync", starredController.sync);
 
     return protectedRouter;
   }
@@ -93,6 +95,8 @@ export default class GitHubEndpoint {
     publicRouter.get("/api/activity", activityController.api);
     publicRouter.get("/api/featured", featuredController.api);
     publicRouter.get("/api/changelog", changelogController.api);
+    publicRouter.get("/api/starred/all", starredController.all);
+    publicRouter.get("/api/starred/recent", starredController.recent);
 
     return publicRouter;
   }
@@ -100,8 +104,24 @@ export default class GitHubEndpoint {
   init(Indiekit) {
     Indiekit.addEndpoint(this);
 
-    // Store GitHub config in application for controller access
+    // Register MongoDB collections for starred cache
+    Indiekit.addCollection("github_stars");
+    Indiekit.addCollection("github_sync_state");
+
+    // Store config and DB getter for controller access
     Indiekit.config.application.githubConfig = this.options;
     Indiekit.config.application.githubEndpoint = this.mountPath;
+    Indiekit.config.application.getGithubDb = () => Indiekit.database;
+
+    // Start background sync for starred repos (if token + DB available)
+    if (this.options.token && Indiekit.database) {
+      import("./lib/starred-sync.js")
+        .then(({ startStarredSync }) => {
+          startStarredSync(Indiekit, this.options);
+        })
+        .catch((error) => {
+          console.error("[GitHub Stars] Sync scheduler failed to start:", error.message);
+        });
+    }
   }
 }

package/lib/controllers/starred.js

@@ -0,0 +1,118 @@
+/**
+ * Starred repositories API controller
+ * Serves cached data from MongoDB for Eleventy build + live updates
+ */
+
+import {
+  getAllStars,
+  getRecentStars,
+  getStarCount,
+  getSyncState,
+} from "../starred-cache.js";
+import {
+  runIncrementalSync,
+  runFullSync,
+  getStarredSyncStatus,
+} from "../starred-sync.js";
+
+export const starredController = {
+  /**
+   * GET /api/starred/all — Public JSON API
+   * Returns all cached starred repos for Eleventy build
+   */
+  async all(request, response, next) {
+    try {
+      const { application } = request.app.locals;
+      const db = application.getGithubDb();
+
+      if (!db) {
+        return response.json({ stars: [], totalCount: 0, lastSync: null });
+      }
+
+      const collection = db.collection("github_stars");
+      const [stars, totalCount, syncState] = await Promise.all([
+        getAllStars(collection),
+        getStarCount(collection),
+        getSyncState(db),
+      ]);
+
+      response.json({
+        stars,
+        totalCount,
+        lastSync: syncState.lastIncrementalSync || syncState.lastFullSync || null,
+      });
+    } catch (error) {
+      next(error);
+    }
+  },
+
+  /**
+   * GET /api/starred/recent?since=ISO — Public JSON API
+   * Returns stars newer than the given timestamp (for live updates)
+   */
+  async recent(request, response, next) {
+    try {
+      const { application } = request.app.locals;
+      const db = application.getGithubDb();
+
+      const since = request.query.since;
+      if (!since) {
+        return response.status(400).json({ error: "Missing 'since' query parameter (ISO date)" });
+      }
+
+      if (!db) {
+        return response.json({ stars: [], totalCount: 0 });
+      }
+
+      const collection = db.collection("github_stars");
+      const [stars, totalCount] = await Promise.all([
+        getRecentStars(collection, since),
+        getStarCount(collection),
+      ]);
+
+      response.json({ stars, totalCount });
+    } catch (error) {
+      next(error);
+    }
+  },
+
+  /**
+   * GET /api/starred/sync — Protected, triggers manual sync
+   * Query param: ?full=true for full re-sync
+   */
+  async sync(request, response, next) {
+    try {
+      const { application } = request.app.locals;
+      const db = application.getGithubDb();
+      const config = application.githubConfig;
+
+      if (!db) {
+        return response.status(503).json({ error: "Database not available" });
+      }
+
+      if (!config.token) {
+        return response.status(400).json({ error: "No GitHub token configured" });
+      }
+
+      const status = getStarredSyncStatus();
+      if (status.syncing) {
+        return response.status(409).json({ error: "Sync already in progress" });
+      }
+
+      const isFull = request.query.full === "true";
+
+      // Run sync in background, respond immediately
+      const syncFn = isFull ? runFullSync : runIncrementalSync;
+      syncFn(db, config).catch((err) => {
+        console.error("[GitHub Stars] Manual sync error:", err.message);
+      });
+
+      response.json({
+        message: `${isFull ? "Full" : "Incremental"} sync started`,
+        currentStatus: getStarredSyncStatus(),
+      });
+    } catch (error) {
+      next(error);
+    }
+  },
+};

package/lib/github-client.js

@@ -40,7 +40,17 @@ export class GitHubClient {
     const response = await fetch(url, { headers });
 
     if (!response.ok) {
-      throw await IndiekitError.fromFetch(response);
+      // Only use fromFetch for JSON error responses; GitHub sometimes returns
+      // HTML error pages (e.g., 502 Bad Gateway) which cause SyntaxError noise
+      const contentType = response.headers.get("content-type") || "";
+      if (contentType.includes("json")) {
+        throw await IndiekitError.fromFetch(response);
+      }
+
+      throw new IndiekitError(response.statusText, {
+        status: response.status,
+        code: response.statusText,
+      });
     }
 
     const data = await response.json();

package/lib/github-graphql.js

@@ -0,0 +1,131 @@
+/**
+ * GitHub GraphQL API client for fetching all starred repositories
+ * Uses cursor-based pagination to fetch 100 repos per request
+ * @module github-graphql
+ */
+
+const GITHUB_GRAPHQL_URL = "https://api.github.com/graphql";
+
+const STARRED_QUERY = `
+  query($cursor: String) {
+    viewer {
+      starredRepositories(first: 100, after: $cursor, orderBy: {field: STARRED_AT, direction: DESC}) {
+        totalCount
+        edges {
+          starredAt
+          node {
+            nameWithOwner
+            name
+            description
+            url
+            primaryLanguage { name }
+            stargazerCount
+            forkCount
+            pushedAt
+            isArchived
+            owner { avatarUrl login }
+            licenseInfo { spdxId name }
+            repositoryTopics(first: 10) {
+              nodes { topic { name } }
+            }
+          }
+        }
+        pageInfo { endCursor hasNextPage }
+      }
+    }
+  }
+`;
+
+/**
+ * Format a single starred repo edge from GraphQL response
+ * @param {object} edge - GraphQL edge with starredAt + node
+ * @returns {object} Formatted starred repo
+ */
+function formatStarredRepo(edge) {
+  const repo = edge.node;
+  return {
+    fullName: repo.nameWithOwner,
+    name: repo.name,
+    description: repo.description || "",
+    url: repo.url,
+    language: repo.primaryLanguage?.name || null,
+    stars: repo.stargazerCount,
+    forks: repo.forkCount,
+    topics: (repo.repositoryTopics?.nodes || []).map((n) => n.topic.name),
+    license: repo.licenseInfo?.spdxId || null,
+    archived: repo.isArchived,
+    starredAt: edge.starredAt,
+    ownerAvatar: repo.owner?.avatarUrl || "",
+    ownerLogin: repo.owner?.login || "",
+    pushedAt: repo.pushedAt,
+  };
+}
+
+/**
+ * Fetch all starred repositories via GraphQL pagination
+ * @param {string} token - GitHub personal access token (REQUIRED for GraphQL)
+ * @param {object} [options] - Fetch options
+ * @param {number} [options.maxPages] - Max pages to fetch (null = all)
+ * @param {Function} [options.onPage] - Callback after each page: (pageNum, totalFetched, totalCount)
+ * @returns {Promise<{stars: Array, totalCount: number}>}
+ */
+export async function fetchAllStarred(token, options = {}) {
+  if (!token) {
+    throw new Error("GitHub token is required for GraphQL API");
+  }
+
+  const { maxPages = null, onPage = null } = options;
+  const allStars = [];
+  let cursor = null;
+  let hasNextPage = true;
+  let totalCount = 0;
+  let pageNum = 0;
+
+  while (hasNextPage) {
+    if (maxPages !== null && pageNum >= maxPages) break;
+
+    const response = await fetch(GITHUB_GRAPHQL_URL, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        query: STARRED_QUERY,
+        variables: { cursor },
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`GraphQL request failed: ${response.status} ${response.statusText}`);
+    }
+
+    const body = await response.json();
+
+    if (body.errors) {
+      throw new Error(`GraphQL errors: ${body.errors.map((e) => e.message).join(", ")}`);
+    }
+
+    const starred = body.data.viewer.starredRepositories;
+    totalCount = starred.totalCount;
+
+    for (const edge of starred.edges) {
+      allStars.push(formatStarredRepo(edge));
+    }
+
+    cursor = starred.pageInfo.endCursor;
+    hasNextPage = starred.pageInfo.hasNextPage;
+    pageNum++;
+
+    if (onPage) {
+      onPage(pageNum, allStars.length, totalCount);
+    }
+
+    // Small delay between pages to avoid secondary rate limits
+    if (hasNextPage) {
+      await new Promise((resolve) => setTimeout(resolve, 200));
+    }
+  }
+
+  return { stars: allStars, totalCount };
+}

package/lib/starred-cache.js

@@ -0,0 +1,128 @@
+/**
+ * MongoDB cache layer for starred repositories
+ * Follows the webmention-io pattern: receives db object, not Indiekit instance
+ * @module starred-cache
+ */
+
+/**
+ * Ensure required indexes exist on the github_stars collection
+ * @param {import("mongodb").Collection} collection
+ */
+export async function ensureIndexes(collection) {
+  await collection.createIndex({ fullName: 1 }, { unique: true });
+  await collection.createIndex({ starredAt: -1 });
+  await collection.createIndex({ language: 1 });
+}
+
+/**
+ * Bulk upsert starred repos into the collection
+ * Uses fullName as unique key
+ * @param {import("mongodb").Collection} collection
+ * @param {Array} stars - Array of formatted starred repos
+ * @returns {Promise<{upserted: number, modified: number}>}
+ */
+export async function syncStars(collection, stars) {
+  if (!stars || stars.length === 0) return { upserted: 0, modified: 0 };
+
+  const operations = stars.map((star) => ({
+    updateOne: {
+      filter: { fullName: star.fullName },
+      update: {
+        $set: {
+          ...star,
+          updatedAt: new Date().toISOString(),
+        },
+      },
+      upsert: true,
+    },
+  }));
+
+  // Process in batches of 500 to avoid MongoDB limits
+  let upserted = 0;
+  let modified = 0;
+
+  for (let i = 0; i < operations.length; i += 500) {
+    const batch = operations.slice(i, i + 500);
+    const result = await collection.bulkWrite(batch, { ordered: false });
+    upserted += result.upsertedCount;
+    modified += result.modifiedCount;
+  }
+
+  return { upserted, modified };
+}
+
+/**
+ * Remove starred repos that no longer exist in the full list
+ * Called during full sync to handle unstarred repos
+ * @param {import("mongodb").Collection} collection
+ * @param {Set<string>} currentFullNames - Set of all current starred repo fullNames
+ * @returns {Promise<number>} Number of removed repos
+ */
+export async function removeUnstarred(collection, currentFullNames) {
+  const result = await collection.deleteMany({
+    fullName: { $nin: [...currentFullNames] },
+  });
+  return result.deletedCount;
+}
+
+/**
+ * Get all starred repos sorted by starredAt descending
+ * @param {import("mongodb").Collection} collection
+ * @returns {Promise<Array>}
+ */
+export async function getAllStars(collection) {
+  return collection
+    .find({}, { projection: { _id: 0, updatedAt: 0 } })
+    .sort({ starredAt: -1 })
+    .toArray();
+}
+
+/**
+ * Get stars added since a given date
+ * @param {import("mongodb").Collection} collection
+ * @param {string} since - ISO 8601 date string
+ * @returns {Promise<Array>}
+ */
+export async function getRecentStars(collection, since) {
+  return collection
+    .find(
+      { starredAt: { $gt: since } },
+      { projection: { _id: 0, updatedAt: 0 } },
+    )
+    .sort({ starredAt: -1 })
+    .toArray();
+}
+
+/**
+ * Get total count of starred repos in cache
+ * @param {import("mongodb").Collection} collection
+ * @returns {Promise<number>}
+ */
+export async function getStarCount(collection) {
+  return collection.countDocuments();
+}
+
+/**
+ * Get sync state from github_sync_state collection
+ * @param {import("mongodb").Db} db
+ * @returns {Promise<object>}
+ */
+export async function getSyncState(db) {
+  const stateCollection = db.collection("github_sync_state");
+  const state = await stateCollection.findOne({ _id: "starred_sync" });
+  return state || { lastIncrementalSync: null, lastFullSync: null };
+}
+
+/**
+ * Update sync state
+ * @param {import("mongodb").Db} db
+ * @param {object} update - Fields to update
+ */
+export async function updateSyncState(db, update) {
+  const stateCollection = db.collection("github_sync_state");
+  await stateCollection.findOneAndUpdate(
+    { _id: "starred_sync" },
+    { $set: update },
+    { upsert: true },
+  );
+}

package/lib/starred-sync.js

@@ -0,0 +1,203 @@
+/**
+ * Background sync scheduler for starred repositories
+ * Follows the webmention-io startSync/stopSync pattern
+ * @module starred-sync
+ */
+
+import { fetchAllStarred } from "./github-graphql.js";
+import {
+  ensureIndexes,
+  syncStars,
+  removeUnstarred,
+  getStarCount,
+  getSyncState,
+  updateSyncState,
+} from "./starred-cache.js";
+
+const SIX_HOURS = 6 * 60 * 60 * 1000;
+const SEVEN_DAYS = 7 * 24 * 60 * 60 * 1000;
+
+let incrementalTimer = null;
+let fullSyncTimer = null;
+
+let syncStatus = {
+  syncing: false,
+  lastSync: null,
+  lastError: null,
+  totalStars: 0,
+};
+
+/**
+ * Get current sync status (safe copy for templates/API)
+ * @returns {object}
+ */
+export function getStarredSyncStatus() {
+  return { ...syncStatus };
+}
+
+/**
+ * Start the background sync scheduler
+ * @param {object} Indiekit - Indiekit instance
+ * @param {object} options - Plugin options (needs token, username)
+ */
+export function startStarredSync(Indiekit, options) {
+  if (!options.token) {
+    console.warn("[GitHub Stars] No token configured, starred sync disabled");
+    return;
+  }
+
+  const db = Indiekit.database;
+  if (!db) {
+    console.warn("[GitHub Stars] No database available, starred sync disabled");
+    return;
+  }
+
+  console.log("[GitHub Stars] Starting background sync");
+
+  // Initial sync after 15s delay (let other plugins init first)
+  setTimeout(async () => {
+    try {
+      const collection = db.collection("github_stars");
+      await ensureIndexes(collection);
+
+      const count = await getStarCount(collection);
+      if (count === 0) {
+        console.log("[GitHub Stars] Empty cache, running full sync...");
+        await runFullSync(db, options);
+      } else {
+        console.log(`[GitHub Stars] Cache has ${count} stars, running incremental sync...`);
+        await runIncrementalSync(db, options);
+      }
+    } catch (error) {
+      console.error("[GitHub Stars] Initial sync error:", error.message);
+    }
+  }, 15_000);
+
+  // Incremental sync every 6 hours
+  incrementalTimer = setInterval(() => {
+    runIncrementalSync(db, options).catch((err) => {
+      console.error("[GitHub Stars] Incremental sync error:", err.message);
+    });
+  }, SIX_HOURS);
+
+  // Full sync every 7 days
+  fullSyncTimer = setInterval(() => {
+    runFullSync(db, options).catch((err) => {
+      console.error("[GitHub Stars] Full sync error:", err.message);
+    });
+  }, SEVEN_DAYS);
+}
+
+/**
+ * Stop all sync timers
+ */
+export function stopStarredSync() {
+  if (incrementalTimer) {
+    clearInterval(incrementalTimer);
+    incrementalTimer = null;
+  }
+  if (fullSyncTimer) {
+    clearInterval(fullSyncTimer);
+    fullSyncTimer = null;
+  }
+}
+
+/**
+ * Run incremental sync (fetch first 2 pages = 200 most recent stars)
+ * @param {import("mongodb").Db} db
+ * @param {object} options - Plugin options
+ * @returns {Promise<object>} Sync result
+ */
+export async function runIncrementalSync(db, options) {
+  if (syncStatus.syncing) {
+    return { error: "Sync already in progress" };
+  }
+
+  syncStatus.syncing = true;
+  syncStatus.lastError = null;
+
+  try {
+    const collection = db.collection("github_stars");
+    await ensureIndexes(collection);
+
+    const { stars } = await fetchAllStarred(options.token, { maxPages: 2 });
+
+    const result = await syncStars(collection, stars);
+    const totalStars = await getStarCount(collection);
+
+    syncStatus.lastSync = new Date().toISOString();
+    syncStatus.totalStars = totalStars;
+    syncStatus.syncing = false;
+
+    await updateSyncState(db, {
+      lastIncrementalSync: syncStatus.lastSync,
+      totalStars,
+    });
+
+    console.log(
+      `[GitHub Stars] Incremental sync: ${result.upserted} new, ${result.modified} updated, ${totalStars} total`,
+    );
+
+    return { ...result, totalStars };
+  } catch (error) {
+    syncStatus.lastError = error.message;
+    syncStatus.syncing = false;
+    console.error("[GitHub Stars] Incremental sync failed:", error.message);
+    return { error: error.message };
+  }
+}
+
+/**
+ * Run full sync (fetch ALL starred repos, remove unstarred)
+ * @param {import("mongodb").Db} db
+ * @param {object} options - Plugin options
+ * @returns {Promise<object>} Sync result
+ */
+export async function runFullSync(db, options) {
+  if (syncStatus.syncing) {
+    return { error: "Sync already in progress" };
+  }
+
+  syncStatus.syncing = true;
+  syncStatus.lastError = null;
+
+  try {
+    const collection = db.collection("github_stars");
+    await ensureIndexes(collection);
+
+    const { stars, totalCount } = await fetchAllStarred(options.token, {
+      onPage: (page, fetched, total) => {
+        console.log(`[GitHub Stars] Full sync: page ${page}, ${fetched}/${total} fetched`);
+      },
+    });
+
+    const result = await syncStars(collection, stars);
+
+    // Remove repos that were unstarred
+    const currentNames = new Set(stars.map((s) => s.fullName));
+    const removed = await removeUnstarred(collection, currentNames);
+
+    const totalStars = await getStarCount(collection);
+
+    syncStatus.lastSync = new Date().toISOString();
+    syncStatus.totalStars = totalStars;
+    syncStatus.syncing = false;
+
+    await updateSyncState(db, {
+      lastFullSync: syncStatus.lastSync,
+      lastIncrementalSync: syncStatus.lastSync,
+      totalStars,
+    });
+
+    console.log(
+      `[GitHub Stars] Full sync complete: ${result.upserted} new, ${result.modified} updated, ${removed} removed, ${totalStars} total (GitHub reports ${totalCount})`,
+    );
+
+    return { ...result, removed, totalStars, totalCount };
+  } catch (error) {
+    syncStatus.lastError = error.message;
+    syncStatus.syncing = false;
+    console.error("[GitHub Stars] Full sync failed:", error.message);
+    return { error: error.message };
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-github",
-  "version": "1.0.7",
+  "version": "1.1.0",
   "description": "GitHub activity endpoint for Indiekit. Display commits, stars, contributions, and featured repositories.",
   "keywords": [
     "indiekit",