@jant/core 0.3.39 → 0.3.42

package/src/lib/github-sync-trigger.ts

@@ -0,0 +1,199 @@
+/**
+ * GitHub Sync Trigger
+ *
+ * Two dispatch paths:
+ *
+ * - `triggerGitHubSync` (queue-based): the original design, kept for
+ *   future use when a Cloudflare Queue binding is actually wired up.
+ *   Falls back to a no-op queue today, which silently drops jobs.
+ * - `triggerGitHubSyncInline` (inline): runs pushFullSync in the
+ *   current worker invocation via `c.executionCtx.waitUntil`. Works
+ *   uniformly on Workers and Node, no queue binding required. This
+ *   is what every caller uses today.
+ *
+ * Both paths debounce through a PENDING flag. When a new trigger
+ * arrives while a sync is running, the inline runner records it via
+ * DIRTY; the running sync re-runs once more after completion so the
+ * new edits land.
+ */
+
+import type { Context } from "hono";
+import type { SettingsService } from "../services/settings.js";
+import type { GitHubSyncService } from "../services/github-sync.js";
+import type { AppVariables } from "../types/app-context.js";
+import type { Bindings } from "../types/bindings.js";
+import { noopQueue, type JobQueue } from "./job-queue.js";
+import { createCfJobQueue } from "./job-queue-cf.js";
+import { buildSyncSiteConfig } from "./github-sync-site-config.js";
+
+type Env = { Bindings: Bindings; Variables: AppVariables };
+
+/**
+ * Resolve the appropriate job queue from the environment.
+ * Returns the CF Queue adapter if available, otherwise noop.
+ */
+export function resolveJobQueue(env: { GITHUB_SYNC_QUEUE?: Queue }): JobQueue {
+  if (env.GITHUB_SYNC_QUEUE) {
+    return createCfJobQueue(env.GITHUB_SYNC_QUEUE);
+  }
+  return noopQueue;
+}
+
+/**
+ * Maximum time a sync is allowed to be "in flight" before we consider
+ * the PENDING flag stale. Covers worker crashes, timeouts, and any
+ * other path where `runBackgroundSync`'s `finally` didn't execute.
+ *
+ * Normal syncs are a few seconds; a very large site might take a
+ * minute or two. Ten minutes is comfortably above the worst realistic
+ * case without trapping users under a stuck indicator forever.
+ */
+export const SYNC_PENDING_STALE_SECONDS = 10 * 60;
+
+/**
+ * Returns the effective "sync in progress" state.
+ *
+ * A plain read of `GITHUB_SYNC_PENDING` can lie — the flag stays
+ * "true" if the worker died before the finally clause ran. This helper
+ * cross-checks `GITHUB_SYNC_PENDING_AT`: if the timestamp is missing
+ * or older than `SYNC_PENDING_STALE_SECONDS`, we treat PENDING as
+ * not actually pending so the UI and trigger paths self-heal.
+ */
+export async function isSyncPending(
+  settings: SettingsService,
+): Promise<boolean> {
+  const [pending, pendingAt] = await Promise.all([
+    settings.get("GITHUB_SYNC_PENDING"),
+    settings.get("GITHUB_SYNC_PENDING_AT"),
+  ]);
+  if (pending !== "true") return false;
+  const ts = Number(pendingAt);
+  if (!Number.isFinite(ts) || ts <= 0) return false;
+  const nowSec = Math.floor(Date.now() / 1000);
+  return nowSec - ts <= SYNC_PENDING_STALE_SECONDS;
+}
+
+/**
+ * Flip the sync state to "pending" with a timestamp. Call before kicking
+ * off `runBackgroundSync` so the status card shows "Syncing…" while the
+ * push runs and `isSyncPending` can tell stuck flags from live ones.
+ */
+export async function markSyncPending(
+  settings: SettingsService,
+): Promise<void> {
+  await settings.set("GITHUB_SYNC_PENDING", "true");
+  await settings.set(
+    "GITHUB_SYNC_PENDING_AT",
+    String(Math.floor(Date.now() / 1000)),
+  );
+  await settings.set("GITHUB_SYNC_DIRTY", "");
+}
+
+/**
+ * Queue-based trigger. Safe to call on every post mutation. Kept for
+ * compatibility with the original design but currently enqueues to a
+ * noop queue on every known deployment — callers should use
+ * `triggerGitHubSyncInline` instead.
+ */
+export async function triggerGitHubSync(
+  queue: JobQueue,
+  settings: SettingsService,
+  siteId: string,
+): Promise<void> {
+  const enabled = await settings.get("GITHUB_SYNC_ENABLED");
+  if (enabled !== "true") return;
+
+  const alreadyPending = await settings.get("GITHUB_SYNC_PENDING");
+  if (alreadyPending === "true") return;
+
+  await settings.set("GITHUB_SYNC_PENDING", "true");
+  await queue.enqueue({
+    kind: "github-sync-push",
+    siteId,
+    data: {},
+  });
+}
+
+/**
+ * Run a full GitHub Sync push in the background, managing the lifecycle
+ * flags (`GITHUB_SYNC_PENDING`, `GITHUB_SYNC_DIRTY`, `GITHUB_SYNC_LAST_ERROR`).
+ *
+ * If a trigger arrives mid-push, the dirty flag is set and we loop
+ * once more after the current push completes so those edits land.
+ * That's what `triggerGitHubSyncInline` relies on to guarantee
+ * mid-sync writes aren't lost despite the PENDING gate blocking
+ * concurrent pushes.
+ */
+export async function runBackgroundSync(
+  settings: SettingsService,
+  syncService: GitHubSyncService,
+): Promise<void> {
+  try {
+    // Loop: push, then check whether another trigger came in during
+    // execution. Clearing DIRTY *before* the push means anything set
+    // after that cutoff causes exactly one more pass.
+    while (true) {
+      await settings.set("GITHUB_SYNC_DIRTY", "");
+      await syncService.pushFullSync();
+      const dirty = await settings.get("GITHUB_SYNC_DIRTY");
+      if (dirty !== "true") break;
+    }
+    await settings.set("GITHUB_SYNC_LAST_ERROR", "");
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    await settings.set("GITHUB_SYNC_LAST_ERROR", message);
+  } finally {
+    await settings.set("GITHUB_SYNC_PENDING", "");
+    await settings.set("GITHUB_SYNC_PENDING_AT", "");
+    await settings.set("GITHUB_SYNC_DIRTY", "");
+  }
+}
+
+/**
+ * Debounced inline trigger. Every caller that wants a sync after a
+ * content change should use this — it is the direct replacement for
+ * the queue-based `triggerGitHubSync`.
+ *
+ * Debounce semantics:
+ *   - If a sync is running already, mark DIRTY and return. The
+ *     running sync picks up the flag and re-runs once finished.
+ *   - Otherwise, set PENDING, build a sync service, and hand the
+ *     push to `c.executionCtx.waitUntil` so the HTTP response can
+ *     return immediately.
+ *
+ * Safe no-op when GitHub Sync isn't enabled for this site.
+ */
+export async function triggerGitHubSyncInline(c: Context<Env>): Promise<void> {
+  const settings = c.var.services.settings;
+
+  const enabled = await settings.get("GITHUB_SYNC_ENABLED");
+  if (enabled !== "true") return;
+
+  if (await isSyncPending(settings)) {
+    // Another (non-stale) sync is in flight. Record this request so the
+    // running sync re-runs after it finishes. No new push is kicked off
+    // here — the `runBackgroundSync` loop handles it.
+    await settings.set("GITHUB_SYNC_DIRTY", "true");
+    return;
+  }
+
+  await markSyncPending(settings);
+
+  const { createGitHubSyncService } =
+    await import("../services/github-sync.js");
+  const { getGitHubAppConfig } = await import("./env.js");
+  const syncService = createGitHubSyncService(
+    c.var.services,
+    c.var.currentSite.id,
+    await buildSyncSiteConfig(c),
+    { storage: c.var.storage, githubApp: getGitHubAppConfig(c.env) },
+  );
+
+  const run = runBackgroundSync(settings, syncService);
+  try {
+    c.executionCtx?.waitUntil(run);
+  } catch {
+    // executionCtx not available (e.g. tests, Node) — let the promise
+    // resolve on its own; HTTP response still returns immediately.
+  }
+}

package/src/lib/github-sync-worker.ts

@@ -0,0 +1,72 @@
+/**
+ * GitHub Sync Worker
+ *
+ * Processes queued sync jobs. Push jobs always run a full sync.
+ * After completing, re-checks the pending flag to catch changes
+ * that arrived during execution.
+ */
+
+import { createGitHubSyncService } from "../services/github-sync.js";
+import type { SiteConfig } from "../services/export.js";
+import type { Services } from "../services/index.js";
+import type { StorageDriver } from "../lib/storage.js";
+import type { GitHubAppEnvConfig } from "./env.js";
+import type { JobPayload } from "./job-queue.js";
+
+/**
+ * Process a single GitHub Sync job.
+ */
+export async function processGitHubSyncJob(
+  payload: JobPayload,
+  services: Services,
+  siteId: string,
+  siteConfig: SiteConfig,
+  storage?: StorageDriver | null,
+  githubApp?: GitHubAppEnvConfig | null,
+): Promise<void> {
+  const syncService = createGitHubSyncService(
+    {
+      posts: services.posts,
+      paths: services.paths,
+      collections: services.collections,
+      media: services.media,
+      settings: services.settings,
+    },
+    siteId,
+    siteConfig,
+    { storage, githubApp },
+  );
+
+  if (payload.kind === "github-sync-push") {
+    // Clear pending flag before running so new triggers during
+    // execution will set it again.
+    await services.settings.set("GITHUB_SYNC_PENDING", "");
+
+    await syncService.pushFullSync();
+
+    // If the flag was re-set during execution, run once more
+    const stillPending = await services.settings.get("GITHUB_SYNC_PENDING");
+    if (stillPending === "true") {
+      await services.settings.set("GITHUB_SYNC_PENDING", "");
+      await syncService.pushFullSync();
+    }
+    return;
+  }
+
+  if (payload.kind === "github-sync-pull") {
+    const webhookPayload = payload.data as {
+      ref: string;
+      before: string;
+      after: string;
+      commits: Array<{
+        id: string;
+        message: string;
+        added: string[];
+        modified: string[];
+        removed: string[];
+      }>;
+    };
+    await syncService.handleWebhookPush(webhookPayload);
+    return;
+  }
+}

package/src/lib/hosted-signin.ts

@@ -6,7 +6,7 @@ import {
 
 function getHostedAdminContinuationPath(publicRequestUrl: string): string {
   const currentHost = new URL(publicRequestUrl).host;
-  return `/auth/handoff/start?host=${encodeURIComponent(currentHost)}&redirect=${encodeURIComponent("/settings")}`;
+  return `/auth/handoff/start?host=${encodeURIComponent(currentHost)}&redirect=${encodeURIComponent("/")}`;
 }
 
 function buildHostedControlPlaneUrl(

package/src/lib/hugo-markdown.ts

@@ -0,0 +1,255 @@
+/**
+ * Shared Hugo Markdown parsing utilities.
+ *
+ * Jant's static-site export targets Hugo. Posts are stored as branch
+ * bundles (`content/{root-slug}/_index.md`) with each reply as a nested
+ * leaf bundle (`content/{root-slug}/{reply-slug}/index.md`). Front matter
+ * is flat YAML — no nested `extra` / `[extra.jant]` tables.
+ *
+ * This module owns the canonical shape of that front matter plus the
+ * parser / formatter pair used by the exporter, the import CLI, and
+ * GitHub sync.
+ */
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/**
+ * Per-collection membership metadata. Preserved verbatim on round-trip so
+ * curators keep their collected_at / position / pinned_at ordering.
+ */
+export interface HugoCollectionRef {
+  slug: string;
+  /**
+   * Denormalized collection title. Emitted so Hugo templates can render
+   * the collection name directly from front matter without a site-wide
+   * lookup. Import is authoritative via `slug`; `title` is refreshed on
+   * every export and ignored on round-trip.
+   */
+  title?: string;
+  collected_at?: string;
+  position?: number;
+  pinned_at?: string | null;
+}
+
+/**
+ * Jant media attachment descriptor. Stored flat inside the `media:` front-
+ * matter array. `src` is either:
+ *   - a site-relative path (e.g. `/media/{id}.webp`) when the bytes are
+ *     bundled under `static/media/` in the exported site, or
+ *   - an absolute URL (e.g. `https://cdn.example.com/...`) when the media
+ *     provider has a reachable public URL and we link rather than re-
+ *     bundle the bytes.
+ *
+ * `poster` follows the same rule for video poster frames. Round-trip
+ * fields (`provider`, `storage_key`, `poster_key`) preserve the original
+ * provider coordinates so re-imports can rebuild the media record.
+ */
+export interface JantMedia {
+  id: string;
+  kind: "image" | "video" | "audio" | "document" | "file" | "text";
+  src: string;
+  alt?: string;
+  width?: number;
+  height?: number;
+  blurhash?: string;
+  position?: number;
+  poster?: string;
+  original_name?: string;
+  mime_type?: string;
+  size?: number;
+  duration_seconds?: number;
+  waveform?: string;
+  summary?: string;
+  chars?: number;
+  provider?: string;
+  storage_key?: string;
+  poster_key?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Hugo's `build` front-matter block. Used on reply leaf bundles to hide
+ * them from the site's URL space while keeping them visible to the parent
+ * page as `.Pages` entries.
+ */
+export interface HugoBuildOptions {
+  render?: "always" | "never" | "link";
+  list?: "always" | "local" | "never";
+  publishResources?: boolean;
+}
+
+/**
+ * Front matter shape shared by every content file Jant emits. Specific
+ * page `type` values layer additional keys on top.
+ */
+export interface HugoFrontMatter {
+  // Identity
+  id?: string;
+  title?: string;
+  date?: string;
+  updated?: string;
+  slug?: string;
+  type?: string;
+  draft?: boolean;
+
+  // Hugo routing
+  aliases?: string[];
+  build?: HugoBuildOptions;
+
+  // Jant post payload (flat — no `extra` nesting)
+  format?: string;
+  status?: string;
+  visibility?: string;
+  summary_text?: string;
+  link_url?: string;
+  source_name?: string;
+  source_url?: string;
+  quote_text?: string;
+  rating?: number | null;
+  featured_at?: string | null;
+  pinned_at?: string | null;
+
+  // Round-trip bookkeeping
+  root_aliases?: string[];
+
+  // Memberships + attachments
+  collections?: HugoCollectionRef[];
+  media?: JantMedia[];
+
+  // Escape hatch for page-type-specific keys (home default view, etc.)
+  [key: string]: unknown;
+}
+
+// ---------------------------------------------------------------------------
+// Parser
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse front matter from a Markdown file.
+ *
+ * Hugo bundles emitted by Jant use YAML (`---...---`). TOML (`+++...+++`)
+ * is also accepted so content authored by hand or migrated from an older
+ * Zola export still parses cleanly.
+ *
+ * @param content - Raw file content
+ * @returns Parsed front matter object and the remaining body text
+ *
+ * @example
+ * ```ts
+ * const { frontMatter, body } = await parseFrontMatter(fileContent);
+ * console.log(frontMatter.slug, frontMatter.format);
+ * ```
+ */
+export async function parseFrontMatter(
+  content: string,
+): Promise<{ frontMatter: HugoFrontMatter; body: string }> {
+  // YAML front matter (---...---) — primary format.
+  const yamlMatch = content.match(
+    /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/,
+  );
+  if (yamlMatch) {
+    const { parse } = await import("yaml");
+    const frontMatter = (parse(yamlMatch[1] ?? "") ?? {}) as HugoFrontMatter;
+    return { frontMatter, body: yamlMatch[2] ?? "" };
+  }
+
+  // TOML front matter (+++...+++) — legacy / migration fallback.
+  const tomlMatch = content.match(
+    /^\+\+\+\r?\n([\s\S]*?)\r?\n\+\+\+\r?\n?([\s\S]*)$/,
+  );
+  if (tomlMatch) {
+    const { parse } = await import("smol-toml");
+    const frontMatter = parse(tomlMatch[1] ?? "") as HugoFrontMatter;
+    return { frontMatter, body: tomlMatch[2] ?? "" };
+  }
+
+  return { frontMatter: {}, body: content };
+}
+
+// ---------------------------------------------------------------------------
+// Formatter
+// ---------------------------------------------------------------------------
+
+/**
+ * Canonical key order for YAML front matter. Keys not listed here fall
+ * through to the end in insertion order. Stable ordering is essential for
+ * byte-level round-trip tests and clean diffs in the demo-source canonical
+ * export.
+ */
+const FRONT_MATTER_KEY_ORDER: readonly string[] = [
+  // Identity & routing
+  "id",
+  "title",
+  "date",
+  "updated",
+  "slug",
+  "type",
+  "draft",
+  "aliases",
+  "build",
+
+  // Post payload
+  "format",
+  "status",
+  "visibility",
+  "summary_text",
+  "link_url",
+  "source_name",
+  "source_url",
+  "quote_text",
+  "rating",
+  "featured_at",
+  "pinned_at",
+
+  // Bookkeeping / attachments
+  "root_aliases",
+  "collections",
+  "media",
+];
+
+/**
+ * Serialize a front-matter object as a YAML `---...---` block with stable
+ * key order. Keys with `undefined` values are skipped; `null` is emitted
+ * explicitly so "cleared" round-trips survive.
+ *
+ * @example
+ * ```ts
+ * const block = await formatFrontMatter({ slug: "hello", date: "2026-01-01", type: "post" });
+ * // ---
+ * // title: ...
+ * // date: 2026-01-01
+ * // slug: hello
+ * // type: post
+ * // ---
+ * ```
+ */
+export async function formatFrontMatter(
+  frontMatter: HugoFrontMatter,
+): Promise<string> {
+  const { stringify } = await import("yaml");
+
+  const ordered: Record<string, unknown> = {};
+  for (const key of FRONT_MATTER_KEY_ORDER) {
+    if (key in frontMatter) {
+      const value = (frontMatter as Record<string, unknown>)[key];
+      if (value !== undefined) ordered[key] = value;
+    }
+  }
+  // Preserve any extra keys (page-type discriminators etc.) in insertion order.
+  for (const key of Object.keys(frontMatter)) {
+    if (key in ordered) continue;
+    const value = (frontMatter as Record<string, unknown>)[key];
+    if (value !== undefined) ordered[key] = value;
+  }
+
+  const yaml = stringify(ordered, {
+    defaultKeyType: "PLAIN",
+    defaultStringType: "QUOTE_DOUBLE",
+    lineWidth: 0,
+  });
+  // `yaml.stringify` always ends with a newline; trim it so the block shape
+  // below doesn't produce a trailing blank line.
+  return `---\n${yaml.trimEnd()}\n---\n`;
+}

package/src/lib/job-queue-cf.ts

@@ -0,0 +1,18 @@
+/**
+ * Cloudflare Queue adapter for the job queue.
+ */
+
+import type { JobPayload, JobQueue } from "./job-queue.js";
+
+/**
+ * Create a job queue backed by a Cloudflare Queue binding.
+ *
+ * @param queue - The CF Queue binding (e.g. `env.GITHUB_SYNC_QUEUE`)
+ */
+export function createCfJobQueue(queue: Queue): JobQueue {
+  return {
+    async enqueue(payload: JobPayload) {
+      await queue.send(payload);
+    },
+  };
+}

package/src/lib/job-queue-db.ts

@@ -0,0 +1,149 @@
+/**
+ * Database-backed job queue.
+ *
+ * Used as a fallback when Cloudflare Queues are not available (e.g. Node/Postgres
+ * or self-hosted deployments). Jobs are stored in the `sync_job` table and
+ * processed via periodic polling.
+ */
+
+import { eq, and, lte, sql } from "drizzle-orm";
+import type { Database } from "../db/index.js";
+import type { DatabaseSchema } from "../db/schema-bundle.js";
+import type { JobPayload, JobQueue } from "./job-queue.js";
+import { typeidUnboxed } from "typeid-js";
+import { now as unixNow } from "./time.js";
+
+// ---------------------------------------------------------------------------
+// Queue (producer)
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a job queue backed by a database table.
+ *
+ * @param db - Drizzle database instance
+ * @param schema - Database schema bundle
+ */
+export function createDbJobQueue(
+  db: Database,
+  schema: DatabaseSchema,
+): JobQueue {
+  return {
+    async enqueue(payload: JobPayload) {
+      const now = unixNow();
+      await db.insert(schema.syncJobs).values({
+        id: typeidUnboxed("job"),
+        siteId: payload.siteId,
+        kind: payload.kind,
+        payload: JSON.stringify(payload.data),
+        status: "pending",
+        attempts: 0,
+        maxAttempts: 3,
+        createdAt: now,
+        updatedAt: now,
+        lockedUntil: null,
+      });
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Worker (consumer) — poll-based dequeue
+// ---------------------------------------------------------------------------
+
+export interface DbJobWorkerOptions {
+  db: Database;
+  schema: DatabaseSchema;
+  /** Called to process each dequeued job. */
+  handler: (payload: JobPayload) => Promise<void>;
+  /** Lock duration in seconds. Defaults to 60. */
+  lockDurationSec?: number;
+}
+
+/**
+ * Dequeue and process a single pending job.
+ *
+ * @returns `true` if a job was processed, `false` if the queue was empty.
+ */
+export async function dequeueAndProcess(
+  opts: DbJobWorkerOptions,
+): Promise<boolean> {
+  const { db, schema, handler, lockDurationSec = 60 } = opts;
+  const now = unixNow();
+
+  // Find one eligible job (pending or timed-out lock)
+  const [job] = await db
+    .select()
+    .from(schema.syncJobs)
+    .where(
+      and(
+        eq(schema.syncJobs.status, "pending"),
+        sql`(${schema.syncJobs.lockedUntil} IS NULL OR ${schema.syncJobs.lockedUntil} <= ${now})`,
+      ),
+    )
+    .orderBy(schema.syncJobs.createdAt)
+    .limit(1);
+
+  if (!job) return false;
+
+  // Lock the job
+  await db
+    .update(schema.syncJobs)
+    .set({
+      status: "processing",
+      lockedUntil: now + lockDurationSec,
+      attempts: job.attempts + 1,
+      updatedAt: now,
+    })
+    .where(eq(schema.syncJobs.id, job.id));
+
+  try {
+    const payload: JobPayload = {
+      kind: job.kind as JobPayload["kind"],
+      siteId: job.siteId,
+      data: JSON.parse(job.payload) as Record<string, unknown>,
+    };
+    await handler(payload);
+
+    // Mark completed
+    await db
+      .update(schema.syncJobs)
+      .set({ status: "completed", updatedAt: unixNow() })
+      .where(eq(schema.syncJobs.id, job.id));
+  } catch {
+    const updatedAttempts = job.attempts + 1;
+    const maxAttempts = job.maxAttempts;
+
+    await db
+      .update(schema.syncJobs)
+      .set({
+        status: updatedAttempts >= maxAttempts ? "failed" : "pending",
+        lockedUntil: null,
+        updatedAt: unixNow(),
+      })
+      .where(eq(schema.syncJobs.id, job.id));
+  }
+
+  return true;
+}
+
+/**
+ * Clean up completed and failed jobs older than the given age.
+ *
+ * @param maxAgeSec - Maximum age in seconds. Defaults to 7 days.
+ */
+export async function cleanupOldJobs(
+  db: Database,
+  schema: DatabaseSchema,
+  maxAgeSec = 7 * 24 * 3600,
+): Promise<number> {
+  const cutoff = unixNow() - maxAgeSec;
+  const result = await db
+    .delete(schema.syncJobs)
+    .where(
+      and(
+        sql`${schema.syncJobs.status} IN ('completed', 'failed')`,
+        lte(schema.syncJobs.updatedAt, cutoff),
+      ),
+    );
+  return (result as { rowsAffected?: number }).rowsAffected ?? 0;
+}