@igstack/app-catalog-backend-core 6.10.2 → 6.11.0

package/src/generated/prisma/models.ts

@@ -23,4 +23,11 @@ export type * from './models/DbAppTagDefinition'
 export type * from './models/DbAsset'
 export type * from './models/Source'
 export type * from './models/SourceReference'
+export type * from './models/DbCommunityTool'
+export type * from './models/DbCommunityToolOwner'
+export type * from './models/DbCommunityToolLike'
+export type * from './models/DbCommunityToolNamedOwner'
+export type * from './models/DbCommunityToolTag'
+export type * from './models/DbCommunityToolView'
+export type * from './models/DbCommunityToolPin'
 export type * from './commonInputTypes'

package/src/index.ts

@@ -39,6 +39,22 @@ export {
   type FeedbackList,
   type FeedbackView,
 } from './modules/feedback/service'
+
+// Community Tools
+// ToolInput and Writer are part of the router's inferred type, so they must be
+// nameable from outside the package or every consumer's trpc hooks become unportable.
+export { createCommunityToolsRouter } from './modules/communityTools/router'
+export {
+  toolInputSchema,
+  type ToolInput,
+  type ToolPreview,
+  type Writer,
+} from './modules/communityTools/writeService'
+export type { CommunityToolsOptions } from './middleware/types'
+export type {
+  CaptureResult,
+  ScreenshotCapturer,
+} from './modules/communityTools/capture'
 // The identity every surface shares — rows store `hash`, clients may hold `publicId`,
 // and `visitorFromSeed` is how a test or a dev session becomes a chosen visitor.
 export {

package/src/middleware/createAcMiddleware.ts

@@ -13,6 +13,8 @@ import { createAcTrpcContext } from '../server/acTrpcContext'
 import { createAuth } from '../modules/auth/auth'
 import { createMockUserFromDevConfig } from '../modules/auth/devMockUserUtils'
 import { resolveActor } from '../modules/visitor/visitorIdentity'
+import { createPlaywrightCapturer } from '../modules/communityTools/capture'
+import { startHealthJob } from '../modules/communityTools/healthJob'
 
 export async function createAcMiddleware(
   options: AcMiddlewareOptions,
@@ -29,11 +31,28 @@ export async function createAcMiddleware(
   // Create auth instance
   const auth = createAuth(options.auth.betterAuthOptions)
 
+  // Resolve the default capturer once, so the router and the health job share it
+  const communityTools = options.communityTools && {
+    ...options.communityTools,
+    capturer:
+      options.communityTools.capturer === undefined
+        ? createPlaywrightCapturer()
+        : options.communityTools.capturer,
+  }
+
   // Create tRPC router
   const trpcRouter = createTrpcRouter(auth, {
     devLoginEnabled: !!options.auth.devMockUser,
+    communityTools,
   })
 
+  const stopHealthJob = communityTools?.enabled
+    ? startHealthJob({
+        captureHosts: communityTools.captureHosts ?? [],
+        capturer: communityTools.capturer,
+      })
+    : undefined
+
   // Normalize backend provider to async factory function
   const resolveBackend = createBackendResolver(options.backend)
 
@@ -140,6 +159,7 @@ export async function createAcMiddleware(
       if (options.hooks?.onDatabaseDisconnecting) {
         await options.hooks.onDatabaseDisconnecting()
       }
+      stopHealthJob?.()
       await dbManager.disconnect()
     },
 

package/src/middleware/types.ts

@@ -5,6 +5,7 @@ import type { Session, User } from 'better-auth/types'
 import type { AppCatalogCompanySpecificBackend } from '../types/backend/companySpecificBackend'
 import type { BetterAuth } from '../modules/auth/auth'
 import type { TRPCRouter } from '../server/controller'
+import type { ScreenshotCapturer } from '../modules/communityTools/capture'
 
 /**
  * Database connection configuration.
@@ -171,6 +172,26 @@ export interface AcMiddlewareOptions {
    * customScripts: ['/assets-static/analytics/analytics.js']
    */
   customScripts?: string[]
+
+  /** Community tools configuration (optional) */
+  communityTools?: CommunityToolsOptions
+}
+
+/**
+ * Community tools feature configuration.
+ */
+export interface CommunityToolsOptions {
+  enabled: boolean
+  /**
+   * Host suffixes the headless browser may visit, e.g. `['example.com']` for
+   * `example.com` and its subdomains. Empty by default: no screenshots, no health check.
+   */
+  captureHosts?: string[]
+  /**
+   * Overrides the default `playwright-core` capturer. `null` disables capture; omitted
+   * means use playwright-core when it is installed.
+   */
+  capturer?: ScreenshotCapturer | null
 }
 
 /**

package/src/modules/communityTools/capture.ts

@@ -0,0 +1,332 @@
+import { createRequire } from 'node:module'
+import { isCaptureAllowed } from '@igstack/app-catalog-shared-core'
+
+export type CaptureResult =
+  | {
+      ok: true
+      png: Buffer
+      title: string | null
+      favicon: Buffer | null
+      /** The page's `Last-Modified`: a static host sets it on each deploy. */
+      lastModified: Date | null
+    }
+  | { ok: false; error: string }
+
+/**
+ * Visits a page and photographs it, reaching only `allowedHosts`.
+ *
+ * Resolves `{ ok: false }` when the page itself failed (no response, an error status, a
+ * timeout, a redirect off the list) — that is the tool's health. Throws when the
+ * browser could not be started, which is ours: the health job must not mark every tool
+ * broken because Chromium is missing on one pod.
+ */
+export interface ScreenshotCapturer {
+  capture: (url: string, allowedHosts: string[]) => Promise<CaptureResult>
+  /** Captures queued and not started yet; a capturer without a queue leaves it out. */
+  waiting?: () => number
+}
+
+/** The part of a playwright `Page` a capture uses, so it can be tested without a browser. */
+export interface CapturePage {
+  setDefaultTimeout: (ms: number) => void
+  goto: (
+    url: string,
+    options: { timeout: number; waitUntil: 'load' },
+  ) => Promise<{
+    ok: () => boolean
+    status: () => number
+    headers: () => Record<string, string>
+  } | null>
+  waitForLoadState: (
+    state: 'networkidle',
+    options: { timeout: number },
+  ) => Promise<void>
+  waitForTimeout: (ms: number) => Promise<void>
+  url: () => string
+  screenshot: (options: { type: 'png' }) => Promise<Buffer>
+  title: () => Promise<string>
+  evaluate: (expression: string) => Promise<unknown>
+  request: {
+    get: (
+      url: string,
+      options: { timeout: number; maxRedirects: number },
+    ) => Promise<{ ok: () => boolean; body: () => Promise<Buffer> }>
+  }
+}
+
+const VIEWPORT = { width: 1280, height: 800 }
+const TIMEOUT_MS = 20_000
+/** After the network goes quiet, for the page to render what it fetched. */
+const SETTLE_MS = 1_500
+
+/**
+ * Whether the browser may load this URL: navigations, redirects and subresources alike.
+ * Inline `data:`/`blob:` URLs reach no host, and pages need them to render.
+ */
+export function allowRequest(url: string, allowedHosts: string[]): boolean {
+  return (
+    url.startsWith('data:') ||
+    url.startsWith('blob:') ||
+    isCaptureAllowed(url, allowedHosts)
+  )
+}
+
+const MAX_REDIRECTS = 5
+const HOP_TIMEOUT_MS = 10_000
+/** Hops a browser repeats as a bodyless GET; 307 and 308 keep the method and body. */
+const REDIRECTS_TO_GET = [301, 302, 303]
+
+/** `same` repeats the original method and body; `get` is a bodyless GET. */
+export type HopMode = 'same' | 'get'
+
+/**
+ * Fetches a URL, following redirects only while every hop stays on the allowed hosts.
+ * Null when a hop leaves them, or after too many hops; the off-list hop is never requested.
+ *
+ * Done by hand because Chromium follows a redirect, even one answered through
+ * `route.fulfill`, without consulting the route again (measured against a real browser).
+ */
+export async function followWithinHosts<
+  TResponse extends {
+    status: () => number
+    headers: () => Record<string, string>
+  },
+>(
+  fetch: (url: string, mode: HopMode) => Promise<TResponse>,
+  url: string,
+  allowedHosts: string[],
+): Promise<TResponse | null> {
+  let hop = url
+  let mode: HopMode = 'same'
+  for (let hops = 0; ; hops++) {
+    const response = await fetch(hop, mode)
+    const location = response.headers()['location']
+    const status = response.status()
+    if (status < 300 || status >= 400 || !location) return response
+    if (REDIRECTS_TO_GET.includes(status)) mode = 'get'
+    hop = new URL(location, hop).href
+    if (hops >= MAX_REDIRECTS || !isCaptureAllowed(hop, allowedHosts)) {
+      return null
+    }
+  }
+}
+
+/** The parts of a playwright `Route` the fence uses, so it can be tested without a browser. */
+export interface FenceRoute<TResponse> {
+  url: string
+  abort: (code: 'blockedbyclient' | 'failed') => Promise<void>
+  continue: () => Promise<void>
+  fetch: (url: string, mode: HopMode) => Promise<TResponse>
+  fulfill: (response: TResponse) => Promise<void>
+}
+
+/**
+ * Answers one browser request: off-list is aborted, on-list is fetched hop by hop.
+ *
+ * Never rejects. A rejection in a route handler surfaces as an unhandledRejection and
+ * exits Node (measured), so one dead image on a tool's page would take the pod down.
+ */
+export async function fenceRoute<
+  TResponse extends {
+    status: () => number
+    headers: () => Record<string, string>
+  },
+>(route: FenceRoute<TResponse>, allowedHosts: string[]): Promise<void> {
+  try {
+    if (!allowRequest(route.url, allowedHosts)) {
+      await route.abort('blockedbyclient')
+    } else if (route.url.startsWith('data:') || route.url.startsWith('blob:')) {
+      await route.continue()
+    } else {
+      const response = await followWithinHosts(
+        route.fetch,
+        route.url,
+        allowedHosts,
+      )
+      if (response) await route.fulfill(response)
+      else await route.abort('blockedbyclient')
+    }
+  } catch {
+    await route.abort('failed').catch(() => undefined)
+  }
+}
+
+/**
+ * Chromium flags that fence the network stack itself, below anything a page can reach:
+ * every name but the capture hosts fails to resolve (IP literals included), which also
+ * covers worker fetches and sockets that `route` never sees; WebRTC gets no direct UDP.
+ * A host that is not a plain name is dropped, so config cannot inject resolver rules.
+ */
+export function chromiumFenceArgs(allowedHosts: string[]): string[] {
+  const rules = [
+    'MAP * ~NOTFOUND',
+    ...allowedHosts
+      .filter((host) => /^[a-z0-9.-]+$/i.test(host))
+      .flatMap((host) => [`EXCLUDE *.${host}`, `EXCLUDE ${host}`]),
+  ]
+  return [
+    `--host-resolver-rules=${rules.join(', ')}`,
+    '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
+    '--webrtc-ip-handling-policy=disable_non_proxied_udp',
+  ]
+}
+
+/** Navigates an already-fenced page and collects the screenshot, title and favicon. */
+export async function capturePage(
+  page: CapturePage,
+  url: string,
+  allowedHosts: string[],
+): Promise<CaptureResult> {
+  page.setDefaultTimeout(TIMEOUT_MS)
+  const deadline = Date.now() + TIMEOUT_MS
+  const response = await page.goto(url, {
+    timeout: TIMEOUT_MS,
+    waitUntil: 'load',
+  })
+  // The route fence aborts an off-list redirect, but the result must not depend on it.
+  if (!isCaptureAllowed(page.url(), allowedHosts)) {
+    return { ok: false, error: 'redirected off the allowed hosts' }
+  }
+  if (!response || !response.ok()) {
+    return {
+      ok: false,
+      error: response ? `HTTP ${response.status()}` : 'No response',
+    }
+  }
+  // `load` is only the HTML and its assets: a page that fetches its data after
+  // that would be photographed saying "Loading…". So wait for the network to go
+  // quiet, then a moment for it to render, all inside the one timeout. A page
+  // that never goes quiet (polling, a live feed) is photographed as it is.
+  const idleBudget = deadline - Date.now() - SETTLE_MS
+  if (idleBudget > 0) {
+    await page
+      .waitForLoadState('networkidle', { timeout: idleBudget })
+      .catch(() => undefined)
+  }
+  const settle = Math.min(SETTLE_MS, deadline - Date.now())
+  if (settle > 0) await page.waitForTimeout(settle)
+  const png = await page.screenshot({ type: 'png' })
+  const title = (await page.title()).trim() || null
+
+  // A string, so this file needs no DOM types.
+  const iconHref = await page.evaluate(
+    `document.querySelector('link[rel~="icon"]')?.href ?? null`,
+  )
+  const iconUrl =
+    typeof iconHref === 'string'
+      ? iconHref
+      : new URL('/favicon.ico', page.url()).href
+  let favicon: Buffer | null = null
+  // `page.request` bypasses the route fence, so the href the page chose is checked here,
+  // and a redirect is not followed anywhere.
+  if (isCaptureAllowed(iconUrl, allowedHosts)) {
+    try {
+      const icon = await page.request.get(iconUrl, {
+        timeout: 5_000,
+        maxRedirects: 0,
+      })
+      if (icon.ok()) favicon = await icon.body()
+    } catch {
+      // No favicon is fine: the card falls back to a monogram.
+    }
+  }
+
+  return {
+    ok: true,
+    png,
+    title,
+    favicon,
+    lastModified: httpDate(response.headers()['last-modified']),
+  }
+}
+
+/** An HTTP date header as a Date; null when absent or unreadable. */
+export function httpDate(value: string | undefined): Date | null {
+  if (!value) return null
+  const date = new Date(value)
+  return Number.isNaN(date.getTime()) ? null : date
+}
+
+/** Runs captures one at a time, so concurrent previews cannot start a browser each. */
+export function oneAtATime(
+  capture: ScreenshotCapturer['capture'],
+): Required<ScreenshotCapturer> {
+  let queue: Promise<unknown> = Promise.resolve()
+  let waiting = 0
+  return {
+    waiting: () => waiting,
+    capture: (url, allowedHosts) => {
+      waiting++
+      const run = queue.then(() => {
+        waiting--
+        return capture(url, allowedHosts)
+      })
+      queue = run.catch(() => undefined)
+      return run
+    },
+  }
+}
+
+/**
+ * A capturer backed by the optional `playwright-core` peer, or null when it is not
+ * installed — the feature then works without screenshots.
+ */
+export function createPlaywrightCapturer(): ScreenshotCapturer | null {
+  try {
+    createRequire(import.meta.url).resolve('playwright-core')
+  } catch {
+    return null
+  }
+
+  return oneAtATime(async (url, allowedHosts) => {
+    const { chromium } = await import('playwright-core')
+    // Launched per capture because the fence flags depend on the hosts; the queue
+    // keeps that to one browser at a time.
+    const browser = await chromium.launch({
+      executablePath:
+        process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined,
+      args: chromiumFenceArgs(allowedHosts),
+    })
+    try {
+      // A service worker could answer requests the route never sees.
+      const context = await browser.newContext({
+        viewport: VIEWPORT,
+        serviceWorkers: 'block',
+      })
+      // Second layer under the resolver fence: per-request and per-hop checks.
+      await context.route('**/*', (route) =>
+        fenceRoute(
+          {
+            url: route.request().url(),
+            abort: (code) => route.abort(code),
+            continue: () => route.continue(),
+            fetch: (hop, mode) =>
+              mode === 'get'
+                ? context.request.get(hop, {
+                    maxRedirects: 0,
+                    timeout: HOP_TIMEOUT_MS,
+                  })
+                : route.fetch({
+                    url: hop,
+                    maxRedirects: 0,
+                    timeout: HOP_TIMEOUT_MS,
+                  }),
+            fulfill: (response) => route.fulfill({ response }),
+          },
+          allowedHosts,
+        ),
+      )
+      // A screenshot needs no live socket, and `route` does not see them.
+      await context.routeWebSocket(/.*/, (ws) => ws.close())
+      const page = await context.newPage()
+      return await capturePage(page, url, allowedHosts)
+    } catch (error) {
+      return {
+        ok: false,
+        error: error instanceof Error ? error.message : String(error),
+      }
+    } finally {
+      await browser.close()
+    }
+  })
+}

package/src/modules/communityTools/healthJob.ts

@@ -0,0 +1,191 @@
+import {
+  isCaptureAllowed,
+  nextLifecycle,
+} from '@igstack/app-catalog-shared-core'
+import { getDbClient } from '../../db/client'
+import { resizeImage } from '../assets/assetUtils'
+import { upsertAsset } from '../assets/upsertAsset'
+import type { CaptureResult, ScreenshotCapturer } from './capture'
+
+export interface CaptureDeps {
+  captureHosts: string[]
+  capturer: ScreenshotCapturer | null
+}
+
+const MINUTE_MS = 60 * 1000
+const HOUR_MS = 60 * MINUTE_MS
+const DAY_MS = 24 * HOUR_MS
+/** Wall time one run may spend; what is left is due again on the next tick. */
+const RUN_BUDGET_MS = 40 * MINUTE_MS
+/** Past the budget plus one capture, and short of the next tick. */
+const RUN_TRANSACTION_TIMEOUT_MS = 55 * MINUTE_MS
+const FIRST_RUN_DELAY_MS = MINUTE_MS
+
+interface CheckableTool {
+  id: string
+  slug: string
+  url: string
+  failingSince: Date | null
+  archivedAt: Date | null
+  archivedBy: string | null
+}
+
+/** Stores the screenshot and, when it decodes, the favicon as assets. */
+async function storeCapture(
+  slug: string,
+  result: Extract<CaptureResult, { ok: true }>,
+  now: Date,
+) {
+  const prisma = getDbClient()
+  const screenshotAssetId = await upsertAsset({
+    prisma,
+    buffer: result.png,
+    name: `community-${slug}-screenshot`,
+    originalFilename: `community-${slug}-screenshot.png`,
+    assetType: 'screenshot',
+  })
+
+  let faviconAssetId: string | undefined
+  if (result.favicon) {
+    try {
+      // ponytail: sharp cannot decode .ico, so those tools get no favicon; add an ico decoder if it matters
+      const png = await resizeImage(result.favicon, 64, 64, 'png')
+      faviconAssetId = await upsertAsset({
+        prisma,
+        buffer: png,
+        name: `community-${slug}-icon`,
+        originalFilename: `community-${slug}-icon.png`,
+        assetType: 'icon',
+      })
+    } catch {
+      // An undecodable favicon leaves the previous one, or none.
+    }
+  }
+
+  return {
+    screenshotAssetId,
+    ...(faviconAssetId && { faviconAssetId }),
+    capturedAt: now,
+  }
+}
+
+/**
+ * Visits one tool and records the result. Hosts outside the configured suffixes, and a
+ * deployment without a capturer, are left untouched: no check means no state change.
+ */
+export async function checkTool(
+  tool: CheckableTool,
+  deps: CaptureDeps,
+  now: Date,
+): Promise<void> {
+  if (!deps.capturer || !isCaptureAllowed(tool.url, deps.captureHosts)) return
+
+  let result: CaptureResult
+  try {
+    result = await deps.capturer.capture(tool.url, deps.captureHosts)
+  } catch (error) {
+    // A browser that would not start says nothing about the page, so the
+    // lifecycle stays as it was. The attempt is still recorded: without it a
+    // new tool reads "capture pending" forever and its open card keeps asking.
+    await getDbClient().dbCommunityTool.update({
+      where: { id: tool.id },
+      data: { lastCheckAt: now },
+    })
+    throw error
+  }
+  await getDbClient().dbCommunityTool.update({
+    where: { id: tool.id },
+    data: {
+      lastCheckAt: now,
+      ...nextLifecycle(tool, result.ok, now),
+      ...(result.ok && (await storeCapture(tool.slug, result, now))),
+      // A visit without the header leaves the date it last had.
+      ...(result.ok &&
+        result.lastModified && {
+          siteUpdatedAt: result.lastModified,
+        }),
+    },
+  })
+}
+
+/**
+ * One pass over every tool not checked in the last day.
+ *
+ * The transaction-scoped lock rather than `pg_try_advisory_lock`: a session lock is
+ * held by whichever pooled connection ran the query, and the unlock may run on another,
+ * leaving the lock stuck forever. The transaction pins one connection and releases the
+ * lock when it ends, even if the pod dies mid-run.
+ *
+ * ponytail: one transaction holds the lock for the whole run, so a run is capped at
+ * RUN_BUDGET_MS (about 90 tools at 25 s each); per-tool short transactions or a job
+ * queue if the catalog outgrows that.
+ */
+export async function runHealthCheckOnce(
+  deps: CaptureDeps,
+  now: Date,
+  clock: () => number = Date.now,
+): Promise<void> {
+  const db = getDbClient()
+  await db.$transaction(
+    async (tx) => {
+      const [row] = await tx.$queryRaw<{ locked: boolean }[]>`
+        SELECT pg_try_advisory_xact_lock(hashtext('community-tools-health')) AS locked`
+      if (!row?.locked) return
+
+      const cutoff = now.getTime() - DAY_MS
+      // ponytail: reads every tool each hour; move the filters into the query if the table grows large
+      const tools = await db.dbCommunityTool.findMany({
+        select: {
+          id: true,
+          slug: true,
+          url: true,
+          lastCheckAt: true,
+          failingSince: true,
+          archivedAt: true,
+          archivedBy: true,
+        },
+      })
+      const due = tools.filter(
+        (t) =>
+          !(t.archivedAt && t.archivedBy !== 'auto') &&
+          (!t.lastCheckAt || t.lastCheckAt.getTime() < cutoff),
+      )
+
+      const startedAt = clock()
+      for (const tool of due) {
+        if (clock() - startedAt >= RUN_BUDGET_MS) break
+        try {
+          await checkTool(tool, deps, now)
+        } catch (error) {
+          console.error(`[communityTools] health check of ${tool.slug}:`, error)
+        }
+      }
+    },
+    { maxWait: 10_000, timeout: RUN_TRANSACTION_TIMEOUT_MS },
+  )
+}
+
+/**
+ * Starts the pass a minute after boot, then hourly: a pod restarted more often than
+ * hourly would otherwise never check. The lock keeps pods from overlapping.
+ * Returns a stop function.
+ */
+export function startHealthJob(
+  deps: CaptureDeps,
+  now: () => Date = () => new Date(),
+): () => void {
+  const run = () => {
+    runHealthCheckOnce(deps, now()).catch((error: unknown) => {
+      console.error('[communityTools] health run failed:', error)
+    })
+  }
+  const first = setTimeout(run, FIRST_RUN_DELAY_MS)
+  const timer = setInterval(run, HOUR_MS)
+  // Neither keeps the process, or a test runner, alive.
+  first.unref()
+  timer.unref()
+  return () => {
+    clearTimeout(first)
+    clearInterval(timer)
+  }
+}