@learnpack/learnpack 5.0.352 → 5.0.354

package/src/utils/descriptions/publishStage.ts

@@ -0,0 +1,394 @@
+import * as fs from "fs"
+import * as path from "path"
+import { Bucket } from "@google-cloud/storage"
+import { Syllabus } from "../../models/creator"
+import {
+  finalizeJournal,
+  JournalStorage,
+  markStage,
+  PublishJournal,
+} from "../publishJournal"
+import {
+  CourseDescriptionsStorage,
+  generateCourseDescriptions,
+  GenerateCourseDescriptionsOptions,
+  GenerateCourseDescriptionsResult,
+} from "./generateCourseDescriptions"
+import { createGcsDescriptionsStorage } from "./gcsStorage"
+import { mirrorDescriptionsToDraft, MirrorResult } from "./mirrorDescriptions"
+import { createS3DescriptionsStorageFromEnv } from "./s3Storage"
+import {
+  buildPackageManifestUpdatedEvent,
+  PublishEventContext,
+  sendPublishEvent,
+} from "../publishEvents"
+import {
+  buildDescriptionWorkList,
+  countWorkItems,
+  CourseExercise,
+  CourseReadmes,
+} from "./workList"
+
+/**
+ * Stage B of a publication: generate the descriptions the new content needs.
+ *
+ * It runs in the background, after the HTTP response, because a single
+ * completion takes ~15s and a course may need several — far past any request
+ * limit. The publication itself is already finished and durable by then; this
+ * stage only enriches the published package.
+ */
+
+/**
+ * Whether the freshly built package needs any description, decided from the
+ * build directory that is still on disk after zipping.
+ *
+ * Reading from disk (instead of re-reading GCS or S3) is free and, at this
+ * exact moment, all three hold the same content: the answer is what the publish
+ * response promises to breathecode, so it must be cheap enough to compute
+ * before responding.
+ */
+/** Reads a path relative to the build root; null when it does not exist. */
+export type BuildFileReader = (relativePath: string) => string | null;
+
+export function createBuildFileReader(buildRoot: string): BuildFileReader {
+  return relativePath => {
+    try {
+      return fs.readFileSync(
+        path.join(buildRoot, ...relativePath.split("/")),
+        "utf8"
+      )
+    } catch {
+      return null
+    }
+  }
+}
+
+export function countPendingDescriptions(
+  buildRoot: string,
+  exercises: CourseExercise[],
+  readFile: BuildFileReader = createBuildFileReader(buildRoot)
+): number {
+  const rawSyllabus = readFile(".learn/initialSyllabus.json")
+  if (!rawSyllabus) {
+    return 0
+  }
+
+  let syllabus: Syllabus
+  try {
+    syllabus = JSON.parse(rawSyllabus) as Syllabus
+  } catch {
+    return 0
+  }
+
+  const readmes: CourseReadmes = {}
+  for (const exercise of exercises) {
+    for (const [lang, fileName] of Object.entries(
+      exercise.translations || {}
+    )) {
+      const content = readFile(`exercises/${exercise.slug}/${fileName}`)
+      if (content === null) {
+        // Missing translation file: nothing to fingerprint, nothing to ask for.
+        continue
+      }
+
+      if (!readmes[exercise.slug]) {
+        readmes[exercise.slug] = {}
+      }
+
+      readmes[exercise.slug][lang] = content
+    }
+  }
+
+  return countWorkItems(
+    buildDescriptionWorkList({ syllabus, exercises, readmes })
+  )
+}
+
+export type PublishDescriptionsStageResult = {
+  generation: GenerateCourseDescriptionsResult | null;
+  mirror: MirrorResult | null;
+  /** False when generation failed outright; drives the event status. */
+  succeeded: boolean;
+};
+
+/**
+ * Generate against the published package, then mirror what still matches back
+ * into the draft. Stages are recorded as they go so an interrupted run can be
+ * resumed by the sweep.
+ */
+export async function runPublishDescriptionsStage(params: {
+  courseSlug: string;
+  rigobotToken: string;
+  gcsBucket?: Bucket;
+  journal: PublishJournal;
+  journalStorage: JournalStorage;
+  /** Injection points for tests; production resolves them from env/bucket. */
+  publishedStorage?: CourseDescriptionsStorage;
+  draftStorage?: CourseDescriptionsStorage;
+  generate?: GenerateCourseDescriptionsOptions["generate"];
+}): Promise<PublishDescriptionsStageResult> {
+  const { courseSlug, journal, journalStorage } = params
+  const publishedStorage =
+    params.publishedStorage ?? createS3DescriptionsStorageFromEnv()
+
+  await markStage(journalStorage, journal, "descriptions", "running")
+
+  let generation: GenerateCourseDescriptionsResult | null = null
+  try {
+    generation = await generateCourseDescriptions(
+      publishedStorage,
+      courseSlug,
+      {
+        token: params.rigobotToken,
+        reconcile: true,
+        generate: params.generate,
+      }
+    )
+  } catch (error) {
+    await markStage(journalStorage, journal, "descriptions", "failed", {
+      error: (error as Error).message,
+    })
+    return { generation: null, mirror: null, succeeded: false }
+  }
+
+  const succeeded = generation.status !== "failed"
+  await markStage(
+    journalStorage,
+    journal,
+    "descriptions",
+    succeeded ? "done" : "failed",
+    {
+      error: succeeded ? undefined : generation.errors.join("; "),
+      meta: {
+        steps: generation.steps,
+        generated: generation.generated,
+        failed: generation.failed,
+        missing: generation.missing,
+        durationsSeconds: generation.durationsSeconds,
+      },
+    }
+  )
+
+  if (!succeeded) {
+    return { generation, mirror: null, succeeded: false }
+  }
+
+  // Mirror back so the draft keeps the fingerprints: without it the next
+  // publication would regenerate the whole course.
+  await markStage(journalStorage, journal, "gcsMirror", "running")
+  try {
+    const draftStorage =
+      params.draftStorage ??
+      createGcsDescriptionsStorage(params.gcsBucket as Bucket)
+    const publishedSyllabus = await publishedStorage.readSyllabus(courseSlug)
+    const mirror = await mirrorDescriptionsToDraft(
+      draftStorage,
+      courseSlug,
+      publishedSyllabus
+    )
+    await markStage(journalStorage, journal, "gcsMirror", "done", {
+      meta: mirror,
+    })
+    return { generation, mirror, succeeded: true }
+  } catch (error) {
+    // The published package is already correct; only the draft lags behind.
+    await markStage(journalStorage, journal, "gcsMirror", "failed", {
+      error: (error as Error).message,
+    })
+    return { generation, mirror: null, succeeded: true }
+  }
+}
+
+/**
+ * Entry point for the detached run after publishing. Never throws: it is not
+ * awaited by anyone, so an escaping rejection would only surface as an
+ * unhandled promise.
+ */
+export async function runDescriptionsStageAfterPublish(params: {
+  courseSlug: string;
+  gcsBucket?: Bucket;
+  journal: PublishJournal;
+  journalStorage: JournalStorage;
+  hasWork: boolean;
+  /** Present when the caller announced `descriptions: "queued"`. */
+  eventContext?: PublishEventContext;
+  breathecodeToken?: string;
+  publishedStorage?: CourseDescriptionsStorage;
+  draftStorage?: CourseDescriptionsStorage;
+  generate?: GenerateCourseDescriptionsOptions["generate"];
+  /** Injection point for tests. */
+  emit?: typeof sendPublishEvent;
+}): Promise<PublishDescriptionsStageResult> {
+  const { courseSlug, journal, journalStorage } = params
+  let outcome: PublishDescriptionsStageResult = {
+    generation: null,
+    mirror: null,
+    succeeded: true,
+  }
+
+  // Resolved on demand: the common "nothing to describe" path must not build
+  // AWS clients it will never use.
+  let resolved = params.publishedStorage
+  const publishedStorage = () => {
+    if (!resolved) {
+      resolved = createS3DescriptionsStorageFromEnv()
+    }
+
+    return resolved
+  }
+
+  try {
+    if (!params.hasWork) {
+      // Nothing changed enough to describe; both stages are settled, and no
+      // second event was promised, so none is sent.
+      await markStage(journalStorage, journal, "descriptions", "done", {
+        meta: { reason: "up_to_date" },
+      })
+      await markStage(journalStorage, journal, "gcsMirror", "done", {
+        meta: { reason: "up_to_date" },
+      })
+      await markStage(journalStorage, journal, "manifestEvent", "done", {
+        meta: { reason: "not_promised" },
+      })
+      return outcome
+    }
+
+    // Descriptions are platform enrichment, not something the publisher is
+    // billed for, so they always run under the service account.
+    const token = process.env.RIGOBOT_SYSTEM_TOKEN
+    if (!token) {
+      await markStage(journalStorage, journal, "descriptions", "failed", {
+        error: "RIGOBOT_SYSTEM_TOKEN is not configured",
+      })
+      console.error(
+        `[descriptions] Skipping stage for "${courseSlug}": RIGOBOT_SYSTEM_TOKEN is not configured`
+      )
+      outcome = { generation: null, mirror: null, succeeded: false }
+      return outcome
+    }
+
+    outcome = await runPublishDescriptionsStage({
+      courseSlug,
+      rigobotToken: token,
+      gcsBucket: params.gcsBucket,
+      journal,
+      journalStorage,
+      publishedStorage: publishedStorage(),
+      draftStorage: params.draftStorage,
+      generate: params.generate,
+    })
+
+    if (outcome.succeeded) {
+      console.log(
+        `[descriptions] "${courseSlug}": ${
+          outcome.generation?.generated ?? 0
+        } description(s), ${outcome.mirror?.copied ?? 0} mirrored`
+      )
+    } else {
+      console.error(
+        `[descriptions] "${courseSlug}": stage failed${
+          outcome.generation?.errors.length ?
+            ` (${outcome.generation.errors.join("; ")})` :
+            ""
+        }`
+      )
+    }
+
+    return outcome
+  } catch (error) {
+    console.error(
+      `[descriptions] Stage failed for "${courseSlug}":`,
+      (error as Error).message
+    )
+    outcome = { generation: null, mirror: null, succeeded: false }
+    return outcome
+  } finally {
+    // The contract: a publication that announced `descriptions: "queued"` is
+    // always followed by exactly one manifest event, success or failure.
+    // Emitting from `finally` is what makes "exactly one, on every exit path"
+    // true by construction rather than by remembering to call it.
+    if (params.hasWork) {
+      await emitManifestUpdated({
+        courseSlug,
+        journal,
+        journalStorage,
+        outcome,
+        eventContext: params.eventContext,
+        breathecodeToken: params.breathecodeToken,
+        publishedStorage: publishedStorage(),
+        emit: params.emit ?? sendPublishEvent,
+      })
+    }
+
+    await finalizeJournal(journalStorage, journal)
+  }
+}
+
+async function emitManifestUpdated(params: {
+  courseSlug: string;
+  journal: PublishJournal;
+  journalStorage: JournalStorage;
+  outcome: PublishDescriptionsStageResult;
+  eventContext?: PublishEventContext;
+  breathecodeToken?: string;
+  publishedStorage: CourseDescriptionsStorage;
+  emit: typeof sendPublishEvent;
+}): Promise<void> {
+  const { journal, journalStorage } = params
+
+  if (!params.eventContext || !params.breathecodeToken) {
+    // Nothing was announced to breathecode, so nothing is owed.
+    await markStage(journalStorage, journal, "manifestEvent", "done", {
+      meta: { reason: "no_event_context" },
+    })
+    return
+  }
+
+  // Enriching the payload must never prevent the delivery. The manifest is
+  // optional in the contract (`manifest: null` is valid); the event is not:
+  // breathecode was promised one and is waiting for it. Reading the manifest
+  // from the published bucket can fail for reasons that have nothing to do with
+  // the event — a bad AWS credential, say — and an event with less context still
+  // beats an event that never arrives.
+  let updatedManifest = params.eventContext.manifest
+  try {
+    updatedManifest =
+      (await params.publishedStorage.readManifest?.(params.courseSlug)) ??
+      params.eventContext.manifest
+  } catch (error) {
+    console.error(
+      `[descriptions] Could not read the updated manifest for "${params.courseSlug}", reporting without it:`,
+      (error as Error).message
+    )
+  }
+
+  try {
+    const generation = params.outcome.generation
+    const delivered = await params.emit(
+      buildPackageManifestUpdatedEvent(
+        { ...params.eventContext, manifest: updatedManifest },
+        params.outcome.succeeded ? "success" : "failed",
+        generation ?
+          {
+              generated: generation.generated,
+              failed: generation.failed,
+              missing: generation.missing,
+            } :
+          undefined
+      ),
+      params.breathecodeToken
+    )
+
+    await markStage(
+      journalStorage,
+      journal,
+      "manifestEvent",
+      delivered ? "done" : "failed",
+      { error: delivered ? undefined : "delivery failed" }
+    )
+  } catch (error) {
+    await markStage(journalStorage, journal, "manifestEvent", "failed", {
+      error: (error as Error).message,
+    })
+  }
+}

package/src/utils/descriptions/resumePublication.ts

@@ -0,0 +1,217 @@
+import {
+  abandonJournal,
+  finalizeJournal,
+  incrementAttempts,
+  isJournalStale,
+  JournalStorage,
+  markStage,
+  PublishJournal,
+} from "../publishJournal"
+import { PackageManifest } from "../packageManifest"
+import {
+  buildPackageManifestUpdatedEvent,
+  fetchPackageInfo,
+  sendPublishEvent,
+} from "../publishEvents"
+import {
+  CourseDescriptionsStorage,
+  GenerateCourseDescriptionsOptions,
+} from "./generateCourseDescriptions"
+import { runPublishDescriptionsStage } from "./publishStage"
+
+/**
+ * Recovery of a publication whose background stage never finished.
+ *
+ * The stage runs inside the web process, so a dyno restart mid-flight leaves a
+ * package published but not described, and — worse — breathecode waiting for a
+ * manifest event that was promised and will never arrive. Content-wise the
+ * missing descriptions would eventually be picked up by the next publication,
+ * but the promised event has no such second chance: emitting it is the reason
+ * this exists.
+ */
+
+export const MAX_SWEEP_ATTEMPTS = 3
+
+/** Only resume publications nobody is working on any more. */
+export const DEFAULT_STALE_AFTER_MS = 15 * 60 * 1000
+
+export type ResumeOutcome =
+  | "skipped-recent"
+  | "skipped-abandoned"
+  | "abandoned"
+  | "resumed"
+  | "nothing-to-do";
+
+export type ResumeDeps = {
+  journalStorage: JournalStorage;
+  publishedStorage: CourseDescriptionsStorage;
+  draftStorage: CourseDescriptionsStorage;
+  rigobotToken: string;
+  breathecodeToken?: string;
+  staleAfterMs?: number;
+  now?: number;
+  /** Injection points for tests. */
+  emit?: typeof sendPublishEvent;
+  readPackageInfo?: typeof fetchPackageInfo;
+  generate?: GenerateCourseDescriptionsOptions["generate"];
+};
+
+function isDone(
+  journal: PublishJournal,
+  stage: "descriptions" | "gcsMirror" | "manifestEvent"
+): boolean {
+  return journal.stages[stage].status === "done"
+}
+
+/**
+ * Whether breathecode is still owed a manifest event: one was promised (the
+ * publish event announced `queued`) and it never went out.
+ */
+export function owesManifestEvent(journal: PublishJournal): boolean {
+  const promised =
+    (
+      journal.stages.publishedEvent.meta as
+        | { descriptions?: string }
+        | undefined
+    )?.descriptions === "queued"
+  return promised && !isDone(journal, "manifestEvent")
+}
+
+export async function resumePublication(
+  journal: PublishJournal,
+  deps: ResumeDeps
+): Promise<ResumeOutcome> {
+  const { journalStorage } = deps
+  const now = deps.now ?? Date.now()
+
+  if (journal.abandoned) {
+    return "skipped-abandoned"
+  }
+
+  // A publication still in flight would race with its own stage.
+  if (
+    !isJournalStale(journal, deps.staleAfterMs ?? DEFAULT_STALE_AFTER_MS, now)
+  ) {
+    return "skipped-recent"
+  }
+
+  const needsDescriptions =
+    !isDone(journal, "descriptions") || !isDone(journal, "gcsMirror")
+  const needsEvent = owesManifestEvent(journal)
+
+  if (!needsDescriptions && !needsEvent) {
+    // Nothing left that the sweep can do; stage A failures are not its job.
+    await finalizeJournal(journalStorage, journal)
+    return "nothing-to-do"
+  }
+
+  if (journal.attempts >= MAX_SWEEP_ATTEMPTS) {
+    await abandonJournal(journalStorage, journal)
+    console.error(
+      `[sweep] Giving up on "${journal.courseSlug}" (${journal.publishId}) after ${journal.attempts} attempts`
+    )
+    return "abandoned"
+  }
+
+  await incrementAttempts(journalStorage, journal)
+
+  let succeeded = true
+  let stats: { generated: number; failed: number; missing: number } | undefined
+
+  if (needsDescriptions) {
+    const outcome = await runPublishDescriptionsStage({
+      courseSlug: journal.courseSlug,
+      rigobotToken: deps.rigobotToken,
+      journal,
+      journalStorage,
+      publishedStorage: deps.publishedStorage,
+      draftStorage: deps.draftStorage,
+      generate: deps.generate,
+    })
+    succeeded = outcome.succeeded
+    if (outcome.generation) {
+      stats = {
+        generated: outcome.generation.generated,
+        failed: outcome.generation.failed,
+        missing: outcome.generation.missing,
+      }
+    }
+  }
+
+  if (needsEvent) {
+    await emitPendingManifestEvent(journal, deps, succeeded, stats)
+  }
+
+  await finalizeJournal(journalStorage, journal)
+  return "resumed"
+}
+
+async function emitPendingManifestEvent(
+  journal: PublishJournal,
+  deps: ResumeDeps,
+  succeeded: boolean,
+  stats?: { generated: number; failed: number; missing: number }
+): Promise<void> {
+  const emit = deps.emit ?? sendPublishEvent
+  const readPackageInfo = deps.readPackageInfo ?? fetchPackageInfo
+
+  if (!deps.breathecodeToken) {
+    await markStage(deps.journalStorage, journal, "manifestEvent", "failed", {
+      error: "BREATHECODE_SYSTEM_TOKEN is not configured",
+    })
+    return
+  }
+
+  // Enriching the payload must never prevent the delivery: this event is the
+  // one breathecode has been waiting for since the publication stalled, and
+  // both the manifest and the package info are optional in the contract.
+  let manifest: PackageManifest | null = null
+  let packageInfo: Record<string, unknown> | null = null
+  try {
+    if (deps.publishedStorage.readManifest) {
+      manifest = await deps.publishedStorage.readManifest(journal.courseSlug)
+    }
+  } catch (error) {
+    console.error(
+      `[sweep] Could not read the manifest of "${journal.courseSlug}", reporting without it:`,
+      (error as Error).message
+    )
+  }
+
+  try {
+    packageInfo = await readPackageInfo(journal.courseSlug, deps.rigobotToken)
+  } catch (error) {
+    console.error(
+      `[sweep] Could not read the package of "${journal.courseSlug}", reporting without it:`,
+      (error as Error).message
+    )
+  }
+
+  try {
+    const delivered = await emit(
+      buildPackageManifestUpdatedEvent(
+        {
+          publishId: journal.publishId,
+          courseSlug: journal.courseSlug,
+          packageInfo,
+          manifest,
+        },
+        succeeded ? "success" : "failed",
+        stats
+      ),
+      deps.breathecodeToken
+    )
+
+    await markStage(
+      deps.journalStorage,
+      journal,
+      "manifestEvent",
+      delivered ? "done" : "failed",
+      { error: delivered ? undefined : "delivery failed" }
+    )
+  } catch (error) {
+    await markStage(deps.journalStorage, journal, "manifestEvent", "failed", {
+      error: (error as Error).message,
+    })
+  }
+}