@learnpack/learnpack 5.0.351 → 5.0.353

package/src/utils/descriptions/mirrorDescriptions.ts

@@ -0,0 +1,191 @@
+import { Syllabus } from "../../models/creator"
+import { fingerprintReadme } from "../descriptionHash"
+import { DESCRIPTION_PROMPT_VERSION } from "../packageManifest"
+import {
+  CourseDescriptionsStorage,
+  readCourseReadmes,
+  DEFAULT_DESCRIPTIONS_CONCURRENCY,
+} from "./generateCourseDescriptions"
+import { CourseExercise, CourseReadmes, findSyllabusLesson } from "./workList"
+
+/**
+ * Copy descriptions from the published syllabus back into the draft one.
+ *
+ * Descriptions are generated against the published package because it is an
+ * immutable snapshot. The draft in GCS, meanwhile, keeps moving: between the
+ * publication and the moment the descriptions land, steps may have been added,
+ * removed, renamed, reordered or edited.
+ *
+ * The guard against that drift is the content hash, not a timestamp or a
+ * position: a description is copied only when the draft README hashes to the
+ * very content the description was generated from. Anything that moved simply
+ * misses and waits for the next publication — a stale description is never
+ * written onto content it does not describe.
+ *
+ * Without this mirror the draft would never hold a description, and every
+ * publication would regenerate the whole course from scratch: the freshness
+ * check reads the fingerprints stored in the syllabus that travels inside the
+ * zip.
+ */
+
+export type MirrorResult = {
+  /** (step, language) pairs copied into the draft. */
+  copied: number;
+  /** Pairs with no reusable description: content moved on, or none published. */
+  missed: number;
+  /** Pairs the draft already had up to date. */
+  fresh: number;
+};
+
+export type MirrorOptions = {
+  promptVersion?: number;
+  dryRun?: boolean;
+  concurrency?: number;
+};
+
+function emptyMirrorResult(): MirrorResult {
+  return { copied: 0, missed: 0, fresh: 0 }
+}
+
+/**
+ * Pure core: writes into `targetSyllabus` in place. Callers own the I/O, which
+ * keeps the drift rules testable without a bucket.
+ */
+export function mirrorDescriptionsIntoSyllabus(input: {
+  sourceSyllabus: Syllabus | null;
+  targetSyllabus: Syllabus;
+  exercises: CourseExercise[];
+  /** READMEs of the TARGET (draft), which is what must match the hash. */
+  readmes: CourseReadmes;
+  promptVersion?: number;
+}): MirrorResult {
+  const result = emptyMirrorResult()
+  if (!input.sourceSyllabus) {
+    return result
+  }
+
+  const promptVersion = input.promptVersion ?? DESCRIPTION_PROMPT_VERSION
+
+  for (const exercise of input.exercises) {
+    const targetLesson = findSyllabusLesson(
+      input.targetSyllabus,
+      exercise.slug
+    )
+    if (!targetLesson) {
+      // The step is not in the draft (removed, or renamed after publishing).
+      continue
+    }
+
+    const sourceLesson = findSyllabusLesson(
+      input.sourceSyllabus,
+      exercise.slug
+    )
+    const exerciseReadmes = input.readmes[exercise.slug] || {}
+
+    for (const lang of Object.keys(exercise.translations || {})) {
+      const content = exerciseReadmes[lang]
+      if (typeof content !== "string") {
+        continue
+      }
+
+      const fingerprint = fingerprintReadme(content)
+      const targetSlot = targetLesson.translations?.[lang]
+
+      if (targetSlot?.descriptionSource === "human") {
+        result.fresh += 1
+        continue
+      }
+
+      if (
+        targetSlot?.sourceContentHash === fingerprint.sha256 &&
+        (targetSlot?.descriptionPromptVersion ?? 0) >= promptVersion
+      ) {
+        result.fresh += 1
+        continue
+      }
+
+      const sourceSlot = sourceLesson?.translations?.[lang]
+      // The hash guard: only reuse a description generated from exactly this
+      // content. Note a published `description: null` is reusable too — it is a
+      // real answer ("not enough content to describe"), not a missing one.
+      const reusable =
+        sourceSlot !== undefined &&
+        sourceSlot.sourceContentHash === fingerprint.sha256 &&
+        (sourceSlot.descriptionPromptVersion ?? 0) >= promptVersion &&
+        sourceSlot.descriptionStatus !== "error"
+
+      if (!reusable) {
+        result.missed += 1
+        continue
+      }
+
+      if (!targetLesson.translations) {
+        targetLesson.translations = {}
+      }
+
+      if (!targetLesson.translations[lang]) {
+        targetLesson.translations[lang] = {
+          completionId: 0,
+          startedAt: Date.now(),
+        }
+      }
+
+      const destination = targetLesson.translations[lang]
+      destination.description = sourceSlot.description ?? null
+      destination.descriptionStatus = sourceSlot.description ?
+        "generated" :
+        "none"
+      destination.descriptionSource = "auto"
+      destination.descriptionPromptVersion =
+        sourceSlot.descriptionPromptVersion ?? promptVersion
+      destination.sourceContentHash = fingerprint.sha256
+      destination.sourceSimHash = fingerprint.simhash
+      result.copied += 1
+    }
+  }
+
+  return result
+}
+
+/** Read the draft, mirror into it, and save it when something changed. */
+export async function mirrorDescriptionsToDraft(
+  storage: CourseDescriptionsStorage,
+  courseSlug: string,
+  sourceSyllabus: Syllabus | null,
+  options: MirrorOptions = {}
+): Promise<MirrorResult> {
+  if (!sourceSyllabus) {
+    return emptyMirrorResult()
+  }
+
+  const targetSyllabus = await storage.readSyllabus(courseSlug)
+  if (!targetSyllabus) {
+    return emptyMirrorResult()
+  }
+
+  const exercises = await storage.listExercises(courseSlug)
+  if (exercises.length === 0) {
+    return emptyMirrorResult()
+  }
+
+  const readmes = await readCourseReadmes(
+    storage,
+    courseSlug,
+    exercises,
+    options.concurrency ?? DEFAULT_DESCRIPTIONS_CONCURRENCY
+  )
+
+  const result = mirrorDescriptionsIntoSyllabus({
+    sourceSyllabus,
+    targetSyllabus,
+    exercises,
+    readmes,
+    promptVersion: options.promptVersion,
+  })
+
+  if (result.copied > 0 && !options.dryRun) {
+    await storage.writeSyllabus(courseSlug, targetSyllabus)
+  }
+
+  return result
+}

package/src/utils/descriptions/publishStage.ts

@@ -0,0 +1,382 @@
+import * as fs from "fs"
+import * as path from "path"
+import { Bucket } from "@google-cloud/storage"
+import { Syllabus } from "../../models/creator"
+import {
+  finalizeJournal,
+  JournalStorage,
+  markStage,
+  PublishJournal,
+} from "../publishJournal"
+import {
+  CourseDescriptionsStorage,
+  generateCourseDescriptions,
+  GenerateCourseDescriptionsOptions,
+  GenerateCourseDescriptionsResult,
+} from "./generateCourseDescriptions"
+import { createGcsDescriptionsStorage } from "./gcsStorage"
+import { mirrorDescriptionsToDraft, MirrorResult } from "./mirrorDescriptions"
+import { createS3DescriptionsStorageFromEnv } from "./s3Storage"
+import {
+  buildPackageManifestUpdatedEvent,
+  PublishEventContext,
+  sendPublishEvent,
+} from "../publishEvents"
+import {
+  buildDescriptionWorkList,
+  countWorkItems,
+  CourseExercise,
+  CourseReadmes,
+} from "./workList"
+
+/**
+ * Stage B of a publication: generate the descriptions the new content needs.
+ *
+ * It runs in the background, after the HTTP response, because a single
+ * completion takes ~15s and a course may need several — far past any request
+ * limit. The publication itself is already finished and durable by then; this
+ * stage only enriches the published package.
+ */
+
+/**
+ * Whether the freshly built package needs any description, decided from the
+ * build directory that is still on disk after zipping.
+ *
+ * Reading from disk (instead of re-reading GCS or S3) is free and, at this
+ * exact moment, all three hold the same content: the answer is what the publish
+ * response promises to breathecode, so it must be cheap enough to compute
+ * before responding.
+ */
+/** Reads a path relative to the build root; null when it does not exist. */
+export type BuildFileReader = (relativePath: string) => string | null;
+
+export function createBuildFileReader(buildRoot: string): BuildFileReader {
+  return relativePath => {
+    try {
+      return fs.readFileSync(
+        path.join(buildRoot, ...relativePath.split("/")),
+        "utf8"
+      )
+    } catch {
+      return null
+    }
+  }
+}
+
+export function countPendingDescriptions(
+  buildRoot: string,
+  exercises: CourseExercise[],
+  readFile: BuildFileReader = createBuildFileReader(buildRoot)
+): number {
+  const rawSyllabus = readFile(".learn/initialSyllabus.json")
+  if (!rawSyllabus) {
+    return 0
+  }
+
+  let syllabus: Syllabus
+  try {
+    syllabus = JSON.parse(rawSyllabus) as Syllabus
+  } catch {
+    return 0
+  }
+
+  const readmes: CourseReadmes = {}
+  for (const exercise of exercises) {
+    for (const [lang, fileName] of Object.entries(
+      exercise.translations || {}
+    )) {
+      const content = readFile(`exercises/${exercise.slug}/${fileName}`)
+      if (content === null) {
+        // Missing translation file: nothing to fingerprint, nothing to ask for.
+        continue
+      }
+
+      if (!readmes[exercise.slug]) {
+        readmes[exercise.slug] = {}
+      }
+
+      readmes[exercise.slug][lang] = content
+    }
+  }
+
+  return countWorkItems(
+    buildDescriptionWorkList({ syllabus, exercises, readmes })
+  )
+}
+
+export type PublishDescriptionsStageResult = {
+  generation: GenerateCourseDescriptionsResult | null;
+  mirror: MirrorResult | null;
+  /** False when generation failed outright; drives the event status. */
+  succeeded: boolean;
+};
+
+/**
+ * Generate against the published package, then mirror what still matches back
+ * into the draft. Stages are recorded as they go so an interrupted run can be
+ * resumed by the sweep.
+ */
+export async function runPublishDescriptionsStage(params: {
+  courseSlug: string;
+  rigobotToken: string;
+  gcsBucket?: Bucket;
+  journal: PublishJournal;
+  journalStorage: JournalStorage;
+  /** Injection points for tests; production resolves them from env/bucket. */
+  publishedStorage?: CourseDescriptionsStorage;
+  draftStorage?: CourseDescriptionsStorage;
+  generate?: GenerateCourseDescriptionsOptions["generate"];
+}): Promise<PublishDescriptionsStageResult> {
+  const { courseSlug, journal, journalStorage } = params
+  const publishedStorage =
+    params.publishedStorage ?? createS3DescriptionsStorageFromEnv()
+
+  await markStage(journalStorage, journal, "descriptions", "running")
+
+  let generation: GenerateCourseDescriptionsResult | null = null
+  try {
+    generation = await generateCourseDescriptions(
+      publishedStorage,
+      courseSlug,
+      {
+        token: params.rigobotToken,
+        reconcile: true,
+        generate: params.generate,
+      }
+    )
+  } catch (error) {
+    await markStage(journalStorage, journal, "descriptions", "failed", {
+      error: (error as Error).message,
+    })
+    return { generation: null, mirror: null, succeeded: false }
+  }
+
+  const succeeded = generation.status !== "failed"
+  await markStage(
+    journalStorage,
+    journal,
+    "descriptions",
+    succeeded ? "done" : "failed",
+    {
+      error: succeeded ? undefined : generation.errors.join("; "),
+      meta: {
+        steps: generation.steps,
+        generated: generation.generated,
+        failed: generation.failed,
+        missing: generation.missing,
+        durationsSeconds: generation.durationsSeconds,
+      },
+    }
+  )
+
+  if (!succeeded) {
+    return { generation, mirror: null, succeeded: false }
+  }
+
+  // Mirror back so the draft keeps the fingerprints: without it the next
+  // publication would regenerate the whole course.
+  await markStage(journalStorage, journal, "gcsMirror", "running")
+  try {
+    const draftStorage =
+      params.draftStorage ??
+      createGcsDescriptionsStorage(params.gcsBucket as Bucket)
+    const publishedSyllabus = await publishedStorage.readSyllabus(courseSlug)
+    const mirror = await mirrorDescriptionsToDraft(
+      draftStorage,
+      courseSlug,
+      publishedSyllabus
+    )
+    await markStage(journalStorage, journal, "gcsMirror", "done", {
+      meta: mirror,
+    })
+    return { generation, mirror, succeeded: true }
+  } catch (error) {
+    // The published package is already correct; only the draft lags behind.
+    await markStage(journalStorage, journal, "gcsMirror", "failed", {
+      error: (error as Error).message,
+    })
+    return { generation, mirror: null, succeeded: true }
+  }
+}
+
+/**
+ * Entry point for the detached run after publishing. Never throws: it is not
+ * awaited by anyone, so an escaping rejection would only surface as an
+ * unhandled promise.
+ */
+export async function runDescriptionsStageAfterPublish(params: {
+  courseSlug: string;
+  gcsBucket?: Bucket;
+  journal: PublishJournal;
+  journalStorage: JournalStorage;
+  hasWork: boolean;
+  /** Present when the caller announced `descriptions: "queued"`. */
+  eventContext?: PublishEventContext;
+  breathecodeToken?: string;
+  publishedStorage?: CourseDescriptionsStorage;
+  draftStorage?: CourseDescriptionsStorage;
+  generate?: GenerateCourseDescriptionsOptions["generate"];
+  /** Injection point for tests. */
+  emit?: typeof sendPublishEvent;
+}): Promise<PublishDescriptionsStageResult> {
+  const { courseSlug, journal, journalStorage } = params
+  let outcome: PublishDescriptionsStageResult = {
+    generation: null,
+    mirror: null,
+    succeeded: true,
+  }
+
+  // Resolved on demand: the common "nothing to describe" path must not build
+  // AWS clients it will never use.
+  let resolved = params.publishedStorage
+  const publishedStorage = () => {
+    if (!resolved) {
+      resolved = createS3DescriptionsStorageFromEnv()
+    }
+
+    return resolved
+  }
+
+  try {
+    if (!params.hasWork) {
+      // Nothing changed enough to describe; both stages are settled, and no
+      // second event was promised, so none is sent.
+      await markStage(journalStorage, journal, "descriptions", "done", {
+        meta: { reason: "up_to_date" },
+      })
+      await markStage(journalStorage, journal, "gcsMirror", "done", {
+        meta: { reason: "up_to_date" },
+      })
+      await markStage(journalStorage, journal, "manifestEvent", "done", {
+        meta: { reason: "not_promised" },
+      })
+      return outcome
+    }
+
+    // Descriptions are platform enrichment, not something the publisher is
+    // billed for, so they always run under the service account.
+    const token = process.env.RIGOBOT_SYSTEM_TOKEN
+    if (!token) {
+      await markStage(journalStorage, journal, "descriptions", "failed", {
+        error: "RIGOBOT_SYSTEM_TOKEN is not configured",
+      })
+      console.error(
+        `[descriptions] Skipping stage for "${courseSlug}": RIGOBOT_SYSTEM_TOKEN is not configured`
+      )
+      outcome = { generation: null, mirror: null, succeeded: false }
+      return outcome
+    }
+
+    outcome = await runPublishDescriptionsStage({
+      courseSlug,
+      rigobotToken: token,
+      gcsBucket: params.gcsBucket,
+      journal,
+      journalStorage,
+      publishedStorage: publishedStorage(),
+      draftStorage: params.draftStorage,
+      generate: params.generate,
+    })
+
+    if (outcome.succeeded) {
+      console.log(
+        `[descriptions] "${courseSlug}": ${
+          outcome.generation?.generated ?? 0
+        } description(s), ${outcome.mirror?.copied ?? 0} mirrored`
+      )
+    } else {
+      console.error(
+        `[descriptions] "${courseSlug}": stage failed${
+          outcome.generation?.errors.length ?
+            ` (${outcome.generation.errors.join("; ")})` :
+            ""
+        }`
+      )
+    }
+
+    return outcome
+  } catch (error) {
+    console.error(
+      `[descriptions] Stage failed for "${courseSlug}":`,
+      (error as Error).message
+    )
+    outcome = { generation: null, mirror: null, succeeded: false }
+    return outcome
+  } finally {
+    // The contract: a publication that announced `descriptions: "queued"` is
+    // always followed by exactly one manifest event, success or failure.
+    // Emitting from `finally` is what makes "exactly one, on every exit path"
+    // true by construction rather than by remembering to call it.
+    if (params.hasWork) {
+      await emitManifestUpdated({
+        courseSlug,
+        journal,
+        journalStorage,
+        outcome,
+        eventContext: params.eventContext,
+        breathecodeToken: params.breathecodeToken,
+        publishedStorage: publishedStorage(),
+        emit: params.emit ?? sendPublishEvent,
+      })
+    }
+
+    await finalizeJournal(journalStorage, journal)
+  }
+}
+
+async function emitManifestUpdated(params: {
+  courseSlug: string;
+  journal: PublishJournal;
+  journalStorage: JournalStorage;
+  outcome: PublishDescriptionsStageResult;
+  eventContext?: PublishEventContext;
+  breathecodeToken?: string;
+  publishedStorage: CourseDescriptionsStorage;
+  emit: typeof sendPublishEvent;
+}): Promise<void> {
+  const { journal, journalStorage } = params
+
+  if (!params.eventContext || !params.breathecodeToken) {
+    // Nothing was announced to breathecode, so nothing is owed.
+    await markStage(journalStorage, journal, "manifestEvent", "done", {
+      meta: { reason: "no_event_context" },
+    })
+    return
+  }
+
+  try {
+    // Report the manifest as it is NOW: the descriptions are the whole point of
+    // this event, so the one captured at publish time would be stale.
+    const updatedManifest =
+      (await params.publishedStorage.readManifest?.(params.courseSlug)) ??
+      params.eventContext.manifest
+
+    const generation = params.outcome.generation
+    const delivered = await params.emit(
+      buildPackageManifestUpdatedEvent(
+        { ...params.eventContext, manifest: updatedManifest },
+        params.outcome.succeeded ? "success" : "failed",
+        generation ?
+          {
+              generated: generation.generated,
+              failed: generation.failed,
+              missing: generation.missing,
+            } :
+          undefined
+      ),
+      params.breathecodeToken
+    )
+
+    await markStage(
+      journalStorage,
+      journal,
+      "manifestEvent",
+      delivered ? "done" : "failed",
+      { error: delivered ? undefined : "delivery failed" }
+    )
+  } catch (error) {
+    await markStage(journalStorage, journal, "manifestEvent", "failed", {
+      error: (error as Error).message,
+    })
+  }
+}