@learnpack/learnpack 5.0.352 → 5.0.353

package/src/utils/packageManifest.ts

@@ -13,6 +13,14 @@ const frontMatter = require("front-matter")
 export const PACKAGE_MANIFEST_FILENAME = "package-manifest.json"
 export const PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json"
 export const SCHEMA_VERSION = 1
+// Version of the Rigobot prompt/model used to generate step descriptions. It
+// travels inside the completion inputs, so bumping it both busts Rigobot's
+// cache and marks every description stored at a lower version for regeneration.
+// Keep in sync with the prompt body provisioned in Rigobot.
+export const DESCRIPTION_PROMPT_VERSION = 1
+
+// Target length, in words, of a generated step description.
+export const DESCRIPTION_TARGET_WORD_COUNT = 25
 
 export type LessonType = "READ" | "CODE" | "QUIZ";
 
@@ -167,13 +175,18 @@ function resolveLessonTitle(
 }
 
 function resolveLessonDescriptions(
-  exercise: Exercise
+  exercise: Exercise,
+  syllabusLesson: Lesson | undefined
 ): Record<string, string | null> {
   const description: Record<string, string | null> = {}
 
+  // Descriptions are a projection: the source of truth lives in the syllabus
+  // lesson (translations[lang].description), populated by the sweep/backfill.
+  // When absent we emit null (the frontend hides the area) rather than a
+  // low-quality placeholder.
   for (const lang of Object.keys(exercise.translations)) {
-    // TODO: add description content
-    description[lang] = null
+    description[lang] =
+      syllabusLesson?.translations?.[lang]?.description ?? null
   }
 
   return description
@@ -240,7 +253,7 @@ export function buildPackageManifestFromSources(
         courseLang,
         readmeFrontmatters[exercise.slug] ?? {}
       ),
-      description: resolveLessonDescriptions(exercise),
+      description: resolveLessonDescriptions(exercise, syllabusLesson),
       video: resolveLessonVideos(exercise, readmeFrontmatters),
     }
   })
@@ -312,6 +325,43 @@ export function serializePackageManifest(manifest: PackageManifest): string {
   return JSON.stringify(manifest, null, 2)
 }
 
+/** The manifest as it travels inside a publication event. */
+export type PackageManifestEventView = Omit<PackageManifest, "generatedAt">;
+
+/**
+ * Project a manifest into the shape sent to breathecode.
+ *
+ * Today this is nearly the identity — only `generatedAt`, an internal detail of
+ * the projection, is dropped. It exists as an explicit pick so that fields
+ * added to the manifest later (hashes, generation flags) do not silently become
+ * part of an external contract: joining the event payload has to be a decision
+ * visible in a diff.
+ */
+export function serializePackageManifestForEvent(
+  manifest: PackageManifest
+): PackageManifestEventView {
+  return {
+    schemaVersion: manifest.schemaVersion,
+    publishedAt: manifest.publishedAt,
+    slug: manifest.slug,
+    title: manifest.title,
+    description: manifest.description,
+    preview: manifest.preview,
+    technologies: manifest.technologies,
+    difficulty: manifest.difficulty,
+    duration: manifest.duration,
+    lessons: manifest.lessons.map(lesson => ({
+      id: lesson.id,
+      slug: lesson.slug,
+      position: lesson.position,
+      type: lesson.type,
+      title: lesson.title,
+      description: lesson.description,
+      video: lesson.video,
+    })),
+  }
+}
+
 async function readReadmeFrontmatter(
   bucket: Bucket,
   exerciseSlug: string,

package/src/utils/publishEvents.ts

@@ -0,0 +1,181 @@
+import axios from "axios"
+import { BREATHECODE_EVENTS_URL, RIGOBOT_HOST } from "./api"
+import {
+  PackageManifest,
+  PackageManifestEventView,
+  serializePackageManifestForEvent,
+} from "./packageManifest"
+
+/**
+ * Publication events sent to breathecode.
+ *
+ * They are emitted from here, and not from Rigobot, because learnpack-cli is
+ * the only party that witnesses the facts: Rigobot's part ends when the zip is
+ * deployed, before the breathecode asset sync, and it never sees the end of the
+ * asynchronous description stage.
+ *
+ * Transport is the existing telemetry endpoint. The discriminator is a
+ * top-level `event` key, which is what breathecode's webhook log already looks
+ * for; a body without it keeps being processed as plain telemetry.
+ *
+ * `package_slug` and `package_id` are duplicated at the top level on purpose:
+ * `LearnPack.add_webhook_to_log` fills the indexed columns from there, so
+ * without them the events would land as rows nobody can filter in the webhook
+ * viewer.
+ */
+
+export const PACKAGE_PUBLISHED_EVENT = "package_published"
+export const PACKAGE_MANIFEST_UPDATED_EVENT = "package_manifest_updated"
+
+/** Whether a second event is coming for this publication. */
+export type DescriptionsOutcome = "queued" | "up_to_date";
+
+export type PublishEventEnvelope = {
+  event: string;
+  package_slug: string;
+  package_id?: number;
+  payload: Record<string, unknown>;
+};
+
+export type PackagePublishedPayload = {
+  publish_id: string;
+  package: Record<string, unknown> | null;
+  manifest: PackageManifestEventView | null;
+  descriptions: DescriptionsOutcome;
+};
+
+export type PackageManifestUpdatedPayload = {
+  publish_id: string;
+  status: "success" | "failed";
+  package: Record<string, unknown> | null;
+  manifest: PackageManifestEventView | null;
+  stats?: {
+    generated: number;
+    failed: number;
+    missing: number;
+  };
+};
+
+export type PublishEventContext = {
+  publishId: string;
+  courseSlug: string;
+  /** `GET /v1/learnpack/package/{slug}/` from Rigobot. */
+  packageInfo: Record<string, unknown> | null;
+  manifest: PackageManifest | null;
+};
+
+function packageIdOf(
+  packageInfo: Record<string, unknown> | null
+): number | undefined {
+  const id = packageInfo?.id
+  return typeof id === "number" ? id : undefined
+}
+
+export function buildPackagePublishedEvent(
+  context: PublishEventContext,
+  descriptions: DescriptionsOutcome
+): PublishEventEnvelope {
+  const payload: PackagePublishedPayload = {
+    publish_id: context.publishId,
+    package: context.packageInfo,
+    manifest: context.manifest ?
+      serializePackageManifestForEvent(context.manifest) :
+      null,
+    descriptions,
+  }
+
+  return {
+    event: PACKAGE_PUBLISHED_EVENT,
+    package_slug: context.courseSlug,
+    package_id: packageIdOf(context.packageInfo),
+    payload: payload as unknown as Record<string, unknown>,
+  }
+}
+
+export function buildPackageManifestUpdatedEvent(
+  context: PublishEventContext,
+  status: "success" | "failed",
+  stats?: PackageManifestUpdatedPayload["stats"]
+): PublishEventEnvelope {
+  const payload: PackageManifestUpdatedPayload = {
+    publish_id: context.publishId,
+    status,
+    package: context.packageInfo,
+    manifest: context.manifest ?
+      serializePackageManifestForEvent(context.manifest) :
+      null,
+  }
+
+  if (stats) {
+    payload.stats = stats
+  }
+
+  return {
+    event: PACKAGE_MANIFEST_UPDATED_EVENT,
+    package_slug: context.courseSlug,
+    package_id: packageIdOf(context.packageInfo),
+    payload: payload as unknown as Record<string, unknown>,
+  }
+}
+
+/**
+ * Fire and forget: a publication must never fail because breathecode is down,
+ * so every error is swallowed and logged. Returns whether it was delivered, for
+ * the journal, not for control flow.
+ */
+export async function sendPublishEvent(
+  envelope: PublishEventEnvelope,
+  breathecodeToken: string,
+  timeoutMs = 10_000
+): Promise<boolean> {
+  if (!breathecodeToken) {
+    console.warn(
+      `[publish-events] Skipping "${envelope.event}": no breathecode token`
+    )
+    return false
+  }
+
+  try {
+    await axios.post(BREATHECODE_EVENTS_URL, envelope, {
+      timeout: timeoutMs,
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: "Token " + breathecodeToken.trim(),
+      },
+    })
+    return true
+  } catch (error) {
+    console.error(
+      `[publish-events] Could not deliver "${envelope.event}" for "${envelope.package_slug}":`,
+      (error as Error).message
+    )
+    return false
+  }
+}
+
+/**
+ * Package information for the event payload, straight from Rigobot's own
+ * serializer so the shape stays owned by the system that owns the model.
+ * Returns null on failure: an event with less context still beats no event.
+ */
+export async function fetchPackageInfo(
+  courseSlug: string,
+  rigobotToken: string
+): Promise<Record<string, unknown> | null> {
+  try {
+    const response = await axios.get(
+      `${RIGOBOT_HOST}/v1/learnpack/package/${courseSlug}/`,
+      {
+        timeout: 10_000,
+        headers: { Authorization: "Token " + rigobotToken.trim() },
+      }
+    )
+    return response.data as Record<string, unknown>
+  } catch (error) {
+    console.error(
+      `[publish-events] Could not read package "${courseSlug}" from Rigobot:`,
+      (error as Error).message
+    )
+    return null
+  }
+}

package/src/utils/publishJournal.ts

@@ -0,0 +1,383 @@
+import { Bucket } from "@google-cloud/storage"
+import { v4 as uuidv4 } from "uuid"
+
+/**
+ * Publish journal — per-stage state of a publication.
+ *
+ * Publishing is a distributed transaction across three systems (S3 via Rigobot,
+ * breathecode assets, and our own buckets) plus an asynchronous description
+ * stage. Some of its outcomes are NOT derivable from content afterwards: in
+ * particular "was the manifest-updated event already emitted?" leaves no trace
+ * in S3 or GCS. The journal records what happened per stage so the sweep can
+ * resume an interrupted publication and honour the event contract (a publish
+ * that announced `descriptions: "queued"` must always be followed by exactly
+ * one `package_manifest_updated`).
+ *
+ * Design notes:
+ *
+ * - **Best effort, never a lock.** Every write swallows its errors: if the
+ *   journal cannot be written the publication still proceeds. Losing the
+ *   journal degrades us to the previous behaviour (no recovery, no diagnosis),
+ *   which is acceptable; blocking a publication because a bookkeeping write
+ *   failed is not.
+ * - **Flat key space.** Journals live in `publish-journal/{slug}__{id}.json`,
+ *   a sibling prefix of `courses/`, so the sweep can list every unfinished
+ *   publication with one prefix query and the concurrency guard can list a
+ *   single course with another, without maintaining an index.
+ * - **Explicit cleanup.** A journal is deleted once every stage succeeded, and
+ *   kept when any stage failed (visibility + retry). GCS lifecycle rules would
+ *   be the natural alternative, but the service account only holds
+ *   object-level permissions (`storage.buckets.*` is denied), so the cleanup
+ *   has to happen in code.
+ */
+
+export const JOURNAL_PREFIX = "publish-journal/"
+
+/**
+ * Stages of a publication, in execution order.
+ *
+ * Only stages the orchestrator can observe on its own: writing the asset id
+ * back into Rigobot happens inside the asset sync call, so it is covered by
+ * `assetsSync` rather than being a stage that could never be marked.
+ */
+export const STAGE_NAMES = [
+  "deploy",
+  "assetsSync",
+  "publishedEvent",
+  "descriptions",
+  "gcsMirror",
+  "manifestEvent",
+] as const
+
+export type StageName = typeof STAGE_NAMES[number];
+
+export type StageStatus = "pending" | "running" | "done" | "failed";
+
+export type Stage = {
+  status: StageStatus;
+  at?: string;
+  error?: string;
+  meta?: unknown;
+};
+
+export type PublishJournal = {
+  publishId: string;
+  courseSlug: string;
+  startedAt: string;
+  updatedAt: string;
+  /** Times the sweep has picked this journal up, to bound retries. */
+  attempts: number;
+  /** Set once retries are exhausted: kept for a human, ignored by the sweep. */
+  abandoned?: boolean;
+  stages: Record<StageName, Stage>;
+};
+
+export type JournalRef = {
+  courseSlug: string;
+  publishId: string;
+  key: string;
+};
+
+/**
+ * Minimal storage surface the journal needs, so callers can pass the real
+ * bucket adapter or an in-memory double. Object-level operations only.
+ */
+export interface JournalStorage {
+  save(key: string, contents: string): Promise<void>;
+  read(key: string): Promise<string | null>;
+  remove(key: string): Promise<void>;
+  list(prefix: string): Promise<string[]>;
+}
+
+export function journalKey(courseSlug: string, publishId: string): string {
+  return `${JOURNAL_PREFIX}${courseSlug}__${publishId}.json`
+}
+
+export function coursePrefix(courseSlug: string): string {
+  return `${JOURNAL_PREFIX}${courseSlug}__`
+}
+
+/** Inverse of `journalKey`; returns null for keys that don't match the shape. */
+export function parseJournalKey(key: string): JournalRef | null {
+  if (!key.startsWith(JOURNAL_PREFIX) || !key.endsWith(".json")) {
+    return null
+  }
+
+  const name = key.slice(JOURNAL_PREFIX.length, -".json".length)
+  const separator = name.indexOf("__")
+  if (separator <= 0) {
+    return null
+  }
+
+  const courseSlug = name.slice(0, separator)
+  const publishId = name.slice(separator + 2)
+  if (!courseSlug || !publishId) {
+    return null
+  }
+
+  return { courseSlug, publishId, key }
+}
+
+function emptyStages(): Record<StageName, Stage> {
+  const stages = {} as Record<StageName, Stage>
+  for (const name of STAGE_NAMES) {
+    stages[name] = { status: "pending" }
+  }
+
+  return stages
+}
+
+/** Persist the journal. Never throws: bookkeeping must not break a publish. */
+async function persist(
+  storage: JournalStorage,
+  journal: PublishJournal
+): Promise<void> {
+  try {
+    await storage.save(
+      journalKey(journal.courseSlug, journal.publishId),
+      JSON.stringify(journal, null, 2)
+    )
+  } catch (error) {
+    console.warn(
+      `[publish-journal] Could not persist journal for "${journal.courseSlug}":`,
+      (error as Error).message
+    )
+  }
+}
+
+/**
+ * Start a journal for a new publication. The generated `publishId` correlates
+ * the two publish events and is returned even if persisting failed.
+ */
+export async function createJournal(
+  storage: JournalStorage,
+  courseSlug: string,
+  publishId: string = uuidv4()
+): Promise<PublishJournal> {
+  const now = new Date().toISOString()
+  const journal: PublishJournal = {
+    publishId,
+    courseSlug,
+    startedAt: now,
+    updatedAt: now,
+    attempts: 0,
+    stages: emptyStages(),
+  }
+
+  await persist(storage, journal)
+  return journal
+}
+
+/** Update one stage and persist. Mutates and returns the same journal object. */
+export async function markStage(
+  storage: JournalStorage,
+  journal: PublishJournal,
+  stage: StageName,
+  status: StageStatus,
+  extra: { error?: string; meta?: unknown } = {}
+): Promise<PublishJournal> {
+  const next: Stage = { status, at: new Date().toISOString() }
+  if (extra.error !== undefined) {
+    next.error = extra.error
+  }
+
+  if (extra.meta !== undefined) {
+    next.meta = extra.meta
+  }
+
+  journal.stages[stage] = next
+  journal.updatedAt = next.at as string
+  await persist(storage, journal)
+  return journal
+}
+
+/** Read a journal. Returns null when missing or unreadable. */
+export async function readJournal(
+  storage: JournalStorage,
+  courseSlug: string,
+  publishId: string
+): Promise<PublishJournal | null> {
+  return readJournalByKey(storage, journalKey(courseSlug, publishId))
+}
+
+export async function readJournalByKey(
+  storage: JournalStorage,
+  key: string
+): Promise<PublishJournal | null> {
+  try {
+    const raw = await storage.read(key)
+    if (!raw) {
+      return null
+    }
+
+    const parsed = JSON.parse(raw) as PublishJournal
+    if (!parsed || !parsed.publishId || !parsed.stages) {
+      return null
+    }
+
+    // Tolerate journals written before a stage was added to STAGE_NAMES.
+    for (const name of STAGE_NAMES) {
+      if (!parsed.stages[name]) {
+        parsed.stages[name] = { status: "pending" }
+      }
+    }
+
+    return parsed
+  } catch (error) {
+    console.warn(
+      `[publish-journal] Could not read journal "${key}":`,
+      (error as Error).message
+    )
+    return null
+  }
+}
+
+/**
+ * Every journal still stored, i.e. every publication that did not finish
+ * cleanly. Returns an empty list on error (the sweep falls back to its deep
+ * scan rather than crashing).
+ */
+export async function listJournalRefs(
+  storage: JournalStorage
+): Promise<JournalRef[]> {
+  return listRefsByPrefix(storage, JOURNAL_PREFIX)
+}
+
+/** Journals of a single course; backs the concurrent-publish guard. */
+export async function listJournalRefsForCourse(
+  storage: JournalStorage,
+  courseSlug: string
+): Promise<JournalRef[]> {
+  return listRefsByPrefix(storage, coursePrefix(courseSlug))
+}
+
+async function listRefsByPrefix(
+  storage: JournalStorage,
+  prefix: string
+): Promise<JournalRef[]> {
+  try {
+    const keys = await storage.list(prefix)
+    const refs: JournalRef[] = []
+    for (const key of keys) {
+      const ref = parseJournalKey(key)
+      if (ref) {
+        refs.push(ref)
+      }
+    }
+
+    return refs
+  } catch (error) {
+    console.warn(
+      `[publish-journal] Could not list journals "${prefix}":`,
+      (error as Error).message
+    )
+    return []
+  }
+}
+
+export function isStageTerminal(stage: Stage): boolean {
+  return stage.status === "done" || stage.status === "failed"
+}
+
+export function hasFailedStage(journal: PublishJournal): boolean {
+  return STAGE_NAMES.some(name => journal.stages[name].status === "failed")
+}
+
+/** True when no stage is left to run (all done, or done/failed mix). */
+export function isJournalSettled(journal: PublishJournal): boolean {
+  return STAGE_NAMES.every(name => isStageTerminal(journal.stages[name]))
+}
+
+/** Stages the sweep still has work to do on. */
+export function pendingStages(journal: PublishJournal): StageName[] {
+  return STAGE_NAMES.filter(name => !isStageTerminal(journal.stages[name]))
+}
+
+/** A journal untouched for longer than `maxAgeMs` is presumed interrupted. */
+export function isJournalStale(
+  journal: PublishJournal,
+  maxAgeMs: number,
+  now: number = Date.now()
+): boolean {
+  const updatedAt = Date.parse(journal.updatedAt)
+  if (Number.isNaN(updatedAt)) {
+    return true
+  }
+
+  return now - updatedAt > maxAgeMs
+}
+
+export async function incrementAttempts(
+  storage: JournalStorage,
+  journal: PublishJournal
+): Promise<PublishJournal> {
+  journal.attempts += 1
+  journal.updatedAt = new Date().toISOString()
+  await persist(storage, journal)
+  return journal
+}
+
+/**
+ * Give up on a journal after too many attempts. It is kept, not deleted: a
+ * publication that never completed is exactly what someone needs to see.
+ */
+export async function abandonJournal(
+  storage: JournalStorage,
+  journal: PublishJournal
+): Promise<PublishJournal> {
+  journal.abandoned = true
+  journal.updatedAt = new Date().toISOString()
+  await persist(storage, journal)
+  return journal
+}
+
+/**
+ * Close a journal: delete it when every stage succeeded, keep it otherwise so
+ * the sweep can retry and a human can inspect it. Never throws.
+ */
+export async function finalizeJournal(
+  storage: JournalStorage,
+  journal: PublishJournal
+): Promise<"deleted" | "kept"> {
+  if (!isJournalSettled(journal) || hasFailedStage(journal)) {
+    await persist(storage, journal)
+    return "kept"
+  }
+
+  try {
+    await storage.remove(journalKey(journal.courseSlug, journal.publishId))
+    return "deleted"
+  } catch (error) {
+    console.warn(
+      `[publish-journal] Could not delete journal for "${journal.courseSlug}":`,
+      (error as Error).message
+    )
+    return "kept"
+  }
+}
+
+/** GCS-backed adapter. Object-level operations only, by design. */
+export function createGcsJournalStorage(bucket: Bucket): JournalStorage {
+  return {
+    async save(key, contents) {
+      await bucket.file(key).save(Buffer.from(contents, "utf8"), {
+        contentType: "application/json",
+      })
+    },
+    async read(key) {
+      try {
+        const [buf] = await bucket.file(key).download()
+        return buf.toString()
+      } catch {
+        return null
+      }
+    },
+    async remove(key) {
+      await bucket.file(key).delete()
+    },
+    async list(prefix) {
+      const [files] = await bucket.getFiles({ prefix })
+      return files.map(file => file.name)
+    },
+  }
+}