@learnpack/learnpack 5.0.351 → 5.0.353

package/src/utils/publishJournal.ts

@@ -0,0 +1,383 @@
+import { Bucket } from "@google-cloud/storage"
+import { v4 as uuidv4 } from "uuid"
+
+/**
+ * Publish journal — per-stage state of a publication.
+ *
+ * Publishing is a distributed transaction across three systems (S3 via Rigobot,
+ * breathecode assets, and our own buckets) plus an asynchronous description
+ * stage. Some of its outcomes are NOT derivable from content afterwards: in
+ * particular "was the manifest-updated event already emitted?" leaves no trace
+ * in S3 or GCS. The journal records what happened per stage so the sweep can
+ * resume an interrupted publication and honour the event contract (a publish
+ * that announced `descriptions: "queued"` must always be followed by exactly
+ * one `package_manifest_updated`).
+ *
+ * Design notes:
+ *
+ * - **Best effort, never a lock.** Every write swallows its errors: if the
+ *   journal cannot be written the publication still proceeds. Losing the
+ *   journal degrades us to the previous behaviour (no recovery, no diagnosis),
+ *   which is acceptable; blocking a publication because a bookkeeping write
+ *   failed is not.
+ * - **Flat key space.** Journals live in `publish-journal/{slug}__{id}.json`,
+ *   a sibling prefix of `courses/`, so the sweep can list every unfinished
+ *   publication with one prefix query and the concurrency guard can list a
+ *   single course with another, without maintaining an index.
+ * - **Explicit cleanup.** A journal is deleted once every stage succeeded, and
+ *   kept when any stage failed (visibility + retry). GCS lifecycle rules would
+ *   be the natural alternative, but the service account only holds
+ *   object-level permissions (`storage.buckets.*` is denied), so the cleanup
+ *   has to happen in code.
+ */
+
+export const JOURNAL_PREFIX = "publish-journal/"
+
+/**
+ * Stages of a publication, in execution order.
+ *
+ * Only stages the orchestrator can observe on its own: writing the asset id
+ * back into Rigobot happens inside the asset sync call, so it is covered by
+ * `assetsSync` rather than being a stage that could never be marked.
+ */
+export const STAGE_NAMES = [
+  "deploy",
+  "assetsSync",
+  "publishedEvent",
+  "descriptions",
+  "gcsMirror",
+  "manifestEvent",
+] as const
+
+export type StageName = typeof STAGE_NAMES[number];
+
+export type StageStatus = "pending" | "running" | "done" | "failed";
+
+export type Stage = {
+  status: StageStatus;
+  at?: string;
+  error?: string;
+  meta?: unknown;
+};
+
+export type PublishJournal = {
+  publishId: string;
+  courseSlug: string;
+  startedAt: string;
+  updatedAt: string;
+  /** Times the sweep has picked this journal up, to bound retries. */
+  attempts: number;
+  /** Set once retries are exhausted: kept for a human, ignored by the sweep. */
+  abandoned?: boolean;
+  stages: Record<StageName, Stage>;
+};
+
+export type JournalRef = {
+  courseSlug: string;
+  publishId: string;
+  key: string;
+};
+
+/**
+ * Minimal storage surface the journal needs, so callers can pass the real
+ * bucket adapter or an in-memory double. Object-level operations only.
+ */
+export interface JournalStorage {
+  save(key: string, contents: string): Promise<void>;
+  read(key: string): Promise<string | null>;
+  remove(key: string): Promise<void>;
+  list(prefix: string): Promise<string[]>;
+}
+
+export function journalKey(courseSlug: string, publishId: string): string {
+  return `${JOURNAL_PREFIX}${courseSlug}__${publishId}.json`
+}
+
+export function coursePrefix(courseSlug: string): string {
+  return `${JOURNAL_PREFIX}${courseSlug}__`
+}
+
+/** Inverse of `journalKey`; returns null for keys that don't match the shape. */
+export function parseJournalKey(key: string): JournalRef | null {
+  if (!key.startsWith(JOURNAL_PREFIX) || !key.endsWith(".json")) {
+    return null
+  }
+
+  const name = key.slice(JOURNAL_PREFIX.length, -".json".length)
+  const separator = name.indexOf("__")
+  if (separator <= 0) {
+    return null
+  }
+
+  const courseSlug = name.slice(0, separator)
+  const publishId = name.slice(separator + 2)
+  if (!courseSlug || !publishId) {
+    return null
+  }
+
+  return { courseSlug, publishId, key }
+}
+
+function emptyStages(): Record<StageName, Stage> {
+  const stages = {} as Record<StageName, Stage>
+  for (const name of STAGE_NAMES) {
+    stages[name] = { status: "pending" }
+  }
+
+  return stages
+}
+
+/** Persist the journal. Never throws: bookkeeping must not break a publish. */
+async function persist(
+  storage: JournalStorage,
+  journal: PublishJournal
+): Promise<void> {
+  try {
+    await storage.save(
+      journalKey(journal.courseSlug, journal.publishId),
+      JSON.stringify(journal, null, 2)
+    )
+  } catch (error) {
+    console.warn(
+      `[publish-journal] Could not persist journal for "${journal.courseSlug}":`,
+      (error as Error).message
+    )
+  }
+}
+
+/**
+ * Start a journal for a new publication. The generated `publishId` correlates
+ * the two publish events and is returned even if persisting failed.
+ */
+export async function createJournal(
+  storage: JournalStorage,
+  courseSlug: string,
+  publishId: string = uuidv4()
+): Promise<PublishJournal> {
+  const now = new Date().toISOString()
+  const journal: PublishJournal = {
+    publishId,
+    courseSlug,
+    startedAt: now,
+    updatedAt: now,
+    attempts: 0,
+    stages: emptyStages(),
+  }
+
+  await persist(storage, journal)
+  return journal
+}
+
+/** Update one stage and persist. Mutates and returns the same journal object. */
+export async function markStage(
+  storage: JournalStorage,
+  journal: PublishJournal,
+  stage: StageName,
+  status: StageStatus,
+  extra: { error?: string; meta?: unknown } = {}
+): Promise<PublishJournal> {
+  const next: Stage = { status, at: new Date().toISOString() }
+  if (extra.error !== undefined) {
+    next.error = extra.error
+  }
+
+  if (extra.meta !== undefined) {
+    next.meta = extra.meta
+  }
+
+  journal.stages[stage] = next
+  journal.updatedAt = next.at as string
+  await persist(storage, journal)
+  return journal
+}
+
+/** Read a journal. Returns null when missing or unreadable. */
+export async function readJournal(
+  storage: JournalStorage,
+  courseSlug: string,
+  publishId: string
+): Promise<PublishJournal | null> {
+  return readJournalByKey(storage, journalKey(courseSlug, publishId))
+}
+
+export async function readJournalByKey(
+  storage: JournalStorage,
+  key: string
+): Promise<PublishJournal | null> {
+  try {
+    const raw = await storage.read(key)
+    if (!raw) {
+      return null
+    }
+
+    const parsed = JSON.parse(raw) as PublishJournal
+    if (!parsed || !parsed.publishId || !parsed.stages) {
+      return null
+    }
+
+    // Tolerate journals written before a stage was added to STAGE_NAMES.
+    for (const name of STAGE_NAMES) {
+      if (!parsed.stages[name]) {
+        parsed.stages[name] = { status: "pending" }
+      }
+    }
+
+    return parsed
+  } catch (error) {
+    console.warn(
+      `[publish-journal] Could not read journal "${key}":`,
+      (error as Error).message
+    )
+    return null
+  }
+}
+
+/**
+ * Every journal still stored, i.e. every publication that did not finish
+ * cleanly. Returns an empty list on error (the sweep falls back to its deep
+ * scan rather than crashing).
+ */
+export async function listJournalRefs(
+  storage: JournalStorage
+): Promise<JournalRef[]> {
+  return listRefsByPrefix(storage, JOURNAL_PREFIX)
+}
+
+/** Journals of a single course; backs the concurrent-publish guard. */
+export async function listJournalRefsForCourse(
+  storage: JournalStorage,
+  courseSlug: string
+): Promise<JournalRef[]> {
+  return listRefsByPrefix(storage, coursePrefix(courseSlug))
+}
+
+async function listRefsByPrefix(
+  storage: JournalStorage,
+  prefix: string
+): Promise<JournalRef[]> {
+  try {
+    const keys = await storage.list(prefix)
+    const refs: JournalRef[] = []
+    for (const key of keys) {
+      const ref = parseJournalKey(key)
+      if (ref) {
+        refs.push(ref)
+      }
+    }
+
+    return refs
+  } catch (error) {
+    console.warn(
+      `[publish-journal] Could not list journals "${prefix}":`,
+      (error as Error).message
+    )
+    return []
+  }
+}
+
+export function isStageTerminal(stage: Stage): boolean {
+  return stage.status === "done" || stage.status === "failed"
+}
+
+export function hasFailedStage(journal: PublishJournal): boolean {
+  return STAGE_NAMES.some(name => journal.stages[name].status === "failed")
+}
+
+/** True when no stage is left to run (all done, or done/failed mix). */
+export function isJournalSettled(journal: PublishJournal): boolean {
+  return STAGE_NAMES.every(name => isStageTerminal(journal.stages[name]))
+}
+
+/** Stages the sweep still has work to do on. */
+export function pendingStages(journal: PublishJournal): StageName[] {
+  return STAGE_NAMES.filter(name => !isStageTerminal(journal.stages[name]))
+}
+
+/** A journal untouched for longer than `maxAgeMs` is presumed interrupted. */
+export function isJournalStale(
+  journal: PublishJournal,
+  maxAgeMs: number,
+  now: number = Date.now()
+): boolean {
+  const updatedAt = Date.parse(journal.updatedAt)
+  if (Number.isNaN(updatedAt)) {
+    return true
+  }
+
+  return now - updatedAt > maxAgeMs
+}
+
+export async function incrementAttempts(
+  storage: JournalStorage,
+  journal: PublishJournal
+): Promise<PublishJournal> {
+  journal.attempts += 1
+  journal.updatedAt = new Date().toISOString()
+  await persist(storage, journal)
+  return journal
+}
+
+/**
+ * Give up on a journal after too many attempts. It is kept, not deleted: a
+ * publication that never completed is exactly what someone needs to see.
+ */
+export async function abandonJournal(
+  storage: JournalStorage,
+  journal: PublishJournal
+): Promise<PublishJournal> {
+  journal.abandoned = true
+  journal.updatedAt = new Date().toISOString()
+  await persist(storage, journal)
+  return journal
+}
+
+/**
+ * Close a journal: delete it when every stage succeeded, keep it otherwise so
+ * the sweep can retry and a human can inspect it. Never throws.
+ */
+export async function finalizeJournal(
+  storage: JournalStorage,
+  journal: PublishJournal
+): Promise<"deleted" | "kept"> {
+  if (!isJournalSettled(journal) || hasFailedStage(journal)) {
+    await persist(storage, journal)
+    return "kept"
+  }
+
+  try {
+    await storage.remove(journalKey(journal.courseSlug, journal.publishId))
+    return "deleted"
+  } catch (error) {
+    console.warn(
+      `[publish-journal] Could not delete journal for "${journal.courseSlug}":`,
+      (error as Error).message
+    )
+    return "kept"
+  }
+}
+
+/** GCS-backed adapter. Object-level operations only, by design. */
+export function createGcsJournalStorage(bucket: Bucket): JournalStorage {
+  return {
+    async save(key, contents) {
+      await bucket.file(key).save(Buffer.from(contents, "utf8"), {
+        contentType: "application/json",
+      })
+    },
+    async read(key) {
+      try {
+        const [buf] = await bucket.file(key).download()
+        return buf.toString()
+      } catch {
+        return null
+      }
+    },
+    async remove(key) {
+      await bucket.file(key).delete()
+    },
+    async list(prefix) {
+      const [files] = await bucket.getFiles({ prefix })
+      return files.map(file => file.name)
+    },
+  }
+}

package/src/utils/rigoActions.ts

@@ -5,6 +5,11 @@ import { PackageInfo } from "./creatorUtilities"
 import * as fs from "fs"
 import * as path from "path"
 import { RIGOBOT_HOST } from "./api"
+import { stripMarkdownImages } from "./descriptionHash"
+import {
+  DESCRIPTION_PROMPT_VERSION,
+  DESCRIPTION_TARGET_WORD_COUNT,
+} from "./packageManifest"
 
 type TCreateReadmeInputs = {
   title: string;
@@ -542,6 +547,131 @@ export const initialContentGenerator = async (
   }
 }
 
+// Step description generation. Populates initialSyllabus, which is then
+// projected into the package-manifest. Contract (as built) in
+// docs/rigobot-descriptions-contrato.md.
+
+export type TGenerateStepDescriptionsParams = {
+  /** README of the step, in `sourceLanguage`. Images are stripped before sending. */
+  readmeContent: string;
+  sourceLanguage: string;
+  /** Every language to produce; may or may not include `sourceLanguage`. */
+  outputLanguages: string[];
+  lessonTitle: string;
+  courseTitle?: string;
+  targetWordCount?: number;
+  promptVersion?: number;
+};
+
+export type TGenerateStepDescriptionsResult = {
+  /**
+   * Descriptions keyed by the `language_code` Rigobot answered with — building
+   * this record IS the reassociation, the response order is not contractual.
+   * A `null` value is the deliberate fallback for content too thin to describe.
+   */
+  descriptionsByLanguage: Record<string, string | null>;
+  /** Requested languages absent from the response (a protocol problem, not a
+   * "no description" answer: callers should retry these rather than storing a
+   * null that would look settled). */
+  missingLanguages: string[];
+  promptVersion: number;
+  /** Seconds the completion took; worth logging to watch the 30s Heroku edge. */
+  durationSeconds: number | null;
+  completionId: number | null;
+};
+
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+export const generateStepDescriptions = async (
+  token: string,
+  params: TGenerateStepDescriptionsParams,
+  endpointSlug = "generate-step-descriptions"
+): Promise<TGenerateStepDescriptionsResult | null> => {
+  const promptVersion = params.promptVersion ?? DESCRIPTION_PROMPT_VERSION
+  const targetWordCount =
+    params.targetWordCount ?? DESCRIPTION_TARGET_WORD_COUNT
+
+  const inputs = {
+    // Images never help a description and their alt text may be a whole
+    // image-generation prompt; markdown structure is kept, it helps the model.
+    readme_content: stripMarkdownImages(params.readmeContent),
+    source_language: params.sourceLanguage,
+    output_languages: params.outputLanguages.join(","),
+    lesson_title: params.lessonTitle,
+    course_title: params.courseTitle ?? "",
+    target_word_count: String(targetWordCount),
+    prompt_version: String(promptVersion),
+  }
+
+  try {
+    const response = await axios.post(
+      `${RIGOBOT_HOST}/v1/prompting/completion/${endpointSlug}/`,
+      { inputs, execute_async: false },
+      {
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: "Token " + token.trim(),
+        },
+      }
+    )
+
+    const data = response.data
+    if (!data || data.status === "ERROR") {
+      console.error(
+        "Error in generateStepDescriptions:",
+        data?.status_text || "completion returned ERROR"
+      )
+      return null
+    }
+
+    const entries = Array.isArray(data.parsed?.descriptions) ?
+      data.parsed.descriptions :
+      []
+
+    const descriptionsByLanguage: Record<string, string | null> = {}
+    for (const entry of entries) {
+      const code = entry?.language_code
+      if (typeof code !== "string" || !code) {
+        continue
+      }
+
+      descriptionsByLanguage[code] =
+        typeof entry.description === "string" ? entry.description : null
+    }
+
+    const echoedVersion = Number.parseInt(data.inputs?.prompt_version, 10)
+
+    return {
+      descriptionsByLanguage,
+      missingLanguages: params.outputLanguages.filter(
+        lang => !(lang in descriptionsByLanguage)
+      ),
+      promptVersion: Number.isFinite(echoedVersion) ?
+        echoedVersion :
+        promptVersion,
+      durationSeconds: typeof data.duration === "number" ? data.duration : null,
+      completionId: typeof data.id === "number" ? data.id : null,
+    }
+  } catch (error) {
+    console.error("Error in generateStepDescriptions:", error)
+    return null
+  }
+}
+
 type TAddInteractivityInputs = {
   components: string;
   prev_lesson: string;