@learnpack/learnpack 5.0.352 → 5.0.353

package/src/utils/rigoActions.ts

@@ -5,6 +5,11 @@ import { PackageInfo } from "./creatorUtilities"
 import * as fs from "fs"
 import * as path from "path"
 import { RIGOBOT_HOST } from "./api"
+import { stripMarkdownImages } from "./descriptionHash"
+import {
+  DESCRIPTION_PROMPT_VERSION,
+  DESCRIPTION_TARGET_WORD_COUNT,
+} from "./packageManifest"
 
 type TCreateReadmeInputs = {
   title: string;
@@ -542,6 +547,131 @@ export const initialContentGenerator = async (
   }
 }
 
+// Step description generation. Populates initialSyllabus, which is then
+// projected into the package-manifest. Contract (as built) in
+// docs/rigobot-descriptions-contrato.md.
+
+export type TGenerateStepDescriptionsParams = {
+  /** README of the step, in `sourceLanguage`. Images are stripped before sending. */
+  readmeContent: string;
+  sourceLanguage: string;
+  /** Every language to produce; may or may not include `sourceLanguage`. */
+  outputLanguages: string[];
+  lessonTitle: string;
+  courseTitle?: string;
+  targetWordCount?: number;
+  promptVersion?: number;
+};
+
+export type TGenerateStepDescriptionsResult = {
+  /**
+   * Descriptions keyed by the `language_code` Rigobot answered with — building
+   * this record IS the reassociation, the response order is not contractual.
+   * A `null` value is the deliberate fallback for content too thin to describe.
+   */
+  descriptionsByLanguage: Record<string, string | null>;
+  /** Requested languages absent from the response (a protocol problem, not a
+   * "no description" answer: callers should retry these rather than storing a
+   * null that would look settled). */
+  missingLanguages: string[];
+  promptVersion: number;
+  /** Seconds the completion took; worth logging to watch the 30s Heroku edge. */
+  durationSeconds: number | null;
+  completionId: number | null;
+};
+
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+export const generateStepDescriptions = async (
+  token: string,
+  params: TGenerateStepDescriptionsParams,
+  endpointSlug = "generate-step-descriptions"
+): Promise<TGenerateStepDescriptionsResult | null> => {
+  const promptVersion = params.promptVersion ?? DESCRIPTION_PROMPT_VERSION
+  const targetWordCount =
+    params.targetWordCount ?? DESCRIPTION_TARGET_WORD_COUNT
+
+  const inputs = {
+    // Images never help a description and their alt text may be a whole
+    // image-generation prompt; markdown structure is kept, it helps the model.
+    readme_content: stripMarkdownImages(params.readmeContent),
+    source_language: params.sourceLanguage,
+    output_languages: params.outputLanguages.join(","),
+    lesson_title: params.lessonTitle,
+    course_title: params.courseTitle ?? "",
+    target_word_count: String(targetWordCount),
+    prompt_version: String(promptVersion),
+  }
+
+  try {
+    const response = await axios.post(
+      `${RIGOBOT_HOST}/v1/prompting/completion/${endpointSlug}/`,
+      { inputs, execute_async: false },
+      {
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: "Token " + token.trim(),
+        },
+      }
+    )
+
+    const data = response.data
+    if (!data || data.status === "ERROR") {
+      console.error(
+        "Error in generateStepDescriptions:",
+        data?.status_text || "completion returned ERROR"
+      )
+      return null
+    }
+
+    const entries = Array.isArray(data.parsed?.descriptions) ?
+      data.parsed.descriptions :
+      []
+
+    const descriptionsByLanguage: Record<string, string | null> = {}
+    for (const entry of entries) {
+      const code = entry?.language_code
+      if (typeof code !== "string" || !code) {
+        continue
+      }
+
+      descriptionsByLanguage[code] =
+        typeof entry.description === "string" ? entry.description : null
+    }
+
+    const echoedVersion = Number.parseInt(data.inputs?.prompt_version, 10)
+
+    return {
+      descriptionsByLanguage,
+      missingLanguages: params.outputLanguages.filter(
+        lang => !(lang in descriptionsByLanguage)
+      ),
+      promptVersion: Number.isFinite(echoedVersion) ?
+        echoedVersion :
+        promptVersion,
+      durationSeconds: typeof data.duration === "number" ? data.duration : null,
+      completionId: typeof data.id === "number" ? data.id : null,
+    }
+  } catch (error) {
+    console.error("Error in generateStepDescriptions:", error)
+    return null
+  }
+}
+
 type TAddInteractivityInputs = {
   components: string;
   prev_lesson: string;

package/src/utils/s3/packageManifestBackfill.ts

@@ -259,7 +259,7 @@ export async function fetchJsonObject<T>(
   }
 }
 
-async function fetchTextObject(
+export async function fetchTextObject(
   s3: AwsClient,
   bucket: string,
   key: string
@@ -279,7 +279,7 @@ async function fetchTextObject(
   }
 }
 
-async function listObjectKeys(
+export async function listObjectKeys(
   s3: AwsClient,
   bucket: string,
   prefix: string

package/src/utils/syllabusSync.ts

@@ -0,0 +1,390 @@
+import { Bucket } from "@google-cloud/storage"
+import { Lesson, Syllabus } from "../models/creator"
+import { buildConfig } from "./configBuilder"
+import { slugify } from "./creatorUtilities"
+
+/**
+ * Provisional syllabus ↔ bucket reconciliation.
+ *
+ * The `initialSyllabus.json` occasionally drifts from what actually lives in the
+ * bucket (lessons/translations present as files but missing from the syllabus,
+ * or vice versa). Because step descriptions are stored per (step, lang) in the
+ * syllabus, that drift leaves missing slots and causes skipped/spurious
+ * generations. This reconciles the two.
+ *
+ * Extracted from the inline `/actions/synchronize-syllabus` handler so it can be
+ * reused by the sweep/backfill, and generalized over a minimal storage adapter
+ * so it works for both GCS and S3.
+ *
+ * NOTE: this is a temporary patch until the root cause of the drift is fixed.
+ * Track its removal via the `DESCRIPTIONS_RECONCILE_SYLLABUS` flag on callers.
+ */
+
+export interface SyllabusSyncStorage {
+  /**
+   * Map of `folderSlug -> fileCount` for every folder directly under
+   * `exercises/`. A single authoritative listing: if it throws, callers must
+   * abort any destructive action (never delete on an errored/unknown listing).
+   */
+  listExerciseFolderFileCounts(
+    courseSlug: string
+  ): Promise<Map<string, number>>;
+  /** README language codes (lowercased) present per exercise folder slug. */
+  translationLangsBySlug(courseSlug: string): Promise<Map<string, string[]>>;
+}
+
+export type SyllabusSyncOptions = {
+  /**
+   * When false (default, "additive" mode) only non-destructive passes run:
+   * add-missing, repair-translations, unstick-status. Removal/dedupe are
+   * skipped. When true (manual dev button / root-cause runs) removals apply.
+   */
+  prune?: boolean;
+};
+
+export type SyllabusSyncCounters = {
+  totalLessons: number;
+  keptLessons: number;
+  removedLessons: number;
+  duplicatesResolved: number;
+  addedLessons: number;
+  fixedLessons: number;
+  repairedTranslationsInLessons: number;
+  repairedTranslationEntries: number;
+};
+
+type LessonRef = { id: string; title: string; slug: string };
+type LessonRefWithCount = LessonRef & { fileCount: number };
+
+export type SyllabusSyncResult = {
+  syllabus: Syllabus;
+  changed: boolean;
+  counters: SyllabusSyncCounters;
+  details: {
+    removed: LessonRef[];
+    duplicates: LessonRefWithCount[];
+    kept: Array<LessonRef & { fileCount?: number }>;
+    added: LessonRef[];
+  };
+};
+
+function isInFlight(lesson: Lesson): boolean {
+  // A lesson still being generated may legitimately have no files yet; never
+  // treat it as "missing" (E4 hardening against deleting something real).
+  return (
+    lesson.generated === false ||
+    lesson.status === "GENERATING" ||
+    lesson.status === "PENDING"
+  )
+}
+
+export async function synchronizeSyllabusWithBucket(
+  storage: SyllabusSyncStorage,
+  courseSlug: string,
+  syllabus: Syllabus,
+  options: SyllabusSyncOptions = {}
+): Promise<SyllabusSyncResult> {
+  const prune = options.prune ?? false
+
+  const removed: LessonRef[] = []
+  const duplicates: LessonRefWithCount[] = []
+  const kept: Array<LessonRef & { fileCount?: number }> = []
+  const added: LessonRef[] = []
+  let repairedTranslationsInLessons = 0
+  let repairedTranslationEntries = 0
+  let fixedLessons = 0
+  let changed = false
+
+  // Single authoritative listing. If it fails it throws (callers must not
+  // delete on an unknown listing). The adapter is expected to retry internally.
+  const folderCounts = await storage.listExerciseFolderFileCounts(courseSlug)
+
+  // First pass: resolve existence per lesson, confirming absence under both the
+  // slugified id-title and the uid folder-name variants.
+  const existingLessons: Array<{
+    lesson: Lesson;
+    slug: string;
+    fileCount: number;
+  }> = []
+
+  for (const lesson of syllabus.lessons) {
+    const lessonSlug = slugify(lesson.id + "-" + lesson.title)
+    let resolvedSlug = lessonSlug
+    let fileCount = folderCounts.get(lessonSlug) ?? 0
+
+    if (
+      fileCount === 0 &&
+      lesson.uid &&
+      (folderCounts.get(lesson.uid) ?? 0) > 0
+    ) {
+      resolvedSlug = lesson.uid
+      fileCount = folderCounts.get(lesson.uid) ?? 0
+    }
+
+    if (fileCount === 0) {
+      if (isInFlight(lesson)) {
+        // Keep untouched: do not mark for removal nor as existing.
+        continue
+      }
+
+      removed.push({ id: lesson.id, title: lesson.title, slug: lessonSlug })
+    } else {
+      existingLessons.push({ lesson, slug: resolvedSlug, fileCount })
+    }
+  }
+
+  // Second pass: detect duplicates (multiple lessons resolving to one folder).
+  const slugMap = new Map<
+    string,
+    Array<{ lesson: Lesson; slug: string; fileCount: number }>
+  >()
+  for (const item of existingLessons) {
+    if (!slugMap.has(item.slug)) {
+      slugMap.set(item.slug, [])
+    }
+
+    slugMap.get(item.slug)!.push(item)
+  }
+
+  for (const items of slugMap.values()) {
+    if (items.length > 1) {
+      items.sort((a, b) => b.fileCount - a.fileCount)
+      const winner = items[0]
+      kept.push({
+        id: winner.lesson.id,
+        title: winner.lesson.title,
+        slug: winner.slug,
+        fileCount: winner.fileCount,
+      })
+      for (const loser of items.slice(1)) {
+        duplicates.push({
+          id: loser.lesson.id,
+          title: loser.lesson.title,
+          slug: loser.slug,
+          fileCount: loser.fileCount,
+        })
+      }
+    } else {
+      kept.push({
+        id: items[0].lesson.id,
+        title: items[0].lesson.title,
+        slug: items[0].slug,
+        fileCount: items[0].fileCount,
+      })
+    }
+  }
+
+  // Apply removals only in prune mode (additive mode never deletes).
+  if (prune && (removed.length > 0 || duplicates.length > 0)) {
+    syllabus.lessons = syllabus.lessons.filter(lesson => {
+      const lessonSlug = slugify(lesson.id + "-" + lesson.title)
+      const isRemoved = removed.some(r => r.slug === lessonSlug)
+      const isDuplicate = duplicates.some(
+        d => d.id === lesson.id && d.title === lesson.title
+      )
+      return !isRemoved && !isDuplicate
+    })
+    changed = true
+  }
+
+  // Third pass: add lessons that exist in the bucket but are missing from the
+  // syllabus, then re-sort by numeric id.
+  const existingSlugs = new Set(existingLessons.map(e => e.slug))
+  for (const [folderSlug, count] of folderCounts) {
+    if (count <= 0 || existingSlugs.has(folderSlug)) {
+      continue
+    }
+
+    const idMatch = folderSlug.match(/^(\d+(?:\.\d+)?)(?:-(.*))?$/)
+    const id = idMatch ? idMatch[1] : folderSlug
+    const titlePart = idMatch && idMatch[2] ? idMatch[2] : folderSlug
+    const title = titlePart.replace(/-/g, " ").trim() || folderSlug
+    const newLesson: Lesson = {
+      id,
+      uid: folderSlug,
+      title,
+      type: "READ",
+      description: title,
+      duration: 2,
+      generated: true,
+      status: "DONE",
+    }
+    syllabus.lessons.push(newLesson)
+    added.push({ id, title, slug: folderSlug })
+    changed = true
+  }
+
+  if (added.length > 0) {
+    syllabus.lessons.sort((a, b) => {
+      const na = Number.parseFloat(a.id)
+      const nb = Number.parseFloat(b.id)
+      if (!Number.isNaN(na) && !Number.isNaN(nb)) {
+        return na - nb
+      }
+
+      return String(a.id).localeCompare(String(b.id))
+    })
+  }
+
+  // Fourth + fifth pass: reconcile translations metadata from the actual README
+  // files, and unstick lessons stuck in GENERATING/ERROR whose primary-language
+  // README already exists.
+  try {
+    const translationsBySlug = await storage.translationLangsBySlug(courseSlug)
+
+    for (const lesson of syllabus.lessons) {
+      const candidateSlugs = [
+        slugify(lesson.id + "-" + lesson.title),
+        lesson.uid,
+      ].filter(Boolean) as string[]
+      const matchedSlug = candidateSlugs.find(s => translationsBySlug.has(s))
+      if (!matchedSlug) {
+        continue
+      }
+
+      const languageCodes = translationsBySlug.get(matchedSlug) || []
+      if (languageCodes.length === 0) {
+        continue
+      }
+
+      const currentTranslations = lesson.translations || {}
+      let lessonChanged = false
+
+      for (const lang of languageCodes) {
+        const now = Date.now()
+        if (!currentTranslations[lang]) {
+          currentTranslations[lang] = {
+            completionId: 0,
+            startedAt: now,
+            completedAt: now,
+          }
+          repairedTranslationEntries += 1
+          lessonChanged = true
+        } else {
+          if (!currentTranslations[lang].startedAt) {
+            currentTranslations[lang].startedAt = now
+            lessonChanged = true
+          }
+
+          if (!currentTranslations[lang].completedAt) {
+            currentTranslations[lang].completedAt = now
+            repairedTranslationEntries += 1
+            lessonChanged = true
+          }
+        }
+      }
+
+      if (lessonChanged) {
+        lesson.translations = currentTranslations
+        repairedTranslationsInLessons += 1
+        changed = true
+      }
+    }
+
+    const primaryLanguage = syllabus.courseInfo.language || "en"
+    for (const lesson of syllabus.lessons) {
+      if (lesson.generated !== false) {
+        continue
+      }
+
+      if (lesson.status !== "GENERATING" && lesson.status !== "ERROR") {
+        continue
+      }
+
+      const candidateSlugs = [
+        slugify(lesson.id + "-" + lesson.title),
+        lesson.uid,
+      ].filter(Boolean) as string[]
+      const matchedSlug = candidateSlugs.find(s => translationsBySlug.has(s))
+      if (!matchedSlug) {
+        continue
+      }
+
+      const langs = translationsBySlug.get(matchedSlug) || []
+      if (!langs.includes(primaryLanguage)) {
+        continue
+      }
+
+      lesson.generated = true
+      lesson.status = "DONE"
+      fixedLessons += 1
+      changed = true
+    }
+  } catch (error) {
+    console.error(
+      "⚠️ Could not reconcile lesson translations during syllabus sync:",
+      error
+    )
+  }
+
+  const counters: SyllabusSyncCounters = {
+    totalLessons: syllabus.lessons.length,
+    keptLessons: kept.length,
+    removedLessons: prune ? removed.length : 0,
+    duplicatesResolved: prune ? duplicates.length : 0,
+    addedLessons: added.length,
+    fixedLessons,
+    repairedTranslationsInLessons,
+    repairedTranslationEntries,
+  }
+
+  return {
+    syllabus,
+    changed,
+    counters,
+    details: { removed, duplicates, kept, added },
+  }
+}
+
+async function withRetry<T>(fn: () => Promise<T>, attempts = 3): Promise<T> {
+  let lastError: unknown
+  for (let attempt = 0; attempt < attempts; attempt++) {
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      return await fn()
+    } catch (error) {
+      lastError = error
+    }
+  }
+
+  throw lastError
+}
+
+/** GCS-backed storage adapter (uses the bucket listing + buildConfig). */
+export function createGcsSyllabusSyncStorage(
+  bucket: Bucket
+): SyllabusSyncStorage {
+  return {
+    async listExerciseFolderFileCounts(courseSlug) {
+      const prefix = `courses/${courseSlug}/exercises/`
+      const [files] = await withRetry(() => bucket.getFiles({ prefix }))
+      const counts = new Map<string, number>()
+      for (const file of files) {
+        if (!file.name.startsWith(prefix)) {
+          continue
+        }
+
+        const segment = file.name.slice(prefix.length).split("/")[0]
+        if (segment) {
+          counts.set(segment, (counts.get(segment) || 0) + 1)
+        }
+      }
+
+      return counts
+    },
+    async translationLangsBySlug(courseSlug) {
+      const { exercises } = await buildConfig(bucket, courseSlug)
+      const map = new Map<string, string[]>()
+      for (const exercise of exercises) {
+        const langs = Object.keys(exercise.translations || {})
+          .map(lang => lang.toLowerCase())
+          .filter(Boolean)
+        if (langs.length > 0) {
+          map.set(exercise.slug, [...new Set(langs)])
+        }
+      }
+
+      return map
+    },
+  }
+}