@learnpack/learnpack 5.0.363 → 5.0.365

package/src/utils/generationRetry.ts

@@ -0,0 +1,100 @@
+/**
+ * Automatic retries for the two lesson generation phases. Without a cap, an
+ * error that always repeats, like a broken prompt template, starts jobs
+ * forever: Rigobot fails in under a second and the webhook starts another one.
+ */
+
+/** Automatic retries allowed per phase, after the first job. */
+export const MAX_AUTO_RETRIES = 2
+
+/**
+ * Errors from Rigobot that repeat on every attempt, so retrying only spends
+ * jobs. They come from the template, the inputs or the model configuration.
+ */
+const PERMANENT_ERRORS: RegExp[] = [
+  // A template placeholder without an input.
+  /^Missing key /,
+  // Braces in a template body that Python's str.format cannot parse.
+  /format string|in field name|out of range for positional args/,
+  // The purpose's engine cannot return the template's fields.
+  /doesn't support structured outputs/,
+  // The model API rejected the request itself. 429 and 5xx are not here,
+  // because they can pass.
+  /^Error code: (400|401|403|404|422)\b/,
+]
+
+/**
+ * Whether a failed job would fail the same way if started again.
+ * @param statusText - The `status_text` Rigobot sends with the error.
+ * @returns True when retrying cannot help.
+ */
+export const isPermanentGenerationError = (statusText: unknown): boolean => {
+  if (typeof statusText !== "string") return false
+
+  const text = statusText.trim()
+  // A Python KeyError prints as its quoted key: a placeholder in the template
+  // that matches no input.
+  if (/^["']/.test(text) || text.startsWith("{'")) return true
+
+  return PERMANENT_ERRORS.some(pattern => pattern.test(text))
+}
+
+type LessonState = {
+  status?: string;
+  translations?: {
+    [language: string]: { completionId?: number; autoRetries?: number };
+  };
+};
+
+/**
+ * Whether a webhook belongs to a job the lesson no longer waits for. That
+ * happens when Rigobot sends a second webhook for a finished job, or when an
+ * old job answers after a newer one started.
+ * @param lesson - The lesson as saved in the syllabus.
+ * @param language - The course language.
+ * @param jobId - The id of the job the webhook reports.
+ * @returns True when the webhook must not change the lesson.
+ */
+export const isStaleWebhook = (
+  lesson: LessonState,
+  language: string,
+  jobId: unknown
+): boolean => {
+  if (lesson.status === "DONE") return true
+
+  const current = lesson.translations?.[language]?.completionId
+  // Job ids grow, so a lower id is an older job. Without both ids, nothing
+  // tells the webhook apart, and it is handled.
+  return (
+    typeof jobId === "number" &&
+    typeof current === "number" &&
+    current > 0 &&
+    jobId < current
+  )
+}
+
+export type RetryPlan =
+  | { retry: true; attempt: number }
+  | { retry: false; reason: "permanent" | "exhausted" };
+
+/**
+ * Decides whether a failed job starts another one.
+ * @param lesson - The lesson as saved in the syllabus.
+ * @param language - The course language.
+ * @param statusText - The `status_text` Rigobot sends with the error.
+ * @returns The retry to start, numbered from 1, or why there is none.
+ */
+export const planGenerationRetry = (
+  lesson: LessonState,
+  language: string,
+  statusText: unknown
+): RetryPlan => {
+  if (isPermanentGenerationError(statusText)) {
+    return { retry: false, reason: "permanent" }
+  }
+
+  const used = lesson.translations?.[language]?.autoRetries ?? 0
+  if (used >= MAX_AUTO_RETRIES) return { retry: false, reason: "exhausted" }
+
+  return { retry: true, attempt: used + 1 }
+}

package/src/utils/interactivity/componentsGuide.ts

@@ -0,0 +1,294 @@
+// Loads docs/components_v2.yml from learnpack/ide and turns it into the
+// COMPONENTS input of the structured interactivity template.
+//
+// The file is the single source of truth for component names and for the
+// guidance the model receives. Without it the model does not know what each
+// field means, so a generation must not start when no valid copy is available.
+
+import axios from "axios"
+import * as yaml from "js-yaml"
+import { COMPONENT_NAMES, DecidedBy } from "./structuredComponents"
+
+export const COMPONENTS_V2_URL =
+  "https://raw.githubusercontent.com/learnpack/ide/refs/heads/master/docs/components_v2.yml"
+
+export interface GuideComponent {
+  name: string;
+  category: string;
+  decidedBy: DecidedBy;
+  choose: {
+    summary: string;
+    goodFor?: string[];
+    avoid?: string[];
+    tests?: string[];
+  };
+  write: {
+    fields: Record<string, string>;
+    rules?: string[];
+  };
+  visualStyle?: string;
+}
+
+export interface ComponentsGuide {
+  version: string;
+  components: GuideComponent[];
+}
+
+export class ComponentsGuideUnavailableError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = "ComponentsGuideUnavailableError"
+    Object.setPrototypeOf(this, ComponentsGuideUnavailableError.prototype)
+  }
+}
+
+const isNonEmptyString = (value: unknown): value is string =>
+  typeof value === "string" && value.trim().length > 0
+
+const isPlainObject = (value: unknown): value is Record<string, any> =>
+  value !== null && typeof value === "object" && !Array.isArray(value)
+
+// Parses and checks the guide. It is lenient on purpose: the IDE repository
+// can add or rename a component before this CLI knows it, and that must not
+// stop every generation. Unknown or incomplete components are skipped with a
+// warning; the guide is unavailable only when it does not parse or has no
+// usable component.
+export const parseComponentsGuide = (
+  text: string
+): { guide: ComponentsGuide; warnings: string[] } => {
+  let document: any
+  try {
+    document = yaml.load(text)
+  } catch (error) {
+    throw new ComponentsGuideUnavailableError(
+      `components_v2.yml is not valid YAML: ${(error as Error).message}`
+    )
+  }
+
+  if (!isPlainObject(document) || !Array.isArray(document.components)) {
+    throw new ComponentsGuideUnavailableError(
+      "components_v2.yml has no components list"
+    )
+  }
+
+  const warnings: string[] = []
+  const known = new Set<string>(COMPONENT_NAMES)
+  const seen = new Set<string>()
+  const components: GuideComponent[] = []
+
+  for (const raw of document.components) {
+    const name =
+      isPlainObject(raw) && isNonEmptyString(raw.name) ? raw.name : "(no name)"
+    if (!known.has(name)) {
+      warnings.push(
+        `components_v2.yml: "${name}" is not a component this CLI can render; skipped`
+      )
+      continue
+    }
+
+    if (seen.has(name)) {
+      warnings.push(
+        `components_v2.yml: "${name}" appears twice; the second one is skipped`
+      )
+      continue
+    }
+
+    const decidedBy = raw.decidedBy
+    if (
+      (decidedBy !== "syllabus" && decidedBy !== "interactivity") ||
+      !isPlainObject(raw.choose) ||
+      !isNonEmptyString(raw.choose.summary) ||
+      !isPlainObject(raw.write) ||
+      !isPlainObject(raw.write.fields)
+    ) {
+      warnings.push(
+        `components_v2.yml: "${name}" lacks decidedBy, choose.summary or write.fields; skipped`
+      )
+      continue
+    }
+
+    seen.add(name)
+    components.push(raw as GuideComponent)
+  }
+
+  if (components.length === 0) {
+    throw new ComponentsGuideUnavailableError(
+      "components_v2.yml has no usable component"
+    )
+  }
+
+  for (const name of COMPONENT_NAMES) {
+    if (!seen.has(name)) {
+      warnings.push(
+        `components_v2.yml: "${name}" is missing; the model can still return it, without guidance`
+      )
+    }
+  }
+
+  return {
+    guide: { version: String(document.version || ""), components },
+    warnings,
+  }
+}
+
+// The COMPONENTS input of the template: what each field means and the
+// writing rules, plus the choice criteria. Full criteria only go to the
+// components this step decides; for the ones the syllabus plans, the summary
+// is enough to pick a format when a step has no plan. Syntax never goes in.
+export const buildInteractivityGuide = (guide: ComponentsGuide): string => {
+  const components = guide.components.map(component => {
+    const choose =
+      component.decidedBy === "interactivity" ?
+        component.choose :
+        { summary: component.choose.summary }
+    const entry: Record<string, unknown> = {
+      name: component.name,
+      choose,
+      write: component.write,
+    }
+    return entry
+  })
+  return yaml.dump({ components }, { lineWidth: -1, noRefs: true })
+}
+
+export const decidedByFromGuide = (
+  guide: ComponentsGuide
+): Record<string, DecidedBy> => {
+  const result: Record<string, DecidedBy> = {}
+  for (const component of guide.components)
+    result[component.name] = component.decidedBy
+  return result
+}
+
+type Fetcher = (url: string, timeoutMs: number) => Promise<string>;
+
+const defaultFetcher: Fetcher = async (url, timeoutMs) => {
+  const response = await axios.get(url, {
+    timeout: timeoutMs,
+    responseType: "text",
+    transformResponse: [(data: unknown) => data],
+  })
+  return String(response.data)
+}
+
+// Where the guide is downloaded from. COMPONENTS_V2_URL overrides master, for
+// example with the raw URL of a branch, to test a guide before merging it:
+// master is read by every hosted generation. Never set it in production.
+export const resolveComponentsGuideUrl = (
+  env: Record<string, string | undefined> = process.env
+): string => env.COMPONENTS_V2_URL || COMPONENTS_V2_URL
+
+export interface GuideLoaderOptions {
+  // Fixed URL; when absent, resolveComponentsGuideUrl() is read on every
+  // download, so the .env file applies whatever the import order.
+  url?: string;
+  fetcher?: Fetcher;
+  timeoutMs?: number;
+  // Waits between attempts; there is one attempt more than waits.
+  retryDelaysMs?: number[];
+  maxAgeMs?: number;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+  log?: (message: string) => void;
+}
+
+export interface GuideLoader {
+  // The guide, from memory when possible. Throws
+  // ComponentsGuideUnavailableError when there is no valid copy.
+  load(): Promise<ComponentsGuide>;
+  // The copy in memory, if any. Never fetches and never throws.
+  peek(): ComponentsGuide | null;
+}
+
+// Keeps the last valid copy in memory (per process). A copy older than
+// maxAgeMs is still served while a new one is fetched in the background
+// (stale-while-revalidate), so a GitHub hiccup does not stop a generation that
+// already had the guide. Concurrent calls share one download.
+export const createGuideLoader = (
+  options: GuideLoaderOptions = {}
+): GuideLoader => {
+  const fetcher = options.fetcher || defaultFetcher
+  const timeoutMs = options.timeoutMs ?? 5000
+  const retryDelaysMs = options.retryDelaysMs || [1000, 2000]
+  const maxAgeMs = options.maxAgeMs ?? 10 * 60 * 1000
+  const now = options.now || Date.now
+  const sleep =
+    options.sleep ||
+    ((ms: number) =>
+      new Promise<void>(resolve => {
+        setTimeout(resolve, ms)
+      }))
+  const log = options.log || ((message: string) => console.warn(message))
+
+  let cached: { guide: ComponentsGuide; fetchedAt: number } | null = null
+  let inFlight: Promise<ComponentsGuide> | null = null
+
+  const fetchAndParse = async (): Promise<ComponentsGuide> => {
+    const url = options.url || resolveComponentsGuideUrl()
+    if (url !== COMPONENTS_V2_URL) {
+      log(`Loading the components guide from ${url} instead of master`)
+    }
+
+    let lastError = "unknown error"
+    for (let attempt = 0; attempt <= retryDelaysMs.length; attempt++) {
+      // eslint-disable-next-line no-await-in-loop -- attempts are sequential on purpose
+      if (attempt > 0) await sleep(retryDelaysMs[attempt - 1])
+      let text: string
+      try {
+        // eslint-disable-next-line no-await-in-loop -- attempts are sequential on purpose
+        text = await fetcher(url, timeoutMs)
+      } catch (error) {
+        lastError = `could not download ${url}: ${(error as Error).message}`
+        continue
+      }
+
+      // A file that does not parse will not parse on the next attempt either.
+      const { guide, warnings } = parseComponentsGuide(text)
+      for (const warning of warnings) log(warning)
+      return guide
+    }
+
+    throw new ComponentsGuideUnavailableError(lastError)
+  }
+
+  const refresh = (): Promise<ComponentsGuide> => {
+    if (!inFlight) {
+      inFlight = fetchAndParse().then(
+        guide => {
+          cached = { guide, fetchedAt: now() }
+          inFlight = null
+          return guide
+        },
+        error => {
+          inFlight = null
+          throw error
+        }
+      )
+    }
+
+    return inFlight
+  }
+
+  return {
+    async load() {
+      if (cached) {
+        if (now() - cached.fetchedAt > maxAgeMs) {
+          refresh().catch(error =>
+            log(
+              `Could not refresh the components guide, using the copy in memory: ${error.message}`
+            )
+          )
+        }
+
+        return cached.guide
+      }
+
+      return refresh()
+    },
+    peek() {
+      return cached ? cached.guide : null
+    },
+  }
+}
+
+export const componentsGuideLoader = createGuideLoader()

package/src/utils/interactivity/interactivityPlan.ts

@@ -0,0 +1,57 @@
+// Which interactivity template a lesson uses, and what happens when it fails.
+//
+// v2 (structured) returns component data that the CLI renders; v1 is the
+// original template, which writes the component markdown itself. The webhook
+// URL carries the template, because rigobot posts back to that exact URL and
+// its error payload does not say it. How many retries a lesson gets is decided
+// by utils/generationRetry.ts, as for the other generation phases.
+
+export const INTERACTIVITY_V1_SLUG = "add-lesson-interactivy"
+export const INTERACTIVITY_V2_SLUG = "add-lesson-interactivity-v2"
+
+export type InteractivityTemplate = "v1" | "v2";
+
+export const isInteractivityV2Enabled = (
+  env: Record<string, string | undefined> = process.env
+): boolean => env.INTERACTIVITY_V2 === "true"
+
+export interface InteractivityRequest {
+  tpl: InteractivityTemplate;
+  slug: string;
+  webhookQuery: string;
+}
+
+export const planInteractivityRequest = ({
+  v2Enabled,
+  forceV1 = false,
+}: {
+  v2Enabled: boolean;
+  forceV1?: boolean;
+}): InteractivityRequest => {
+  const tpl: InteractivityTemplate = v2Enabled && !forceV1 ? "v2" : "v1"
+  return {
+    tpl,
+    slug: tpl === "v2" ? INTERACTIVITY_V2_SLUG : INTERACTIVITY_V1_SLUG,
+    webhookQuery: `?tpl=${tpl}`,
+  }
+}
+
+export interface InteractivityFailure {
+  // The failed job used the structured template.
+  fromV2: boolean;
+  // The error text that decides whether the lesson is retried.
+  retryStatusText: unknown;
+}
+
+// After a failed job. Every retry uses v1, so a problem of the new template
+// never blocks a lesson. An error that would repeat normally stops the retries,
+// but when the v2 template fails that says nothing about v1, so the error is
+// not passed on and v1 gets its retry within the usual cap. Jobs started
+// before this change have no query and count as v1.
+export const describeInteractivityFailure = (
+  query: { tpl?: unknown },
+  statusText: unknown
+): InteractivityFailure => {
+  const fromV2 = query.tpl === "v2"
+  return { fromV2, retryStatusText: fromV2 ? undefined : statusText }
+}