@learnpack/learnpack 5.0.352 → 5.0.354

package/src/utils/repair/repairStorage.ts

@@ -0,0 +1,168 @@
+import { Bucket } from "@google-cloud/storage"
+import { PutObjectCommand } from "@aws-sdk/client-s3"
+import {
+  AwsClient,
+  fetchJsonObject,
+  fetchTextObject,
+  listObjectKeys,
+  withRetry,
+} from "../s3/packageManifestBackfill"
+
+/**
+ * The two buckets a package lives in, behind one interface.
+ *
+ * A legacy package is repaired **twice, independently** — once against the
+ * published snapshot in S3 and once against the draft in GCS — never by copying
+ * one into the other. The two hold different content (the draft keeps moving
+ * after a publication) and the synthesized structure is a projection of that
+ * content: a syllabus derived from the snapshot could name steps the draft does
+ * not have. This is the same rule `mirrorDescriptions` enforces with its content
+ * hash; deriving from each bucket needs no guard at all.
+ *
+ * What crosses between them is only the descriptions, through
+ * `descriptionsS3Backfill --mirror-draft`, which is hash-guarded per step.
+ */
+
+export type RepairTarget = "s3" | "gcs";
+
+export const ALL_REPAIR_TARGETS: RepairTarget[] = ["s3", "gcs"]
+
+/**
+ * Where each input lives. The two buckets do not agree on this, and the
+ * disagreement is load-bearing:
+ *
+ * - S3 reads `{slug}/config.json` first (`fetchPackageSources`), which is where
+ *   the publish writes it into the zip.
+ * - GCS reads `courses/{slug}/.learn/config.json` — the publish route downloads
+ *   that one before anything else, so a config written anywhere else in the
+ *   draft would not make the package publishable.
+ */
+export type PackageLayout = {
+  learnJsonKey(slug: string): string;
+  /** Config locations in resolution order; the first is the write target. */
+  configKeys(slug: string): string[];
+  syllabusKey(slug: string): string;
+  /** Sidebar locations in resolution order; the first is the write target. */
+  sidebarKeys(slug: string): string[];
+  exercisesPrefix(slug: string): string;
+  /**
+   * Whether `config.json` is the authority on the exercise list, or the folder
+   * listing is.
+   *
+   * This decides the **language keys** of the synthesized syllabus, and the two
+   * buckets genuinely disagree:
+   *
+   * - S3 reads the published `config.json` (`fetchPackageSources`), where these
+   *   legacy packages carry `us` for the unsuffixed README.
+   * - GCS has no such reader. Both the publish-time manifest and the
+   *   descriptions pipeline go through `configBuilder.buildConfig`, which
+   *   derives exercises from the listing and maps an unsuffixed `README.md` to
+   *   `en` — regardless of what the draft's `.learn/config.json` says.
+   *
+   * A syllabus keyed the other way is not read at all: the manifest would look
+   * up `translations["en"]`, find nothing, and publish `description: null` for
+   * every step. So each side is built the way its own reader will read it.
+   */
+  exercisesFromConfig: boolean;
+};
+
+export type RepairStorage = {
+  target: RepairTarget;
+  layout: PackageLayout;
+  readJson<T>(key: string): Promise<T | null>;
+  readText(key: string): Promise<string | null>;
+  listKeys(prefix: string): Promise<string[]>;
+  writeJson(key: string, value: unknown): Promise<void>;
+};
+
+export const s3Layout: PackageLayout = {
+  learnJsonKey: slug => `${slug}/learn.json`,
+  configKeys: slug => [`${slug}/config.json`, `${slug}/.learn/config.json`],
+  syllabusKey: slug => `${slug}/.learn/initialSyllabus.json`,
+  // 25 of the published packages keep their sidebar at the root. It has to be
+  // resolved before writing: `fetchPackageSources` prefers the `.learn/` copy,
+  // so a synthesized one would shadow the published titles.
+  sidebarKeys: slug => [
+    `${slug}/.learn/sidebar.json`,
+    `${slug}/sidebar.json`,
+  ],
+  exercisesPrefix: slug => `${slug}/exercises/`,
+  exercisesFromConfig: true,
+}
+
+export const gcsLayout: PackageLayout = {
+  learnJsonKey: slug => `courses/${slug}/learn.json`,
+  configKeys: slug => [
+    `courses/${slug}/.learn/config.json`,
+    `courses/${slug}/config.json`,
+  ],
+  syllabusKey: slug => `courses/${slug}/.learn/initialSyllabus.json`,
+  sidebarKeys: slug => [`courses/${slug}/.learn/sidebar.json`],
+  exercisesPrefix: slug => `courses/${slug}/exercises/`,
+  exercisesFromConfig: false,
+}
+
+export function createS3RepairStorage(
+  s3: AwsClient,
+  bucket: string
+): RepairStorage {
+  return {
+    target: "s3",
+    layout: s3Layout,
+    readJson: key => fetchJsonObject(s3, bucket, key),
+    readText: key => fetchTextObject(s3, bucket, key),
+    listKeys: prefix => listObjectKeys(s3, bucket, prefix),
+    async writeJson(key, value) {
+      await withRetry(() =>
+        s3.send(
+          new PutObjectCommand({
+            Bucket: bucket,
+            Key: key,
+            Body: JSON.stringify(value, null, 2),
+            ContentType: "application/json",
+          })
+        )
+      )
+    },
+  }
+}
+
+export function createGcsRepairStorage(bucket: Bucket): RepairStorage {
+  const download = async (key: string): Promise<string | null> => {
+    try {
+      const [buf] = await bucket.file(key).download()
+      return buf.toString()
+    } catch {
+      return null
+    }
+  }
+
+  return {
+    target: "gcs",
+    layout: gcsLayout,
+    async readJson<T>(key: string): Promise<T | null> {
+      const content = await download(key)
+      if (content === null) {
+        return null
+      }
+
+      try {
+        return JSON.parse(content) as T
+      } catch {
+        return null
+      }
+    },
+    readText: download,
+    async listKeys(prefix) {
+      const [files] = await bucket.getFiles({ prefix })
+      return files.map(file => file.name)
+    },
+    async writeJson(key, value) {
+      await bucket
+        .file(key)
+        .save(Buffer.from(JSON.stringify(value, null, 2), "utf8"), {
+          contentType: "application/json",
+        })
+    },
+  }
+}

package/src/utils/rigoActions.ts

@@ -5,6 +5,11 @@ import { PackageInfo } from "./creatorUtilities"
 import * as fs from "fs"
 import * as path from "path"
 import { RIGOBOT_HOST } from "./api"
+import { stripMarkdownImages } from "./descriptionHash"
+import {
+  DESCRIPTION_PROMPT_VERSION,
+  DESCRIPTION_TARGET_WORD_COUNT,
+} from "./packageManifest"
 
 type TCreateReadmeInputs = {
   title: string;
@@ -542,6 +547,131 @@ export const initialContentGenerator = async (
   }
 }
 
+// Step description generation. Populates initialSyllabus, which is then
+// projected into the package-manifest. Contract (as built) in
+// docs/rigobot-descriptions-contrato.md.
+
+export type TGenerateStepDescriptionsParams = {
+  /** README of the step, in `sourceLanguage`. Images are stripped before sending. */
+  readmeContent: string;
+  sourceLanguage: string;
+  /** Every language to produce; may or may not include `sourceLanguage`. */
+  outputLanguages: string[];
+  lessonTitle: string;
+  courseTitle?: string;
+  targetWordCount?: number;
+  promptVersion?: number;
+};
+
+export type TGenerateStepDescriptionsResult = {
+  /**
+   * Descriptions keyed by the `language_code` Rigobot answered with — building
+   * this record IS the reassociation, the response order is not contractual.
+   * A `null` value is the deliberate fallback for content too thin to describe.
+   */
+  descriptionsByLanguage: Record<string, string | null>;
+  /** Requested languages absent from the response (a protocol problem, not a
+   * "no description" answer: callers should retry these rather than storing a
+   * null that would look settled). */
+  missingLanguages: string[];
+  promptVersion: number;
+  /** Seconds the completion took; worth logging to watch the 30s Heroku edge. */
+  durationSeconds: number | null;
+  completionId: number | null;
+};
+
+/**
+ * Generate one short description per language for a single step.
+ *
+ * One request per step: the eye-catching saving is on the *languages* axis (one
+ * README in, N descriptions out, ~67% fewer input tokens for a typical course),
+ * while batching several steps together would save nothing and would cost cache
+ * granularity and partial results.
+ *
+ * Synchronous by design (`execute_async: false`): the caller owns the write, so
+ * every flow — the post-publish stage, the sweep and the backfill — persists
+ * its own results instead of relying on a callback. Callers throttle with a
+ * concurrency pool.
+ *
+ * All inputs travel as strings: Rigobot fills the template with plain string
+ * replacement and a non-string raises before reaching the LLM.
+ */
+export const generateStepDescriptions = async (
+  token: string,
+  params: TGenerateStepDescriptionsParams,
+  endpointSlug = "generate-step-descriptions"
+): Promise<TGenerateStepDescriptionsResult | null> => {
+  const promptVersion = params.promptVersion ?? DESCRIPTION_PROMPT_VERSION
+  const targetWordCount =
+    params.targetWordCount ?? DESCRIPTION_TARGET_WORD_COUNT
+
+  const inputs = {
+    // Images never help a description and their alt text may be a whole
+    // image-generation prompt; markdown structure is kept, it helps the model.
+    readme_content: stripMarkdownImages(params.readmeContent),
+    source_language: params.sourceLanguage,
+    output_languages: params.outputLanguages.join(","),
+    lesson_title: params.lessonTitle,
+    course_title: params.courseTitle ?? "",
+    target_word_count: String(targetWordCount),
+    prompt_version: String(promptVersion),
+  }
+
+  try {
+    const response = await axios.post(
+      `${RIGOBOT_HOST}/v1/prompting/completion/${endpointSlug}/`,
+      { inputs, execute_async: false },
+      {
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: "Token " + token.trim(),
+        },
+      }
+    )
+
+    const data = response.data
+    if (!data || data.status === "ERROR") {
+      console.error(
+        "Error in generateStepDescriptions:",
+        data?.status_text || "completion returned ERROR"
+      )
+      return null
+    }
+
+    const entries = Array.isArray(data.parsed?.descriptions) ?
+      data.parsed.descriptions :
+      []
+
+    const descriptionsByLanguage: Record<string, string | null> = {}
+    for (const entry of entries) {
+      const code = entry?.language_code
+      if (typeof code !== "string" || !code) {
+        continue
+      }
+
+      descriptionsByLanguage[code] =
+        typeof entry.description === "string" ? entry.description : null
+    }
+
+    const echoedVersion = Number.parseInt(data.inputs?.prompt_version, 10)
+
+    return {
+      descriptionsByLanguage,
+      missingLanguages: params.outputLanguages.filter(
+        lang => !(lang in descriptionsByLanguage)
+      ),
+      promptVersion: Number.isFinite(echoedVersion) ?
+        echoedVersion :
+        promptVersion,
+      durationSeconds: typeof data.duration === "number" ? data.duration : null,
+      completionId: typeof data.id === "number" ? data.id : null,
+    }
+  } catch (error) {
+    console.error("Error in generateStepDescriptions:", error)
+    return null
+  }
+}
+
 type TAddInteractivityInputs = {
   components: string;
   prev_lesson: string;