@learnpack/learnpack 5.0.351 → 5.0.353

package/src/utils/api.ts

@@ -7,6 +7,15 @@ import * as dotenv from "dotenv"
 dotenv.config()
 
 const HOST = "https://breathecode.herokuapp.com"
+
+/**
+ * Student telemetry, and publication events. Both share the endpoint: a body
+ * carrying a top-level `event` key is dispatched as an event, anything else
+ * keeps being processed as telemetry.
+ */
+export const BREATHECODE_TELEMETRY_URL = `${HOST}/v1/assignment/me/telemetry`
+export const BREATHECODE_EVENTS_URL = BREATHECODE_TELEMETRY_URL
+
 export const RIGOBOT_HOST =
   process.env.RIGOBOT_HOST || "https://rigobot.herokuapp.com"
 export const RIGOBOT_REALTIME_HOST = "https://ai.4geeks.com"

package/src/utils/creatorUtilities.ts

@@ -1,4 +1,5 @@
 import Console from "./console"
+import { BREATHECODE_TELEMETRY_URL } from "./api"
 
 const frontMatter = require("front-matter")
 import * as MarkdownIt from "markdown-it"
@@ -189,7 +190,7 @@ export const makePackageInfo = (choices: any) => {
       .replace(/ /g, "-")
       .replace(/[^\w-]+/g, ""),
     telemetry: {
-      batch: "https://breathecode.herokuapp.com/v1/assignment/me/telemetry",
+      batch: BREATHECODE_TELEMETRY_URL,
     },
   }
   return packageInfo

package/src/utils/descriptionHash.ts

@@ -0,0 +1,196 @@
+import * as crypto from "crypto"
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const frontMatter = require("front-matter")
+
+/**
+ * Utilities to detect whether a step description is stale relative to its
+ * README content.
+ *
+ * Two signals are stored per (step, lang) at generation time:
+ *  - sha256(normalizeReadme(content))  -> exact fast-path + content-addressing
+ *  - simhash64(normalizeReadme(content)) -> near-duplicate significance filter
+ *
+ * The sweep re-hashes the current README and compares against the stored
+ * values: exact SHA-256 match => no-op; otherwise if the SimHash Hamming
+ * distance to the *generation baseline* is <= k the change is cosmetic and is
+ * ignored, else the description is regenerated. The H1 title is part of the
+ * normalized content (a deliberate product decision).
+ */
+
+/**
+ * Remove every image from markdown: inline `![alt](url)`, reference-style
+ * `![alt][ref]` and raw `<img>` tags. Regular links `[text](url)` are kept.
+ *
+ * Images are dropped from BOTH sides of the description pipeline — here for
+ * fingerprinting, and again before sending the README to the LLM. Adding,
+ * replacing or removing an illustration does not change what the lesson
+ * teaches, so it must not trigger a regeneration; and the alt text of generated
+ * illustrations is a full image-generation prompt (hundreds of tokens) that
+ * only invites the model to describe the picture instead of the lesson.
+ *
+ * Note the alt text may legitimately contain parentheses, so the alt is matched
+ * as "anything up to the closing bracket" rather than as balanced text.
+ */
+export function stripMarkdownImages(content: string): string {
+  return (
+    content
+      // ![alt](url) — alt may span lines and contain parentheses
+      .replace(/!\[[^\]]*]\([^)]*\)/g, "")
+      // ![alt][ref] — i.e. `!` followed by two bracketed groups
+      .replace(/!(?:\[[^\]]*]){2}/g, "")
+      // <img ...> / <img ... />
+      .replace(/<img\b[^>]*>/gi, "")
+      // [](url) left behind by linked images such as badges
+      .replace(/\[\s*]\([^)]*\)/g, "")
+  )
+}
+
+/**
+ * Canonicalize a README for hashing. Strips the frontmatter (metadata such as
+ * tutorial/intro video URLs, which do not describe the lesson), removes images
+ * (see `stripMarkdownImages`) and collapses all whitespace runs so that
+ * reformatting, blank lines and trailing spaces do not register as content
+ * changes. The H1 and the prose are preserved.
+ */
+export function normalizeReadme(content: string): string {
+  let body = content
+  try {
+    const parsed = frontMatter(content)
+    body = typeof parsed.body === "string" ? parsed.body : content
+  } catch {
+    body = content
+  }
+
+  return stripMarkdownImages(body)
+    .replace(/\r/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+}
+
+export function sha256(input: string): string {
+  return crypto.createHash("sha256").update(input).digest("hex")
+}
+
+// The build targets < ES2020, so BigInt literals are unavailable. The 64-bit
+// SimHash is therefore represented as two unsigned 32-bit halves { hi, lo } and
+// serialized as a 16-char hex string (hi then lo).
+
+function toHex32(x: number): string {
+  return (x >>> 0).toString(16).padStart(8, "0")
+}
+
+function popcount32(input: number): number {
+  let x = input >>> 0
+  let count = 0
+  while (x !== 0) {
+    count += x & 1
+    x >>>= 1
+  }
+
+  return count
+}
+
+function hashToken64(token: string): { hi: number; lo: number } {
+  // First 8 bytes (16 hex chars) of an md5 digest -> two 32-bit halves.
+  const hex = crypto.createHash("md5").update(token).digest("hex").slice(0, 16)
+  return {
+    hi: Number.parseInt(hex.slice(0, 8), 16) >>> 0,
+    lo: Number.parseInt(hex.slice(8, 16), 16) >>> 0,
+  }
+}
+
+/**
+ * 64-bit SimHash (Charikar) over whitespace-delimited tokens, returned as a
+ * zero-padded 16-char hex string.
+ */
+export function simhash64(input: string): string {
+  const tokens = input.split(/\s+/).filter(Boolean)
+  if (tokens.length === 0) {
+    return "0".repeat(16)
+  }
+
+  const weights = Array.from({ length: 64 }, () => 0)
+  for (const token of tokens) {
+    const { hi, lo } = hashToken64(token)
+    for (let i = 0; i < 32; i++) {
+      weights[i] += ((lo >>> i) & 1) === 1 ? 1 : -1
+      weights[32 + i] += ((hi >>> i) & 1) === 1 ? 1 : -1
+    }
+  }
+
+  let lo = 0
+  let hi = 0
+  for (let i = 0; i < 32; i++) {
+    if (weights[i] > 0) {
+      lo |= 1 << i
+    }
+
+    if (weights[32 + i] > 0) {
+      hi |= 1 << i
+    }
+  }
+
+  return toHex32(hi) + toHex32(lo)
+}
+
+/** Hamming distance between two 64-bit hex fingerprints. */
+export function hammingHex(a: string, b: string): number {
+  const hiA = Number.parseInt(a.slice(0, 8), 16) >>> 0
+  const loA = Number.parseInt(a.slice(8, 16), 16) >>> 0
+  const hiB = Number.parseInt(b.slice(0, 8), 16) >>> 0
+  const loB = Number.parseInt(b.slice(8, 16), 16) >>> 0
+  return popcount32((hiA ^ hiB) >>> 0) + popcount32((loA ^ loB) >>> 0)
+}
+
+export type ContentFingerprint = {
+  sha256: string;
+  simhash: string;
+};
+
+/** Convenience: normalize + both fingerprints in one call. */
+export function fingerprintReadme(content: string): ContentFingerprint {
+  const normalized = normalizeReadme(content)
+  return {
+    sha256: sha256(normalized),
+    simhash: simhash64(normalized),
+  }
+}
+
+/**
+ * Default significance threshold (Hamming distance over the 64-bit SimHash).
+ * k=3 is the classic near-duplicate threshold for 64-bit SimHash
+ * (Charikar 2002; Manku, Jain & Das Sarma, WWW 2007). Overridable via env.
+ */
+export function getSimhashThreshold(): number {
+  const raw = process.env.DESCRIPTION_SIMHASH_THRESHOLD
+  const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : 3
+}
+
+/**
+ * Decide whether a description generated from `previous` is stale given the
+ * `current` README content. Exact match => not stale; else stale only if the
+ * SimHash Hamming distance exceeds the threshold (cosmetic edits are ignored).
+ */
+export function isDescriptionStale(
+  previous: { sha256?: string; simhash?: string } | undefined,
+  current: ContentFingerprint,
+  threshold: number = getSimhashThreshold()
+): boolean {
+  if (!previous || !previous.sha256) {
+    // Never generated -> treat as needing generation (i.e. "stale").
+    return true
+  }
+
+  if (previous.sha256 === current.sha256) {
+    return false
+  }
+
+  if (!previous.simhash) {
+    // No baseline fingerprint to compare against -> be conservative.
+    return true
+  }
+
+  return hammingHex(previous.simhash, current.simhash) > threshold
+}

package/src/utils/descriptions/gcsStorage.ts

@@ -0,0 +1,67 @@
+import { Bucket } from "@google-cloud/storage"
+import { Syllabus } from "../../models/creator"
+import { buildConfig } from "../configBuilder"
+import { createGcsSyllabusSyncStorage } from "../syllabusSync"
+import { CourseDescriptionsStorage } from "./generateCourseDescriptions"
+import { CourseExercise } from "./workList"
+
+/**
+ * Draft-bucket (GCS) adapter.
+ *
+ * Descriptions are never generated against this bucket — it keeps moving while
+ * the teacher edits, so it is not a stable snapshot. It is only written by the
+ * mirror-back, which copies descriptions generated from the published package
+ * whenever the draft content still matches.
+ *
+ * Note there is no `reprojectManifest`: the manifest is only rebuilt when
+ * publishing.
+ */
+
+export const gcsSyllabusPath = (courseSlug: string) =>
+  `courses/${courseSlug}/.learn/initialSyllabus.json`
+
+export const gcsReadmePath = (
+  courseSlug: string,
+  exerciseSlug: string,
+  fileName: string
+) => `courses/${courseSlug}/exercises/${exerciseSlug}/${fileName}`
+
+export function createGcsDescriptionsStorage(
+  bucket: Bucket
+): CourseDescriptionsStorage {
+  return {
+    async readSyllabus(courseSlug) {
+      try {
+        const [buf] = await bucket.file(gcsSyllabusPath(courseSlug)).download()
+        return JSON.parse(buf.toString()) as Syllabus
+      } catch {
+        return null
+      }
+    },
+    async writeSyllabus(courseSlug, syllabus) {
+      await bucket
+        .file(gcsSyllabusPath(courseSlug))
+        .save(Buffer.from(JSON.stringify(syllabus), "utf8"), {
+          contentType: "application/json",
+        })
+    },
+    async listExercises(courseSlug): Promise<CourseExercise[]> {
+      const { exercises } = await buildConfig(bucket, courseSlug)
+      return exercises.map(exercise => ({
+        slug: exercise.slug,
+        translations: exercise.translations || {},
+      }))
+    },
+    async readReadme(courseSlug, exerciseSlug, fileName) {
+      try {
+        const [buf] = await bucket
+          .file(gcsReadmePath(courseSlug, exerciseSlug, fileName))
+          .download()
+        return buf.toString()
+      } catch {
+        return null
+      }
+    },
+    syllabusSyncStorage: createGcsSyllabusSyncStorage(bucket),
+  }
+}

package/src/utils/descriptions/generateCourseDescriptions.ts

@@ -0,0 +1,301 @@
+import { Lesson, Syllabus } from "../../models/creator"
+import { PackageManifest } from "../packageManifest"
+import {
+  generateStepDescriptions,
+  TGenerateStepDescriptionsParams,
+  TGenerateStepDescriptionsResult,
+} from "../rigoActions"
+import { mapWithConcurrency } from "../s3/packageManifestBackfill"
+import {
+  SyllabusSyncStorage,
+  synchronizeSyllabusWithBucket,
+} from "../syllabusSync"
+import {
+  buildDescriptionWorkList,
+  countWorkItems,
+  CourseExercise,
+  CourseReadmes,
+  StepWork,
+} from "./workList"
+
+/**
+ * Generate and persist the step descriptions of one published course.
+ *
+ * This is the single implementation behind every flow that produces
+ * descriptions: the background stage that runs after publishing, the sweep that
+ * recovers interrupted publications, and the one-off backfill over the
+ * historical catalogue. They differ only in what triggers them.
+ *
+ * Storage is a port so the service can be driven against the published bucket
+ * (the real case) or an in-memory double (tests), and so the manifest
+ * re-projection stays where the storage-specific knowledge lives.
+ */
+
+export const DEFAULT_DESCRIPTIONS_CONCURRENCY = 5
+
+export interface CourseDescriptionsStorage {
+  readSyllabus(courseSlug: string): Promise<Syllabus | null>;
+  writeSyllabus(courseSlug: string, syllabus: Syllabus): Promise<void>;
+  listExercises(courseSlug: string): Promise<CourseExercise[]>;
+  readReadme(
+    courseSlug: string,
+    exerciseSlug: string,
+    fileName: string
+  ): Promise<string | null>;
+  /** Re-generate `package-manifest.json` from the syllabus and invalidate the CDN. */
+  reprojectManifest?(courseSlug: string): Promise<void>;
+  /** Read the stored manifest, to report the updated one in the event. */
+  readManifest?(courseSlug: string): Promise<PackageManifest | null>;
+  /** Enables the provisional additive syllabus reconciliation. */
+  syllabusSyncStorage?: SyllabusSyncStorage;
+}
+
+export type GenerateCourseDescriptionsOptions = {
+  token: string;
+  dryRun?: boolean;
+  force?: boolean;
+  concurrency?: number;
+  targetWordCount?: number;
+  promptVersion?: number;
+  /** Additive syllabus reconciliation before computing the work list. */
+  reconcile?: boolean;
+  /** Injection point for tests. */
+  generate?: (
+    token: string,
+    params: TGenerateStepDescriptionsParams
+  ) => Promise<TGenerateStepDescriptionsResult | null>;
+};
+
+export type GenerateCourseDescriptionsResult = {
+  /** `skipped` means there was nothing to do, not that something went wrong. */
+  status: "skipped" | "processed" | "failed";
+  steps: number;
+  /** (step, language) pairs written. */
+  generated: number;
+  /** Steps whose completion failed outright. */
+  failed: number;
+  /** (step, language) pairs the model never answered; retried later. */
+  missing: number;
+  errors: string[];
+  /** Completion durations, to watch how close we run to the request limits. */
+  durationsSeconds: number[];
+};
+
+function emptyResult(
+  status: GenerateCourseDescriptionsResult["status"]
+): GenerateCourseDescriptionsResult {
+  return {
+    status,
+    steps: 0,
+    generated: 0,
+    failed: 0,
+    missing: 0,
+    errors: [],
+    durationsSeconds: [],
+  }
+}
+
+/** Read every README of the course, keyed by exercise slug and language. */
+export async function readCourseReadmes(
+  storage: CourseDescriptionsStorage,
+  courseSlug: string,
+  exercises: CourseExercise[],
+  concurrency: number
+): Promise<CourseReadmes> {
+  const readmes: CourseReadmes = {}
+
+  const targets: Array<{ exerciseSlug: string; lang: string; file: string }> =
+    []
+  for (const exercise of exercises) {
+    for (const [lang, file] of Object.entries(exercise.translations || {})) {
+      targets.push({ exerciseSlug: exercise.slug, lang, file })
+    }
+  }
+
+  const contents = await mapWithConcurrency(targets, concurrency, target =>
+    storage.readReadme(courseSlug, target.exerciseSlug, target.file)
+  )
+
+  for (const [index, target] of targets.entries()) {
+    const content = contents[index]
+    if (typeof content !== "string") {
+      continue
+    }
+
+    if (!readmes[target.exerciseSlug]) {
+      readmes[target.exerciseSlug] = {}
+    }
+
+    readmes[target.exerciseSlug][target.lang] = content
+  }
+
+  return readmes
+}
+
+/** Ensure the translation slot exists before writing a description into it. */
+function ensureSlot(
+  lesson: Lesson,
+  lang: string
+): NonNullable<Lesson["translations"]>[string] {
+  if (!lesson.translations) {
+    lesson.translations = {}
+  }
+
+  if (!lesson.translations[lang]) {
+    lesson.translations[lang] = { completionId: 0, startedAt: Date.now() }
+  }
+
+  return lesson.translations[lang]
+}
+
+function applyStepResult(
+  step: StepWork,
+  result: TGenerateStepDescriptionsResult
+): { generated: number; missing: number } {
+  let generated = 0
+
+  for (const language of step.languages) {
+    if (!(language.lang in result.descriptionsByLanguage)) {
+      // Never answered: leave the slot untouched so a later run retries it.
+      // Writing a null plus a fingerprint here would look settled forever.
+      continue
+    }
+
+    const slot = ensureSlot(step.lesson, language.lang)
+    if (slot.descriptionSource === "human") {
+      continue
+    }
+
+    const description = result.descriptionsByLanguage[language.lang]
+    slot.description = description
+    slot.descriptionStatus = description ? "generated" : "none"
+    slot.descriptionSource = "auto"
+    slot.descriptionPromptVersion = result.promptVersion
+    slot.sourceContentHash = language.fingerprint.sha256
+    slot.sourceSimHash = language.fingerprint.simhash
+    generated += 1
+  }
+
+  return { generated, missing: result.missingLanguages.length }
+}
+
+export async function generateCourseDescriptions(
+  storage: CourseDescriptionsStorage,
+  courseSlug: string,
+  options: GenerateCourseDescriptionsOptions
+): Promise<GenerateCourseDescriptionsResult> {
+  const concurrency = options.concurrency ?? DEFAULT_DESCRIPTIONS_CONCURRENCY
+  const generate = options.generate ?? generateStepDescriptions
+
+  const syllabus = await storage.readSyllabus(courseSlug)
+  if (!syllabus) {
+    return emptyResult("skipped")
+  }
+
+  const exercises = await storage.listExercises(courseSlug)
+  if (exercises.length === 0) {
+    return emptyResult("skipped")
+  }
+
+  if (options.reconcile && storage.syllabusSyncStorage) {
+    try {
+      await synchronizeSyllabusWithBucket(
+        storage.syllabusSyncStorage,
+        courseSlug,
+        syllabus,
+        { prune: false }
+      )
+    } catch (error) {
+      // Provisional patch: never let reconciliation block generation.
+      console.error(
+        `[descriptions] Reconciliation failed for "${courseSlug}":`,
+        (error as Error).message
+      )
+    }
+  }
+
+  const readmes = await readCourseReadmes(
+    storage,
+    courseSlug,
+    exercises,
+    concurrency
+  )
+
+  const work = buildDescriptionWorkList({
+    syllabus,
+    exercises,
+    readmes,
+    promptVersion: options.promptVersion,
+    force: options.force,
+  })
+
+  if (work.length === 0) {
+    return emptyResult("skipped")
+  }
+
+  const result = emptyResult("processed")
+  result.steps = work.length
+
+  if (options.dryRun) {
+    result.generated = countWorkItems(work)
+    return result
+  }
+
+  // One completion per step, several in flight: a slow step never stalls the
+  // course and Rigobot is never hit with an unbounded burst.
+  const outcomes = await mapWithConcurrency(
+    work,
+    concurrency,
+    async step => ({
+      step,
+      generated: await generate(options.token, {
+        readmeContent: step.baseContent,
+        sourceLanguage: step.baseLanguage,
+        outputLanguages: step.languages.map(language => language.lang),
+        lessonTitle: step.lesson.title,
+        courseTitle: syllabus.courseInfo?.title,
+        targetWordCount: options.targetWordCount,
+        promptVersion: options.promptVersion,
+      }),
+    })
+  )
+
+  for (const outcome of outcomes) {
+    if (!outcome.generated) {
+      result.failed += 1
+      result.errors.push(
+        `generation failed for "${outcome.step.exerciseSlug}"`
+      )
+      continue
+    }
+
+    if (outcome.generated.durationSeconds !== null) {
+      result.durationsSeconds.push(outcome.generated.durationSeconds)
+    }
+
+    const applied = applyStepResult(outcome.step, outcome.generated)
+    result.generated += applied.generated
+    result.missing += applied.missing
+  }
+
+  if (result.generated === 0 && result.failed > 0) {
+    result.status = "failed"
+    return result
+  }
+
+  await storage.writeSyllabus(courseSlug, syllabus)
+
+  if (storage.reprojectManifest) {
+    try {
+      await storage.reprojectManifest(courseSlug)
+    } catch (error) {
+      // The syllabus (the source of truth) is already saved; a failed
+      // projection is recoverable and must not lose the generated text.
+      result.errors.push(
+        `manifest projection failed: ${(error as Error).message}`
+      )
+    }
+  }
+
+  return result
+}