@learnpack/learnpack 5.0.352 → 5.0.354

package/src/utils/api.ts

@@ -7,6 +7,15 @@ import * as dotenv from "dotenv"
 dotenv.config()
 
 const HOST = "https://breathecode.herokuapp.com"
+
+/**
+ * Student telemetry, and publication events. Both share the endpoint: a body
+ * carrying a top-level `event` key is dispatched as an event, anything else
+ * keeps being processed as telemetry.
+ */
+export const BREATHECODE_TELEMETRY_URL = `${HOST}/v1/assignment/me/telemetry`
+export const BREATHECODE_EVENTS_URL = BREATHECODE_TELEMETRY_URL
+
 export const RIGOBOT_HOST =
   process.env.RIGOBOT_HOST || "https://rigobot.herokuapp.com"
 export const RIGOBOT_REALTIME_HOST = "https://ai.4geeks.com"

package/src/utils/awsCredentials.ts

@@ -0,0 +1,57 @@
+/**
+ * AWS credentials for the published bucket, required rather than implicit.
+ *
+ * Building an `S3Client` with no credentials succeeds: the SDK defers to its
+ * provider chain and only fails at the first request, with
+ * "Could not load credentials from any providers". Inside a background job that
+ * surfaces minutes later, detached from the cause. Validating up front turns it
+ * into an obvious misconfiguration.
+ *
+ * Note this deliberately rejects the empty-string fallback pattern
+ * (`process.env.X || ""`): passing empty credentials defeats the provider chain
+ * instead of failing, which is how the opaque error appeared in the first place.
+ */
+
+export type AwsCredentials = {
+  accessKeyId: string;
+  secretAccessKey: string;
+};
+
+export function requireAwsCredentials(): AwsCredentials {
+  const accessKeyId = (process.env.AWS_ACCESS_KEY_ID || "").trim()
+  const secretAccessKey = (process.env.AWS_SECRET_ACCESS_KEY || "").trim()
+
+  const missing: string[] = []
+  if (!accessKeyId) {
+    missing.push("AWS_ACCESS_KEY_ID")
+  }
+
+  if (!secretAccessKey) {
+    missing.push("AWS_SECRET_ACCESS_KEY")
+  }
+
+  if (missing.length > 0) {
+    throw new Error(
+      `${missing.join(" and ")} (env) ${
+        missing.length > 1 ? "are" : "is"
+      } required to reach the published package bucket`
+    )
+  }
+
+  return { accessKeyId, secretAccessKey }
+}
+
+export function requireS3PackagesBucket(): string {
+  const bucket = (process.env.S3_PACKAGES_BUCKET || "").trim()
+  if (!bucket) {
+    throw new Error(
+      "S3_PACKAGES_BUCKET (env) is required: it names the bucket holding published packages"
+    )
+  }
+
+  return bucket
+}
+
+export function awsRegion(): string {
+  return (process.env.AWS_REGION || "").trim() || "us-east-1"
+}

package/src/utils/creatorUtilities.ts

@@ -1,4 +1,5 @@
 import Console from "./console"
+import { BREATHECODE_TELEMETRY_URL } from "./api"
 
 const frontMatter = require("front-matter")
 import * as MarkdownIt from "markdown-it"
@@ -189,7 +190,7 @@ export const makePackageInfo = (choices: any) => {
       .replace(/ /g, "-")
       .replace(/[^\w-]+/g, ""),
     telemetry: {
-      batch: "https://breathecode.herokuapp.com/v1/assignment/me/telemetry",
+      batch: BREATHECODE_TELEMETRY_URL,
     },
   }
   return packageInfo

package/src/utils/descriptionHash.ts

@@ -0,0 +1,196 @@
+import * as crypto from "crypto"
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const frontMatter = require("front-matter")
+
+/**
+ * Utilities to detect whether a step description is stale relative to its
+ * README content.
+ *
+ * Two signals are stored per (step, lang) at generation time:
+ *  - sha256(normalizeReadme(content))  -> exact fast-path + content-addressing
+ *  - simhash64(normalizeReadme(content)) -> near-duplicate significance filter
+ *
+ * The sweep re-hashes the current README and compares against the stored
+ * values: exact SHA-256 match => no-op; otherwise if the SimHash Hamming
+ * distance to the *generation baseline* is <= k the change is cosmetic and is
+ * ignored, else the description is regenerated. The H1 title is part of the
+ * normalized content (a deliberate product decision).
+ */
+
+/**
+ * Remove every image from markdown: inline `![alt](url)`, reference-style
+ * `![alt][ref]` and raw `<img>` tags. Regular links `[text](url)` are kept.
+ *
+ * Images are dropped from BOTH sides of the description pipeline — here for
+ * fingerprinting, and again before sending the README to the LLM. Adding,
+ * replacing or removing an illustration does not change what the lesson
+ * teaches, so it must not trigger a regeneration; and the alt text of generated
+ * illustrations is a full image-generation prompt (hundreds of tokens) that
+ * only invites the model to describe the picture instead of the lesson.
+ *
+ * Note the alt text may legitimately contain parentheses, so the alt is matched
+ * as "anything up to the closing bracket" rather than as balanced text.
+ */
+export function stripMarkdownImages(content: string): string {
+  return (
+    content
+      // ![alt](url) — alt may span lines and contain parentheses
+      .replace(/!\[[^\]]*]\([^)]*\)/g, "")
+      // ![alt][ref] — i.e. `!` followed by two bracketed groups
+      .replace(/!(?:\[[^\]]*]){2}/g, "")
+      // <img ...> / <img ... />
+      .replace(/<img\b[^>]*>/gi, "")
+      // [](url) left behind by linked images such as badges
+      .replace(/\[\s*]\([^)]*\)/g, "")
+  )
+}
+
+/**
+ * Canonicalize a README for hashing. Strips the frontmatter (metadata such as
+ * tutorial/intro video URLs, which do not describe the lesson), removes images
+ * (see `stripMarkdownImages`) and collapses all whitespace runs so that
+ * reformatting, blank lines and trailing spaces do not register as content
+ * changes. The H1 and the prose are preserved.
+ */
+export function normalizeReadme(content: string): string {
+  let body = content
+  try {
+    const parsed = frontMatter(content)
+    body = typeof parsed.body === "string" ? parsed.body : content
+  } catch {
+    body = content
+  }
+
+  return stripMarkdownImages(body)
+    .replace(/\r/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+}
+
+export function sha256(input: string): string {
+  return crypto.createHash("sha256").update(input).digest("hex")
+}
+
+// The build targets < ES2020, so BigInt literals are unavailable. The 64-bit
+// SimHash is therefore represented as two unsigned 32-bit halves { hi, lo } and
+// serialized as a 16-char hex string (hi then lo).
+
+function toHex32(x: number): string {
+  return (x >>> 0).toString(16).padStart(8, "0")
+}
+
+function popcount32(input: number): number {
+  let x = input >>> 0
+  let count = 0
+  while (x !== 0) {
+    count += x & 1
+    x >>>= 1
+  }
+
+  return count
+}
+
+function hashToken64(token: string): { hi: number; lo: number } {
+  // First 8 bytes (16 hex chars) of an md5 digest -> two 32-bit halves.
+  const hex = crypto.createHash("md5").update(token).digest("hex").slice(0, 16)
+  return {
+    hi: Number.parseInt(hex.slice(0, 8), 16) >>> 0,
+    lo: Number.parseInt(hex.slice(8, 16), 16) >>> 0,
+  }
+}
+
+/**
+ * 64-bit SimHash (Charikar) over whitespace-delimited tokens, returned as a
+ * zero-padded 16-char hex string.
+ */
+export function simhash64(input: string): string {
+  const tokens = input.split(/\s+/).filter(Boolean)
+  if (tokens.length === 0) {
+    return "0".repeat(16)
+  }
+
+  const weights = Array.from({ length: 64 }, () => 0)
+  for (const token of tokens) {
+    const { hi, lo } = hashToken64(token)
+    for (let i = 0; i < 32; i++) {
+      weights[i] += ((lo >>> i) & 1) === 1 ? 1 : -1
+      weights[32 + i] += ((hi >>> i) & 1) === 1 ? 1 : -1
+    }
+  }
+
+  let lo = 0
+  let hi = 0
+  for (let i = 0; i < 32; i++) {
+    if (weights[i] > 0) {
+      lo |= 1 << i
+    }
+
+    if (weights[32 + i] > 0) {
+      hi |= 1 << i
+    }
+  }
+
+  return toHex32(hi) + toHex32(lo)
+}
+
+/** Hamming distance between two 64-bit hex fingerprints. */
+export function hammingHex(a: string, b: string): number {
+  const hiA = Number.parseInt(a.slice(0, 8), 16) >>> 0
+  const loA = Number.parseInt(a.slice(8, 16), 16) >>> 0
+  const hiB = Number.parseInt(b.slice(0, 8), 16) >>> 0
+  const loB = Number.parseInt(b.slice(8, 16), 16) >>> 0
+  return popcount32((hiA ^ hiB) >>> 0) + popcount32((loA ^ loB) >>> 0)
+}
+
+export type ContentFingerprint = {
+  sha256: string;
+  simhash: string;
+};
+
+/** Convenience: normalize + both fingerprints in one call. */
+export function fingerprintReadme(content: string): ContentFingerprint {
+  const normalized = normalizeReadme(content)
+  return {
+    sha256: sha256(normalized),
+    simhash: simhash64(normalized),
+  }
+}
+
+/**
+ * Default significance threshold (Hamming distance over the 64-bit SimHash).
+ * k=3 is the classic near-duplicate threshold for 64-bit SimHash
+ * (Charikar 2002; Manku, Jain & Das Sarma, WWW 2007). Overridable via env.
+ */
+export function getSimhashThreshold(): number {
+  const raw = process.env.DESCRIPTION_SIMHASH_THRESHOLD
+  const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : 3
+}
+
+/**
+ * Decide whether a description generated from `previous` is stale given the
+ * `current` README content. Exact match => not stale; else stale only if the
+ * SimHash Hamming distance exceeds the threshold (cosmetic edits are ignored).
+ */
+export function isDescriptionStale(
+  previous: { sha256?: string; simhash?: string } | undefined,
+  current: ContentFingerprint,
+  threshold: number = getSimhashThreshold()
+): boolean {
+  if (!previous || !previous.sha256) {
+    // Never generated -> treat as needing generation (i.e. "stale").
+    return true
+  }
+
+  if (previous.sha256 === current.sha256) {
+    return false
+  }
+
+  if (!previous.simhash) {
+    // No baseline fingerprint to compare against -> be conservative.
+    return true
+  }
+
+  return hammingHex(previous.simhash, current.simhash) > threshold
+}

package/src/utils/descriptions/backfillEvents.ts

@@ -0,0 +1,152 @@
+import { v4 as uuidv4 } from "uuid"
+import { PackageManifest } from "../packageManifest"
+import {
+  buildPackageManifestUpdatedEvent,
+  fetchPackageInfo,
+  sendPublishEvent,
+} from "../publishEvents"
+import {
+  CourseDescriptionsStorage,
+  GenerateCourseDescriptionsResult,
+} from "./generateCourseDescriptions"
+
+/**
+ * `package_manifest_updated` for courses reconciled by the backfill.
+ *
+ * The backfill regenerates descriptions of packages that are already published,
+ * so breathecode needs the same notification a publication sends — but there is
+ * no publication here: no `package_published` precedes this event, and no
+ * journal tracks it (the sweep resumes publications, and a backfill is simply
+ * re-run instead).
+ *
+ * That is what makes `publish_id` awkward. The field exists to correlate the two
+ * events of one publication, a pair this event has no half of. Since it is
+ * required, the value has to be something that cannot be mistaken for a real
+ * publication: a bare uuid — the obvious choice — would be exactly that mistake,
+ * and if breathecode ever starts pairing events it would search forever for a
+ * `package_published` that was never sent.
+ *
+ * Hence `backfill-{runId}-{slug}`:
+ *
+ * - the `backfill-` prefix makes the origin readable in the webhook log and
+ *   discriminable in code (`isBackfillPublishId`); real ids are bare uuids from
+ *   `createJournal`, so the two spaces cannot collide;
+ * - `{slug}` keeps it unique per event, which a per-run id alone would not be;
+ * - a `{runId}` shared by every course of one invocation makes "every event from
+ *   the run I fired at 15:40" a single substring query, which is the question
+ *   you actually ask when the catalogue is backfilled in small batches.
+ *
+ * Encoding this in `publish_id`, rather than adding an `origin` field, keeps the
+ * event contract untouched: breathecode ignores `publish_id` today, so this
+ * costs them nothing and needs no coordination. If they ever start consuming the
+ * provenance, it can be promoted to a field of its own.
+ */
+
+export const BACKFILL_PUBLISH_ID_PREFIX = "backfill-"
+
+/** One per script invocation, shared by every course it touches. */
+export function newBackfillRunId(): string {
+  return uuidv4()
+}
+
+export function backfillPublishId(runId: string, courseSlug: string): string {
+  return `${BACKFILL_PUBLISH_ID_PREFIX}${runId}-${courseSlug}`
+}
+
+export function isBackfillPublishId(publishId: string): boolean {
+  return publishId.startsWith(BACKFILL_PUBLISH_ID_PREFIX)
+}
+
+/**
+ * `skipped` means the event was never attempted, which is not a failure: a
+ * course the backfill did not change has nothing to announce.
+ */
+export type BackfillEventOutcome = "skipped" | "delivered" | "failed";
+
+export type EmitBackfillManifestEventParams = {
+  courseSlug: string;
+  runId: string;
+  result: GenerateCourseDescriptionsResult;
+  storage: CourseDescriptionsStorage;
+  rigobotToken: string;
+  breathecodeToken: string;
+  /** Injection points for tests. */
+  emit?: typeof sendPublishEvent;
+  readPackageInfo?: typeof fetchPackageInfo;
+};
+
+/**
+ * Announce one reconciled course. Never throws: the descriptions and the
+ * manifest are already written by the time this runs, and losing the
+ * notification must not turn a successful course into a failed one.
+ */
+export async function emitBackfillManifestEvent(
+  params: EmitBackfillManifestEventParams
+): Promise<BackfillEventOutcome> {
+  const { courseSlug, result } = params
+
+  if (result.status === "skipped") {
+    return "skipped"
+  }
+
+  const emit = params.emit ?? sendPublishEvent
+  const readPackageInfo = params.readPackageInfo ?? fetchPackageInfo
+
+  // Enriching the payload must never prevent the delivery: both the manifest and
+  // the package info are optional in the contract, and an event with less
+  // context beats an event that never arrives.
+  let manifest: PackageManifest | null = null
+  try {
+    if (params.storage.readManifest) {
+      manifest = await params.storage.readManifest(courseSlug)
+    }
+  } catch (error) {
+    console.error(
+      `[backfill-events] Could not read the manifest of "${courseSlug}", reporting without it:`,
+      (error as Error).message
+    )
+  }
+
+  let packageInfo: Record<string, unknown> | null = null
+  try {
+    packageInfo = await readPackageInfo(courseSlug, params.rigobotToken)
+  } catch (error) {
+    console.error(
+      `[backfill-events] Could not read the package "${courseSlug}", reporting without it:`,
+      (error as Error).message
+    )
+  }
+
+  try {
+    const delivered = await emit(
+      buildPackageManifestUpdatedEvent(
+        {
+          publishId: backfillPublishId(params.runId, courseSlug),
+          courseSlug,
+          packageInfo,
+          manifest,
+        },
+        // A failed projection does not fail the run — the syllabus is saved
+        // either way — but this event reports the manifest, so it cannot claim
+        // success over one that was never rewritten.
+        result.status === "failed" || result.manifestProjected === false ?
+          "failed" :
+          "success",
+        {
+          generated: result.generated,
+          failed: result.failed,
+          missing: result.missing,
+        }
+      ),
+      params.breathecodeToken
+    )
+
+    return delivered ? "delivered" : "failed"
+  } catch (error) {
+    console.error(
+      `[backfill-events] Could not deliver the manifest event for "${courseSlug}":`,
+      (error as Error).message
+    )
+    return "failed"
+  }
+}

package/src/utils/descriptions/gcsStorage.ts

@@ -0,0 +1,67 @@
+import { Bucket } from "@google-cloud/storage"
+import { Syllabus } from "../../models/creator"
+import { buildConfig } from "../configBuilder"
+import { createGcsSyllabusSyncStorage } from "../syllabusSync"
+import { CourseDescriptionsStorage } from "./generateCourseDescriptions"
+import { CourseExercise } from "./workList"
+
+/**
+ * Draft-bucket (GCS) adapter.
+ *
+ * Descriptions are never generated against this bucket — it keeps moving while
+ * the teacher edits, so it is not a stable snapshot. It is only written by the
+ * mirror-back, which copies descriptions generated from the published package
+ * whenever the draft content still matches.
+ *
+ * Note there is no `reprojectManifest`: the manifest is only rebuilt when
+ * publishing.
+ */
+
+export const gcsSyllabusPath = (courseSlug: string) =>
+  `courses/${courseSlug}/.learn/initialSyllabus.json`
+
+export const gcsReadmePath = (
+  courseSlug: string,
+  exerciseSlug: string,
+  fileName: string
+) => `courses/${courseSlug}/exercises/${exerciseSlug}/${fileName}`
+
+export function createGcsDescriptionsStorage(
+  bucket: Bucket
+): CourseDescriptionsStorage {
+  return {
+    async readSyllabus(courseSlug) {
+      try {
+        const [buf] = await bucket.file(gcsSyllabusPath(courseSlug)).download()
+        return JSON.parse(buf.toString()) as Syllabus
+      } catch {
+        return null
+      }
+    },
+    async writeSyllabus(courseSlug, syllabus) {
+      await bucket
+        .file(gcsSyllabusPath(courseSlug))
+        .save(Buffer.from(JSON.stringify(syllabus), "utf8"), {
+          contentType: "application/json",
+        })
+    },
+    async listExercises(courseSlug): Promise<CourseExercise[]> {
+      const { exercises } = await buildConfig(bucket, courseSlug)
+      return exercises.map(exercise => ({
+        slug: exercise.slug,
+        translations: exercise.translations || {},
+      }))
+    },
+    async readReadme(courseSlug, exerciseSlug, fileName) {
+      try {
+        const [buf] = await bucket
+          .file(gcsReadmePath(courseSlug, exerciseSlug, fileName))
+          .download()
+        return buf.toString()
+      } catch {
+        return null
+      }
+    },
+    syllabusSyncStorage: createGcsSyllabusSyncStorage(bucket),
+  }
+}