@learnpack/learnpack 5.0.351 → 5.0.353

package/lib/scripts/descriptionsS3Backfill.js

@@ -0,0 +1,157 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-await-in-loop */
+/**
+ * Fase 0a — S3 descriptions backfill (step 2 of the S3 pipeline).
+ *
+ * One-off fill of the published catalogue (~400+ courses). The per-course work
+ * is the shared description service (`generateCourseDescriptions`), the same one
+ * the post-publish stage and the sweep use; this script only decides WHICH
+ * courses to run it on and in what batches.
+ *
+ *   - Processes at most --limit courses per run (default 50), ordered by the
+ *     initialSyllabus.json modification time (most recently edited first),
+ *     skipping courses already settled at the current prompt version. Re-run to
+ *     advance through the catalogue.
+ *   - Or target a single course with --slug (remediation / on-demand).
+ *
+ * Writes descriptions into the published initialSyllabus.json. Manifest
+ * projection is a separate pass here: run the package-manifest backfill
+ * (runBatch) afterwards, which is cheaper than re-projecting course by course.
+ *
+ * Secrets/infra via env: RIGOBOT_SYSTEM_TOKEN (required), plus S3_PACKAGES_BUCKET
+ * and AWS_REGION as fallbacks for --s3-bucket / --region.
+ *
+ * Per-run flags:
+ *   --s3-bucket <name>   S3 bucket (or env S3_PACKAGES_BUCKET, default learnpack-paquetes)
+ *   --region <region>    AWS region (default us-east-1, or env AWS_REGION)
+ *   --slug <slug>        process a single course (remediation / on-demand)
+ *   --limit <n>          courses per run (default 50)
+ *   --target-words <n>   words per description (default 25)
+ *   --concurrency <n>    completions in flight per course (default 5)
+ *   --dry-run            preview without writing/generating
+ *   --no-reconcile       skip the additive syllabus reconciliation
+ *   --force              regenerate even courses already settled at this prompt version
+ *
+ * Examples:
+ *   node lib/scripts/descriptionsS3Backfill.js --limit 50
+ *   node lib/scripts/descriptionsS3Backfill.js --slug my-course
+ *   node lib/scripts/descriptionsS3Backfill.js --dry-run --limit 10
+ */
+const node_util_1 = require("node:util");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const packageManifest_1 = require("../utils/packageManifest");
+const generateCourseDescriptions_1 = require("../utils/descriptions/generateCourseDescriptions");
+const s3Storage_1 = require("../utils/descriptions/s3Storage");
+const workList_1 = require("../utils/descriptions/workList");
+const SYLLABUS_KEY_PATTERN = /^([^/]+)\/\.learn\/initialSyllabus\.json$/;
+/** Courses (by slug) ordered by initialSyllabus.json LastModified, newest first. */
+async function listCoursesByRecency(s3, bucket) {
+    var _a, _b;
+    const mtimes = new Map();
+    let continuationToken;
+    do {
+        const response = (await s3.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: bucket,
+            ContinuationToken: continuationToken,
+        })));
+        for (const object of response.Contents || []) {
+            const match = (_a = object.Key) === null || _a === void 0 ? void 0 : _a.match(SYLLABUS_KEY_PATTERN);
+            if (match) {
+                mtimes.set(match[1], ((_b = object.LastModified) === null || _b === void 0 ? void 0 : _b.getTime()) || 0);
+            }
+        }
+        continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+    return [...mtimes.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .map(([slug]) => slug);
+}
+async function main() {
+    var _a, _b, _c;
+    const { values } = (0, node_util_1.parseArgs)({
+        options: {
+            "s3-bucket": { type: "string" },
+            region: { type: "string" },
+            slug: { type: "string" },
+            limit: { type: "string" },
+            "target-words": { type: "string" },
+            concurrency: { type: "string" },
+            "dry-run": { type: "boolean", default: false },
+            "no-reconcile": { type: "boolean", default: false },
+            force: { type: "boolean", default: false },
+        },
+    });
+    const token = process.env.RIGOBOT_SYSTEM_TOKEN;
+    if (!token) {
+        console.error("[s3-backfill] RIGOBOT_SYSTEM_TOKEN (env) is required");
+        process.exit(1);
+    }
+    const bucket = values["s3-bucket"] ||
+        process.env.S3_PACKAGES_BUCKET ||
+        "learnpack-paquetes";
+    const region = values.region || process.env.AWS_REGION || "us-east-1";
+    const limit = Number.parseInt(values.limit || "50", 10) || 50;
+    const dryRun = (_a = values["dry-run"]) !== null && _a !== void 0 ? _a : false;
+    const force = (_b = values.force) !== null && _b !== void 0 ? _b : false;
+    const s3 = new client_s3_1.S3Client({ region });
+    // Manifests are projected in a separate pass after this backfill.
+    const storage = (0, s3Storage_1.createS3DescriptionsStorage)(s3, bucket, {
+        reprojectManifest: false,
+    });
+    const singleSlug = values.slug;
+    const slugs = singleSlug ?
+        [singleSlug] :
+        await listCoursesByRecency(s3, bucket);
+    console.log(`[s3-backfill] Starting${singleSlug ? ` (slug=${singleSlug})` : ` (limit=${limit})`}${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)`);
+    let processed = 0;
+    let generated = 0;
+    let failed = 0;
+    for (const slug of slugs) {
+        if (!singleSlug && processed >= limit) {
+            break;
+        }
+        try {
+            // Cheap pre-filter: skip finished courses without reading their READMEs.
+            if (!force) {
+                const syllabus = await storage.readSyllabus(slug);
+                if (!syllabus) {
+                    continue;
+                }
+                const exercises = await (0, s3Storage_1.fetchExercises)(s3, bucket, slug);
+                if (exercises.length === 0 || (0, workList_1.isCourseSettled)(syllabus, exercises)) {
+                    continue;
+                }
+            }
+            const result = await (0, generateCourseDescriptions_1.generateCourseDescriptions)(storage, slug, {
+                token,
+                dryRun,
+                force,
+                reconcile: !((_c = values["no-reconcile"]) !== null && _c !== void 0 ? _c : false),
+                concurrency: Number.parseInt(values.concurrency || "", 10) ||
+                    generateCourseDescriptions_1.DEFAULT_DESCRIPTIONS_CONCURRENCY,
+                targetWordCount: Number.parseInt(values["target-words"] || "", 10) ||
+                    packageManifest_1.DESCRIPTION_TARGET_WORD_COUNT,
+            });
+            if (result.status === "skipped") {
+                continue;
+            }
+            processed += 1;
+            generated += result.generated;
+            failed += result.failed;
+            console.log(`[s3-backfill] "${slug}": ${result.steps} step(s), ${result.generated} description(s)${dryRun ? " (dry-run)" : ""}${result.failed > 0 ? `, ${result.failed} failed` : ""}${result.missing > 0 ? `, ${result.missing} unanswered` : ""}`);
+        }
+        catch (error) {
+            processed += 1;
+            failed += 1;
+            console.error(`[s3-backfill] Failed processing "${slug}":`, error.message);
+        }
+    }
+    console.log(`[s3-backfill] Done. ${processed} course(s), ${generated} description(s) written, ${failed} failure(s).`);
+}
+main()
+    .then(() => process.exit(0))
+    .catch(error => {
+    console.error("[s3-backfill] Fatal error:", error);
+    process.exit(1);
+});

package/lib/scripts/descriptionsSweep.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/descriptionsSweep.js

@@ -0,0 +1,142 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-await-in-loop */
+/**
+ * Descriptions sweep — safety net for the post-publish stage.
+ *
+ * The stage that generates descriptions runs inside the web process, so a dyno
+ * restart mid-flight leaves a package published but not described and, worse,
+ * breathecode waiting for a manifest event that was promised and never arrived.
+ * This job finishes those publications.
+ *
+ * Runs as a one-off dyno via Heroku Scheduler (hourly is plenty):
+ *   node lib/scripts/descriptionsSweep.js
+ *
+ * Two sources of work:
+ *   1. Publish journals (`publish-journal/` in GCS) — the primary one. Each
+ *      journal that stalled is resumed: missing descriptions are generated, the
+ *      draft is mirrored, and the owed event is emitted. Journals are dropped
+ *      once complete and abandoned after a few attempts.
+ *   2. A deep scan of published courses (DESCRIPTIONS_SWEEP_FULL=1) — catches
+ *      packages that never got a journal at all, at the cost of reading every
+ *      course. Off by default.
+ *
+ * Required env: GCP_CREDENTIALS_JSON, RIGOBOT_SYSTEM_TOKEN.
+ * Optional env: GCP_BUCKET_NAME, S3_PACKAGES_BUCKET, AWS_REGION,
+ * CLOUDFRONT_DISTRIBUTION_ID, BREATHECODE_SYSTEM_TOKEN (needed to emit the
+ * pending events), DESCRIPTIONS_SWEEP_FULL, DESCRIPTIONS_SWEEP_LIMIT,
+ * DESCRIPTION_STALE_AFTER_MS.
+ */
+const storage_1 = require("@google-cloud/storage");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const publishJournal_1 = require("../utils/publishJournal");
+const packageManifestBackfill_1 = require("../utils/s3/packageManifestBackfill");
+const gcsBucketName_1 = require("../utils/gcsBucketName");
+const gcsStorage_1 = require("../utils/descriptions/gcsStorage");
+const s3Storage_1 = require("../utils/descriptions/s3Storage");
+const generateCourseDescriptions_1 = require("../utils/descriptions/generateCourseDescriptions");
+const resumePublication_1 = require("../utils/descriptions/resumePublication");
+async function main() {
+    const credentialsEnv = process.env.GCP_CREDENTIALS_JSON;
+    if (!credentialsEnv) {
+        console.error("[sweep] GCP_CREDENTIALS_JSON (env) is required");
+        process.exit(1);
+    }
+    const rigobotToken = process.env.RIGOBOT_SYSTEM_TOKEN;
+    if (!rigobotToken) {
+        console.error("[sweep] RIGOBOT_SYSTEM_TOKEN (env) is required");
+        process.exit(1);
+    }
+    const breathecodeToken = process.env.BREATHECODE_SYSTEM_TOKEN;
+    if (!breathecodeToken) {
+        console.warn("[sweep] BREATHECODE_SYSTEM_TOKEN is not set: pending events cannot be emitted");
+    }
+    const bucket = new storage_1.Storage({
+        credentials: JSON.parse(credentialsEnv),
+    }).bucket((0, gcsBucketName_1.requireGcsBucketName)());
+    const journalStorage = (0, publishJournal_1.createGcsJournalStorage)(bucket);
+    const publishedStorage = (0, s3Storage_1.createS3DescriptionsStorageFromEnv)();
+    const draftStorage = (0, gcsStorage_1.createGcsDescriptionsStorage)(bucket);
+    const staleAfterMs = Number.parseInt(process.env.DESCRIPTION_STALE_AFTER_MS || "", 10) ||
+        resumePublication_1.DEFAULT_STALE_AFTER_MS;
+    const refs = await (0, publishJournal_1.listJournalRefs)(journalStorage);
+    console.log(`[sweep] ${refs.length} unfinished publication(s)`);
+    const tally = {
+        "skipped-recent": 0,
+        "skipped-abandoned": 0,
+        abandoned: 0,
+        resumed: 0,
+        "nothing-to-do": 0,
+    };
+    for (const ref of refs) {
+        try {
+            const journal = await (0, publishJournal_1.readJournalByKey)(journalStorage, ref.key);
+            if (!journal) {
+                continue;
+            }
+            const outcome = await (0, resumePublication_1.resumePublication)(journal, {
+                journalStorage,
+                publishedStorage,
+                draftStorage,
+                rigobotToken,
+                breathecodeToken,
+                staleAfterMs,
+            });
+            tally[outcome] += 1;
+            if (outcome === "resumed") {
+                console.log(`[sweep] Resumed "${ref.courseSlug}" (${ref.publishId})`);
+            }
+        }
+        catch (error) {
+            console.error(`[sweep] Failed resuming "${ref.courseSlug}":`, error.message);
+        }
+    }
+    console.log(`[sweep] Journals: ${tally.resumed} resumed, ${tally["skipped-recent"]} still in flight, ` +
+        `${tally["nothing-to-do"]} nothing to do, ${tally.abandoned} abandoned, ` +
+        `${tally["skipped-abandoned"]} already abandoned`);
+    if (process.env.DESCRIPTIONS_SWEEP_FULL === "1") {
+        await deepScan(rigobotToken, publishedStorage);
+    }
+}
+/**
+ * Fallback for packages with no journal at all (published before this existed,
+ * or whose journal write failed). Expensive: it reads every published course,
+ * so it is opt-in rather than the default path.
+ */
+async function deepScan(rigobotToken, publishedStorage) {
+    const s3 = new client_s3_1.S3Client({
+        region: process.env.AWS_REGION || "us-east-1",
+    });
+    const bucket = process.env.S3_PACKAGES_BUCKET || "learnpack-paquetes";
+    const limit = Number.parseInt(process.env.DESCRIPTIONS_SWEEP_LIMIT || "", 10) || 25;
+    const slugs = await (0, packageManifestBackfill_1.listPublishedSlugs)(s3, bucket);
+    console.log(`[sweep] Deep scan over ${slugs.length} published course(s), limit ${limit}`);
+    let processed = 0;
+    for (const slug of slugs) {
+        if (processed >= limit) {
+            break;
+        }
+        try {
+            const result = await (0, generateCourseDescriptions_1.generateCourseDescriptions)(publishedStorage, slug, {
+                token: rigobotToken,
+                reconcile: true,
+            });
+            if (result.status === "skipped") {
+                continue;
+            }
+            processed += 1;
+            console.log(`[sweep] "${slug}": ${result.generated} description(s), ${result.failed} failed`);
+        }
+        catch (error) {
+            processed += 1;
+            console.error(`[sweep] Failed on "${slug}":`, error.message);
+        }
+    }
+    console.log(`[sweep] Deep scan done. ${processed} course(s) touched.`);
+}
+main()
+    .then(() => process.exit(0))
+    .catch(error => {
+    console.error("[sweep] Fatal error:", error);
+    process.exit(1);
+});

package/lib/utils/api.d.ts

@@ -1,3 +1,10 @@
+/**
+ * Student telemetry, and publication events. Both share the endpoint: a body
+ * carrying a top-level `event` key is dispatched as an event, anything else
+ * keeps being processed as telemetry.
+ */
+export declare const BREATHECODE_TELEMETRY_URL = "https://breathecode.herokuapp.com/v1/assignment/me/telemetry";
+export declare const BREATHECODE_EVENTS_URL = "https://breathecode.herokuapp.com/v1/assignment/me/telemetry";
 export declare const RIGOBOT_HOST: string;
 export declare const RIGOBOT_REALTIME_HOST = "https://ai.4geeks.com";
 type TConsumableSlug = "ai-conversation-message" | "ai-compilation" | "ai-tutorial-generation" | "ai-generation" | "learnpack-publish";

package/lib/utils/api.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getCurrentTechnologies = exports.fetchTechnologies = exports.doesAssetExists = exports.createAsset = exports.validateToken = exports.listUserAcademies = exports.getConsumable = exports.countConsumables = exports.RIGOBOT_REALTIME_HOST = exports.RIGOBOT_HOST = void 0;
+exports.getCurrentTechnologies = exports.fetchTechnologies = exports.doesAssetExists = exports.createAsset = exports.validateToken = exports.listUserAcademies = exports.getConsumable = exports.countConsumables = exports.RIGOBOT_REALTIME_HOST = exports.RIGOBOT_HOST = exports.BREATHECODE_EVENTS_URL = exports.BREATHECODE_TELEMETRY_URL = void 0;
 exports.resolveLearnpackPackageId = resolveLearnpackPackageId;
 const console_1 = require("../utils/console");
 const storage = require("node-persist");
@@ -9,6 +9,13 @@ const axios_1 = require("axios");
 const dotenv = require("dotenv");
 dotenv.config();
 const HOST = "https://breathecode.herokuapp.com";
+/**
+ * Student telemetry, and publication events. Both share the endpoint: a body
+ * carrying a top-level `event` key is dispatched as an event, anything else
+ * keeps being processed as telemetry.
+ */
+exports.BREATHECODE_TELEMETRY_URL = `${HOST}/v1/assignment/me/telemetry`;
+exports.BREATHECODE_EVENTS_URL = exports.BREATHECODE_TELEMETRY_URL;
 exports.RIGOBOT_HOST = process.env.RIGOBOT_HOST || "https://rigobot.herokuapp.com";
 exports.RIGOBOT_REALTIME_HOST = "https://ai.4geeks.com";
 // export const RIGOBOT_REALTIME_HOST = "http://127.0.0.1:8003"

package/lib/utils/creatorUtilities.js

@@ -17,6 +17,7 @@ exports.countSentences = countSentences;
 exports.howManyDifficultParagraphs = howManyDifficultParagraphs;
 exports.insertStepInCorrectPosition = insertStepInCorrectPosition;
 const console_1 = require("./console");
+const api_1 = require("./api");
 const frontMatter = require("front-matter");
 const MarkdownIt = require("markdown-it");
 const syllable_1 = require("syllable");
@@ -146,7 +147,7 @@ const makePackageInfo = (choices) => {
             .replace(/ /g, "-")
             .replace(/[^\w-]+/g, ""),
         telemetry: {
-            batch: "https://breathecode.herokuapp.com/v1/assignment/me/telemetry",
+            batch: api_1.BREATHECODE_TELEMETRY_URL,
         },
     };
     return packageInfo;

package/lib/utils/descriptionHash.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Utilities to detect whether a step description is stale relative to its
+ * README content.
+ *
+ * Two signals are stored per (step, lang) at generation time:
+ *  - sha256(normalizeReadme(content))  -> exact fast-path + content-addressing
+ *  - simhash64(normalizeReadme(content)) -> near-duplicate significance filter
+ *
+ * The sweep re-hashes the current README and compares against the stored
+ * values: exact SHA-256 match => no-op; otherwise if the SimHash Hamming
+ * distance to the *generation baseline* is <= k the change is cosmetic and is
+ * ignored, else the description is regenerated. The H1 title is part of the
+ * normalized content (a deliberate product decision).
+ */
+/**
+ * Remove every image from markdown: inline `![alt](url)`, reference-style
+ * `![alt][ref]` and raw `<img>` tags. Regular links `[text](url)` are kept.
+ *
+ * Images are dropped from BOTH sides of the description pipeline — here for
+ * fingerprinting, and again before sending the README to the LLM. Adding,
+ * replacing or removing an illustration does not change what the lesson
+ * teaches, so it must not trigger a regeneration; and the alt text of generated
+ * illustrations is a full image-generation prompt (hundreds of tokens) that
+ * only invites the model to describe the picture instead of the lesson.
+ *
+ * Note the alt text may legitimately contain parentheses, so the alt is matched
+ * as "anything up to the closing bracket" rather than as balanced text.
+ */
+export declare function stripMarkdownImages(content: string): string;
+/**
+ * Canonicalize a README for hashing. Strips the frontmatter (metadata such as
+ * tutorial/intro video URLs, which do not describe the lesson), removes images
+ * (see `stripMarkdownImages`) and collapses all whitespace runs so that
+ * reformatting, blank lines and trailing spaces do not register as content
+ * changes. The H1 and the prose are preserved.
+ */
+export declare function normalizeReadme(content: string): string;
+export declare function sha256(input: string): string;
+/**
+ * 64-bit SimHash (Charikar) over whitespace-delimited tokens, returned as a
+ * zero-padded 16-char hex string.
+ */
+export declare function simhash64(input: string): string;
+/** Hamming distance between two 64-bit hex fingerprints. */
+export declare function hammingHex(a: string, b: string): number;
+export type ContentFingerprint = {
+    sha256: string;
+    simhash: string;
+};
+/** Convenience: normalize + both fingerprints in one call. */
+export declare function fingerprintReadme(content: string): ContentFingerprint;
+/**
+ * Default significance threshold (Hamming distance over the 64-bit SimHash).
+ * k=3 is the classic near-duplicate threshold for 64-bit SimHash
+ * (Charikar 2002; Manku, Jain & Das Sarma, WWW 2007). Overridable via env.
+ */
+export declare function getSimhashThreshold(): number;
+/**
+ * Decide whether a description generated from `previous` is stale given the
+ * `current` README content. Exact match => not stale; else stale only if the
+ * SimHash Hamming distance exceeds the threshold (cosmetic edits are ignored).
+ */
+export declare function isDescriptionStale(previous: {
+    sha256?: string;
+    simhash?: string;
+} | undefined, current: ContentFingerprint, threshold?: number): boolean;

package/lib/utils/descriptionHash.js

@@ -0,0 +1,173 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stripMarkdownImages = stripMarkdownImages;
+exports.normalizeReadme = normalizeReadme;
+exports.sha256 = sha256;
+exports.simhash64 = simhash64;
+exports.hammingHex = hammingHex;
+exports.fingerprintReadme = fingerprintReadme;
+exports.getSimhashThreshold = getSimhashThreshold;
+exports.isDescriptionStale = isDescriptionStale;
+const crypto = require("crypto");
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const frontMatter = require("front-matter");
+/**
+ * Utilities to detect whether a step description is stale relative to its
+ * README content.
+ *
+ * Two signals are stored per (step, lang) at generation time:
+ *  - sha256(normalizeReadme(content))  -> exact fast-path + content-addressing
+ *  - simhash64(normalizeReadme(content)) -> near-duplicate significance filter
+ *
+ * The sweep re-hashes the current README and compares against the stored
+ * values: exact SHA-256 match => no-op; otherwise if the SimHash Hamming
+ * distance to the *generation baseline* is <= k the change is cosmetic and is
+ * ignored, else the description is regenerated. The H1 title is part of the
+ * normalized content (a deliberate product decision).
+ */
+/**
+ * Remove every image from markdown: inline `![alt](url)`, reference-style
+ * `![alt][ref]` and raw `<img>` tags. Regular links `[text](url)` are kept.
+ *
+ * Images are dropped from BOTH sides of the description pipeline — here for
+ * fingerprinting, and again before sending the README to the LLM. Adding,
+ * replacing or removing an illustration does not change what the lesson
+ * teaches, so it must not trigger a regeneration; and the alt text of generated
+ * illustrations is a full image-generation prompt (hundreds of tokens) that
+ * only invites the model to describe the picture instead of the lesson.
+ *
+ * Note the alt text may legitimately contain parentheses, so the alt is matched
+ * as "anything up to the closing bracket" rather than as balanced text.
+ */
+function stripMarkdownImages(content) {
+    return (content
+        // ![alt](url) — alt may span lines and contain parentheses
+        .replace(/!\[[^\]]*]\([^)]*\)/g, "")
+        // ![alt][ref] — i.e. `!` followed by two bracketed groups
+        .replace(/!(?:\[[^\]]*]){2}/g, "")
+        // <img ...> / <img ... />
+        .replace(/<img\b[^>]*>/gi, "")
+        // [](url) left behind by linked images such as badges
+        .replace(/\[\s*]\([^)]*\)/g, ""));
+}
+/**
+ * Canonicalize a README for hashing. Strips the frontmatter (metadata such as
+ * tutorial/intro video URLs, which do not describe the lesson), removes images
+ * (see `stripMarkdownImages`) and collapses all whitespace runs so that
+ * reformatting, blank lines and trailing spaces do not register as content
+ * changes. The H1 and the prose are preserved.
+ */
+function normalizeReadme(content) {
+    let body = content;
+    try {
+        const parsed = frontMatter(content);
+        body = typeof parsed.body === "string" ? parsed.body : content;
+    }
+    catch (_a) {
+        body = content;
+    }
+    return stripMarkdownImages(body)
+        .replace(/\r/g, "")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function sha256(input) {
+    return crypto.createHash("sha256").update(input).digest("hex");
+}
+// The build targets < ES2020, so BigInt literals are unavailable. The 64-bit
+// SimHash is therefore represented as two unsigned 32-bit halves { hi, lo } and
+// serialized as a 16-char hex string (hi then lo).
+function toHex32(x) {
+    return (x >>> 0).toString(16).padStart(8, "0");
+}
+function popcount32(input) {
+    let x = input >>> 0;
+    let count = 0;
+    while (x !== 0) {
+        count += x & 1;
+        x >>>= 1;
+    }
+    return count;
+}
+function hashToken64(token) {
+    // First 8 bytes (16 hex chars) of an md5 digest -> two 32-bit halves.
+    const hex = crypto.createHash("md5").update(token).digest("hex").slice(0, 16);
+    return {
+        hi: Number.parseInt(hex.slice(0, 8), 16) >>> 0,
+        lo: Number.parseInt(hex.slice(8, 16), 16) >>> 0,
+    };
+}
+/**
+ * 64-bit SimHash (Charikar) over whitespace-delimited tokens, returned as a
+ * zero-padded 16-char hex string.
+ */
+function simhash64(input) {
+    const tokens = input.split(/\s+/).filter(Boolean);
+    if (tokens.length === 0) {
+        return "0".repeat(16);
+    }
+    const weights = Array.from({ length: 64 }, () => 0);
+    for (const token of tokens) {
+        const { hi, lo } = hashToken64(token);
+        for (let i = 0; i < 32; i++) {
+            weights[i] += ((lo >>> i) & 1) === 1 ? 1 : -1;
+            weights[32 + i] += ((hi >>> i) & 1) === 1 ? 1 : -1;
+        }
+    }
+    let lo = 0;
+    let hi = 0;
+    for (let i = 0; i < 32; i++) {
+        if (weights[i] > 0) {
+            lo |= 1 << i;
+        }
+        if (weights[32 + i] > 0) {
+            hi |= 1 << i;
+        }
+    }
+    return toHex32(hi) + toHex32(lo);
+}
+/** Hamming distance between two 64-bit hex fingerprints. */
+function hammingHex(a, b) {
+    const hiA = Number.parseInt(a.slice(0, 8), 16) >>> 0;
+    const loA = Number.parseInt(a.slice(8, 16), 16) >>> 0;
+    const hiB = Number.parseInt(b.slice(0, 8), 16) >>> 0;
+    const loB = Number.parseInt(b.slice(8, 16), 16) >>> 0;
+    return popcount32((hiA ^ hiB) >>> 0) + popcount32((loA ^ loB) >>> 0);
+}
+/** Convenience: normalize + both fingerprints in one call. */
+function fingerprintReadme(content) {
+    const normalized = normalizeReadme(content);
+    return {
+        sha256: sha256(normalized),
+        simhash: simhash64(normalized),
+    };
+}
+/**
+ * Default significance threshold (Hamming distance over the 64-bit SimHash).
+ * k=3 is the classic near-duplicate threshold for 64-bit SimHash
+ * (Charikar 2002; Manku, Jain & Das Sarma, WWW 2007). Overridable via env.
+ */
+function getSimhashThreshold() {
+    const raw = process.env.DESCRIPTION_SIMHASH_THRESHOLD;
+    const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
+    return Number.isFinite(parsed) && parsed >= 0 ? parsed : 3;
+}
+/**
+ * Decide whether a description generated from `previous` is stale given the
+ * `current` README content. Exact match => not stale; else stale only if the
+ * SimHash Hamming distance exceeds the threshold (cosmetic edits are ignored).
+ */
+function isDescriptionStale(previous, current, threshold = getSimhashThreshold()) {
+    if (!previous || !previous.sha256) {
+        // Never generated -> treat as needing generation (i.e. "stale").
+        return true;
+    }
+    if (previous.sha256 === current.sha256) {
+        return false;
+    }
+    if (!previous.simhash) {
+        // No baseline fingerprint to compare against -> be conservative.
+        return true;
+    }
+    return hammingHex(previous.simhash, current.simhash) > threshold;
+}

package/lib/utils/descriptions/gcsStorage.d.ts

@@ -0,0 +1,16 @@
+import { Bucket } from "@google-cloud/storage";
+import { CourseDescriptionsStorage } from "./generateCourseDescriptions";
+/**
+ * Draft-bucket (GCS) adapter.
+ *
+ * Descriptions are never generated against this bucket — it keeps moving while
+ * the teacher edits, so it is not a stable snapshot. It is only written by the
+ * mirror-back, which copies descriptions generated from the published package
+ * whenever the draft content still matches.
+ *
+ * Note there is no `reprojectManifest`: the manifest is only rebuilt when
+ * publishing.
+ */
+export declare const gcsSyllabusPath: (courseSlug: string) => string;
+export declare const gcsReadmePath: (courseSlug: string, exerciseSlug: string, fileName: string) => string;
+export declare function createGcsDescriptionsStorage(bucket: Bucket): CourseDescriptionsStorage;

package/lib/utils/descriptions/gcsStorage.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.gcsReadmePath = exports.gcsSyllabusPath = void 0;
+exports.createGcsDescriptionsStorage = createGcsDescriptionsStorage;
+const configBuilder_1 = require("../configBuilder");
+const syllabusSync_1 = require("../syllabusSync");
+/**
+ * Draft-bucket (GCS) adapter.
+ *
+ * Descriptions are never generated against this bucket — it keeps moving while
+ * the teacher edits, so it is not a stable snapshot. It is only written by the
+ * mirror-back, which copies descriptions generated from the published package
+ * whenever the draft content still matches.
+ *
+ * Note there is no `reprojectManifest`: the manifest is only rebuilt when
+ * publishing.
+ */
+const gcsSyllabusPath = (courseSlug) => `courses/${courseSlug}/.learn/initialSyllabus.json`;
+exports.gcsSyllabusPath = gcsSyllabusPath;
+const gcsReadmePath = (courseSlug, exerciseSlug, fileName) => `courses/${courseSlug}/exercises/${exerciseSlug}/${fileName}`;
+exports.gcsReadmePath = gcsReadmePath;
+function createGcsDescriptionsStorage(bucket) {
+    return {
+        async readSyllabus(courseSlug) {
+            try {
+                const [buf] = await bucket.file((0, exports.gcsSyllabusPath)(courseSlug)).download();
+                return JSON.parse(buf.toString());
+            }
+            catch (_a) {
+                return null;
+            }
+        },
+        async writeSyllabus(courseSlug, syllabus) {
+            await bucket
+                .file((0, exports.gcsSyllabusPath)(courseSlug))
+                .save(Buffer.from(JSON.stringify(syllabus), "utf8"), {
+                contentType: "application/json",
+            });
+        },
+        async listExercises(courseSlug) {
+            const { exercises } = await (0, configBuilder_1.buildConfig)(bucket, courseSlug);
+            return exercises.map(exercise => ({
+                slug: exercise.slug,
+                translations: exercise.translations || {},
+            }));
+        },
+        async readReadme(courseSlug, exerciseSlug, fileName) {
+            try {
+                const [buf] = await bucket
+                    .file((0, exports.gcsReadmePath)(courseSlug, exerciseSlug, fileName))
+                    .download();
+                return buf.toString();
+            }
+            catch (_a) {
+                return null;
+            }
+        },
+        syllabusSyncStorage: (0, syllabusSync_1.createGcsSyllabusSyncStorage)(bucket),
+    };
+}