@learnpack/learnpack 5.0.352 → 5.0.354

package/lib/scripts/descriptionsS3Backfill.js

@@ -0,0 +1,277 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-await-in-loop */
+/**
+ * Fase 0a — S3 descriptions backfill (step 2 of the S3 pipeline).
+ *
+ * One-off fill of the published catalogue (~400+ courses). The per-course work
+ * is the shared description service (`generateCourseDescriptions`), the same one
+ * the post-publish stage and the sweep use; this script only decides WHICH
+ * courses to run it on and in what batches.
+ *
+ *   - Processes at most --limit courses per run (default 50), ordered by the
+ *     initialSyllabus.json modification time (most recently edited first),
+ *     skipping courses already settled at the current prompt version. Re-run to
+ *     advance through the catalogue.
+ *   - Or target a single course with --slug (remediation / on-demand).
+ *
+ * Writes descriptions into the published initialSyllabus.json. By default the
+ * manifest projection is a separate pass: run the package-manifest backfill
+ * (runBatch) afterwards, which over the whole catalogue is cheaper than
+ * re-projecting course by course.
+ *
+ * That default inverts for small, targeted runs, where three flags reproduce per
+ * course what a publication does, in one process instead of three:
+ *
+ *   --reproject-manifest  projects each course as it is described (the same
+ *                         `processPackage` the separate pass runs, plus a
+ *                         CloudFront invalidation);
+ *   --mirror-draft        copies the new descriptions back into the GCS draft,
+ *                         so the next publication does not regenerate them;
+ *   --emit-events         announces the result to breathecode.
+ *
+ * Secrets/infra via env: RIGOBOT_SYSTEM_TOKEN (required), BREATHECODE_SYSTEM_TOKEN
+ * (required by --emit-events), GCP_CREDENTIALS_JSON and GCP_BUCKET_NAME (required
+ * by --mirror-draft), plus S3_PACKAGES_BUCKET, AWS_REGION and
+ * CLOUDFRONT_DISTRIBUTION_ID as fallbacks/optionals.
+ *
+ * Per-run flags:
+ *   --s3-bucket <name>    S3 bucket (or env S3_PACKAGES_BUCKET, default learnpack-paquetes)
+ *   --gcs-bucket <name>   GCS draft bucket for --mirror-draft (or env GCP_BUCKET_NAME)
+ *   --region <region>     AWS region (default us-east-1, or env AWS_REGION)
+ *   --slug <slug>         process a single course (remediation / on-demand)
+ *   --limit <n>           courses per run (default 50)
+ *   --target-words <n>    words per description (default 25)
+ *   --concurrency <n>     completions in flight per course (default 5)
+ *   --dry-run             preview without writing/generating
+ *   --no-reconcile        skip the additive syllabus reconciliation
+ *   --force               regenerate even courses already settled at this prompt version
+ *   --reproject-manifest  re-project package-manifest.json per course, and invalidate it
+ *   --mirror-draft        mirror the descriptions back into the GCS draft
+ *   --emit-events         send package_manifest_updated per course (implies --reproject-manifest)
+ *
+ * Examples:
+ *   node lib/scripts/descriptionsS3Backfill.js --limit 50
+ *   node lib/scripts/descriptionsS3Backfill.js --slug my-course
+ *   node lib/scripts/descriptionsS3Backfill.js --dry-run --limit 10
+ *   node lib/scripts/descriptionsS3Backfill.js --slug my-course \
+ *     --reproject-manifest --mirror-draft --emit-events
+ */
+const node_util_1 = require("node:util");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
+const storage_1 = require("@google-cloud/storage");
+const packageManifest_1 = require("../utils/packageManifest");
+const generateCourseDescriptions_1 = require("../utils/descriptions/generateCourseDescriptions");
+const s3Storage_1 = require("../utils/descriptions/s3Storage");
+const backfillEvents_1 = require("../utils/descriptions/backfillEvents");
+const gcsStorage_1 = require("../utils/descriptions/gcsStorage");
+const mirrorDescriptions_1 = require("../utils/descriptions/mirrorDescriptions");
+const gcsBucketName_1 = require("../utils/gcsBucketName");
+const workList_1 = require("../utils/descriptions/workList");
+const SYLLABUS_KEY_PATTERN = /^([^/]+)\/\.learn\/initialSyllabus\.json$/;
+/** Courses (by slug) ordered by initialSyllabus.json LastModified, newest first. */
+async function listCoursesByRecency(s3, bucket) {
+    var _a, _b;
+    const mtimes = new Map();
+    let continuationToken;
+    do {
+        const response = (await s3.send(new client_s3_1.ListObjectsV2Command({
+            Bucket: bucket,
+            ContinuationToken: continuationToken,
+        })));
+        for (const object of response.Contents || []) {
+            const match = (_a = object.Key) === null || _a === void 0 ? void 0 : _a.match(SYLLABUS_KEY_PATTERN);
+            if (match) {
+                mtimes.set(match[1], ((_b = object.LastModified) === null || _b === void 0 ? void 0 : _b.getTime()) || 0);
+            }
+        }
+        continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+    return [...mtimes.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .map(([slug]) => slug);
+}
+/**
+ * CDN invalidation for the re-projected manifests. Optional: a stale edge cache
+ * expires on its own, so a missing distribution id degrades the run instead of
+ * stopping it.
+ */
+function cloudFrontFor(region) {
+    const distributionId = (process.env.CLOUDFRONT_DISTRIBUTION_ID || "").trim();
+    if (!distributionId) {
+        console.warn("[s3-backfill] CLOUDFRONT_DISTRIBUTION_ID is not set: manifests will be re-projected but not invalidated");
+        return undefined;
+    }
+    return {
+        client: new client_cloudfront_1.CloudFrontClient({ region }),
+        distributionId,
+    };
+}
+async function main() {
+    var _a, _b, _c, _d, _e, _f;
+    const { values } = (0, node_util_1.parseArgs)({
+        options: {
+            "s3-bucket": { type: "string" },
+            "gcs-bucket": { type: "string" },
+            region: { type: "string" },
+            slug: { type: "string" },
+            limit: { type: "string" },
+            "target-words": { type: "string" },
+            concurrency: { type: "string" },
+            "dry-run": { type: "boolean", default: false },
+            "no-reconcile": { type: "boolean", default: false },
+            force: { type: "boolean", default: false },
+            "reproject-manifest": { type: "boolean", default: false },
+            "mirror-draft": { type: "boolean", default: false },
+            "emit-events": { type: "boolean", default: false },
+        },
+    });
+    const token = process.env.RIGOBOT_SYSTEM_TOKEN;
+    if (!token) {
+        console.error("[s3-backfill] RIGOBOT_SYSTEM_TOKEN (env) is required");
+        process.exit(1);
+    }
+    const bucket = values["s3-bucket"] ||
+        process.env.S3_PACKAGES_BUCKET ||
+        "learnpack-paquetes";
+    const region = values.region || process.env.AWS_REGION || "us-east-1";
+    const limit = Number.parseInt(values.limit || "50", 10) || 50;
+    const dryRun = (_a = values["dry-run"]) !== null && _a !== void 0 ? _a : false;
+    const force = (_b = values.force) !== null && _b !== void 0 ? _b : false;
+    const emitEvents = (_c = values["emit-events"]) !== null && _c !== void 0 ? _c : false;
+    const reprojectManifest = (_d = values["reproject-manifest"]) !== null && _d !== void 0 ? _d : false;
+    const mirrorDraft = (_e = values["mirror-draft"]) !== null && _e !== void 0 ? _e : false;
+    // The event announces that the manifest changed, so emitting it without
+    // projecting the manifest would state something untrue. Required rather than
+    // implied: turning on a write to S3 and the CDN as a side effect of another
+    // flag is exactly the kind of surprise a backfill should not have.
+    if (emitEvents && !reprojectManifest) {
+        console.error("[s3-backfill] --emit-events requires --reproject-manifest: the event " +
+            "reports a manifest update that would not have happened");
+        process.exit(1);
+    }
+    const breathecodeToken = process.env.BREATHECODE_SYSTEM_TOKEN;
+    if (emitEvents && !breathecodeToken) {
+        console.error("[s3-backfill] BREATHECODE_SYSTEM_TOKEN (env) is required by --emit-events");
+        process.exit(1);
+    }
+    if (emitEvents && dryRun) {
+        console.error("[s3-backfill] --emit-events cannot be combined with --dry-run: there is " +
+            "no dry run of an event that breathecode already received");
+        process.exit(1);
+    }
+    // Built up front, not on first use: a misconfigured draft bucket must fail at
+    // boot, not after the first course has already been billed to Rigobot.
+    let draftStorage;
+    if (mirrorDraft) {
+        const credentialsEnv = process.env.GCP_CREDENTIALS_JSON;
+        if (!credentialsEnv) {
+            console.error("[s3-backfill] GCP_CREDENTIALS_JSON (env) is required by --mirror-draft");
+            process.exit(1);
+        }
+        draftStorage = (0, gcsStorage_1.createGcsDescriptionsStorage)(new storage_1.Storage({ credentials: JSON.parse(credentialsEnv) }).bucket(values["gcs-bucket"] || (0, gcsBucketName_1.requireGcsBucketName)()));
+    }
+    const s3 = new client_s3_1.S3Client({ region });
+    const storage = (0, s3Storage_1.createS3DescriptionsStorage)(s3, bucket, 
+    // Off by default: over the whole catalogue manifests are cheaper to project
+    // in a single later pass than course by course.
+    reprojectManifest ?
+        { cloudFront: cloudFrontFor(region) } :
+        { reprojectManifest: false });
+    const runId = (0, backfillEvents_1.newBackfillRunId)();
+    const singleSlug = values.slug;
+    const slugs = singleSlug ?
+        [singleSlug] :
+        await listCoursesByRecency(s3, bucket);
+    console.log(`[s3-backfill] Starting${singleSlug ? ` (slug=${singleSlug})` : ` (limit=${limit})`}${dryRun ? " (dry-run)" : ""} over ${slugs.length} candidate course(s)${reprojectManifest ? ", re-projecting manifests" : ""}${mirrorDraft ? ", mirroring into the draft" : ""}${emitEvents ? `, run ${runId}` : ""}`);
+    let processed = 0;
+    let generated = 0;
+    let failed = 0;
+    let mirrored = 0;
+    const events = {
+        skipped: 0,
+        delivered: 0,
+        failed: 0,
+    };
+    for (const slug of slugs) {
+        if (!singleSlug && processed >= limit) {
+            break;
+        }
+        try {
+            // Cheap pre-filter: skip finished courses without reading their READMEs.
+            if (!force) {
+                const syllabus = await storage.readSyllabus(slug);
+                if (!syllabus) {
+                    continue;
+                }
+                const exercises = await (0, s3Storage_1.fetchExercises)(s3, bucket, slug);
+                if (exercises.length === 0 || (0, workList_1.isCourseSettled)(syllabus, exercises)) {
+                    continue;
+                }
+            }
+            const result = await (0, generateCourseDescriptions_1.generateCourseDescriptions)(storage, slug, {
+                token,
+                dryRun,
+                force,
+                reconcile: !((_f = values["no-reconcile"]) !== null && _f !== void 0 ? _f : false),
+                concurrency: Number.parseInt(values.concurrency || "", 10) ||
+                    generateCourseDescriptions_1.DEFAULT_DESCRIPTIONS_CONCURRENCY,
+                targetWordCount: Number.parseInt(values["target-words"] || "", 10) ||
+                    packageManifest_1.DESCRIPTION_TARGET_WORD_COUNT,
+            });
+            if (result.status === "skipped") {
+                continue;
+            }
+            processed += 1;
+            generated += result.generated;
+            failed += result.failed;
+            console.log(`[s3-backfill] "${slug}": ${result.steps} step(s), ${result.generated} description(s)${dryRun ? " (dry-run)" : ""}${result.failed > 0 ? `, ${result.failed} failed` : ""}${result.missing > 0 ? `, ${result.missing} unanswered` : ""}`);
+            if (draftStorage) {
+                // Before the event, so a course is fully settled by the time it is
+                // announced. A failure here does NOT change the event status: the
+                // published package and its manifest are already correct, and only the
+                // draft lags behind — the next publication regenerates what is missing.
+                try {
+                    const published = await storage.readSyllabus(slug);
+                    const mirror = await (0, mirrorDescriptions_1.mirrorDescriptionsToDraft)(draftStorage, slug, published, { dryRun });
+                    mirrored += mirror.copied;
+                    console.log(`[s3-backfill] "${slug}": ${mirror.copied} mirrored, ${mirror.missed} missed, ${mirror.fresh} already fresh`);
+                }
+                catch (error) {
+                    console.error(`[s3-backfill] Could not mirror "${slug}" into the draft:`, error.message);
+                }
+            }
+            if (emitEvents) {
+                const outcome = await (0, backfillEvents_1.emitBackfillManifestEvent)({
+                    courseSlug: slug,
+                    runId,
+                    result,
+                    storage,
+                    rigobotToken: token,
+                    breathecodeToken: breathecodeToken,
+                });
+                events[outcome] += 1;
+                console.log(`[s3-backfill] "${slug}": manifest event ${outcome}`);
+            }
+        }
+        catch (error) {
+            processed += 1;
+            failed += 1;
+            // No event here on purpose: unlike a publication, a backfill never
+            // announced one, so nothing is owed. The course is simply re-run.
+            console.error(`[s3-backfill] Failed processing "${slug}":`, error.message);
+        }
+    }
+    console.log(`[s3-backfill] Done. ${processed} course(s), ${generated} description(s) written, ${failed} failure(s).` +
+        (mirrorDraft ? ` ${mirrored} mirrored into the draft.` : "") +
+        (emitEvents ?
+            ` Events: ${events.delivered} delivered, ${events.failed} failed (run ${runId}).` :
+            ""));
+}
+main()
+    .then(() => process.exit(0))
+    .catch(error => {
+    console.error("[s3-backfill] Fatal error:", error);
+    process.exit(1);
+});

package/lib/scripts/descriptionsSweep.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/descriptionsSweep.js

@@ -0,0 +1,142 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-await-in-loop */
+/**
+ * Descriptions sweep — safety net for the post-publish stage.
+ *
+ * The stage that generates descriptions runs inside the web process, so a dyno
+ * restart mid-flight leaves a package published but not described and, worse,
+ * breathecode waiting for a manifest event that was promised and never arrived.
+ * This job finishes those publications.
+ *
+ * Runs as a one-off dyno via Heroku Scheduler (hourly is plenty):
+ *   node lib/scripts/descriptionsSweep.js
+ *
+ * Two sources of work:
+ *   1. Publish journals (`publish-journal/` in GCS) — the primary one. Each
+ *      journal that stalled is resumed: missing descriptions are generated, the
+ *      draft is mirrored, and the owed event is emitted. Journals are dropped
+ *      once complete and abandoned after a few attempts.
+ *   2. A deep scan of published courses (DESCRIPTIONS_SWEEP_FULL=1) — catches
+ *      packages that never got a journal at all, at the cost of reading every
+ *      course. Off by default.
+ *
+ * Required env: GCP_CREDENTIALS_JSON, RIGOBOT_SYSTEM_TOKEN.
+ * Optional env: GCP_BUCKET_NAME, S3_PACKAGES_BUCKET, AWS_REGION,
+ * CLOUDFRONT_DISTRIBUTION_ID, BREATHECODE_SYSTEM_TOKEN (needed to emit the
+ * pending events), DESCRIPTIONS_SWEEP_FULL, DESCRIPTIONS_SWEEP_LIMIT,
+ * DESCRIPTION_STALE_AFTER_MS.
+ */
+const storage_1 = require("@google-cloud/storage");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const publishJournal_1 = require("../utils/publishJournal");
+const packageManifestBackfill_1 = require("../utils/s3/packageManifestBackfill");
+const gcsBucketName_1 = require("../utils/gcsBucketName");
+const gcsStorage_1 = require("../utils/descriptions/gcsStorage");
+const s3Storage_1 = require("../utils/descriptions/s3Storage");
+const generateCourseDescriptions_1 = require("../utils/descriptions/generateCourseDescriptions");
+const resumePublication_1 = require("../utils/descriptions/resumePublication");
+async function main() {
+    const credentialsEnv = process.env.GCP_CREDENTIALS_JSON;
+    if (!credentialsEnv) {
+        console.error("[sweep] GCP_CREDENTIALS_JSON (env) is required");
+        process.exit(1);
+    }
+    const rigobotToken = process.env.RIGOBOT_SYSTEM_TOKEN;
+    if (!rigobotToken) {
+        console.error("[sweep] RIGOBOT_SYSTEM_TOKEN (env) is required");
+        process.exit(1);
+    }
+    const breathecodeToken = process.env.BREATHECODE_SYSTEM_TOKEN;
+    if (!breathecodeToken) {
+        console.warn("[sweep] BREATHECODE_SYSTEM_TOKEN is not set: pending events cannot be emitted");
+    }
+    const bucket = new storage_1.Storage({
+        credentials: JSON.parse(credentialsEnv),
+    }).bucket((0, gcsBucketName_1.requireGcsBucketName)());
+    const journalStorage = (0, publishJournal_1.createGcsJournalStorage)(bucket);
+    const publishedStorage = (0, s3Storage_1.createS3DescriptionsStorageFromEnv)();
+    const draftStorage = (0, gcsStorage_1.createGcsDescriptionsStorage)(bucket);
+    const staleAfterMs = Number.parseInt(process.env.DESCRIPTION_STALE_AFTER_MS || "", 10) ||
+        resumePublication_1.DEFAULT_STALE_AFTER_MS;
+    const refs = await (0, publishJournal_1.listJournalRefs)(journalStorage);
+    console.log(`[sweep] ${refs.length} unfinished publication(s)`);
+    const tally = {
+        "skipped-recent": 0,
+        "skipped-abandoned": 0,
+        abandoned: 0,
+        resumed: 0,
+        "nothing-to-do": 0,
+    };
+    for (const ref of refs) {
+        try {
+            const journal = await (0, publishJournal_1.readJournalByKey)(journalStorage, ref.key);
+            if (!journal) {
+                continue;
+            }
+            const outcome = await (0, resumePublication_1.resumePublication)(journal, {
+                journalStorage,
+                publishedStorage,
+                draftStorage,
+                rigobotToken,
+                breathecodeToken,
+                staleAfterMs,
+            });
+            tally[outcome] += 1;
+            if (outcome === "resumed") {
+                console.log(`[sweep] Resumed "${ref.courseSlug}" (${ref.publishId})`);
+            }
+        }
+        catch (error) {
+            console.error(`[sweep] Failed resuming "${ref.courseSlug}":`, error.message);
+        }
+    }
+    console.log(`[sweep] Journals: ${tally.resumed} resumed, ${tally["skipped-recent"]} still in flight, ` +
+        `${tally["nothing-to-do"]} nothing to do, ${tally.abandoned} abandoned, ` +
+        `${tally["skipped-abandoned"]} already abandoned`);
+    if (process.env.DESCRIPTIONS_SWEEP_FULL === "1") {
+        await deepScan(rigobotToken, publishedStorage);
+    }
+}
+/**
+ * Fallback for packages with no journal at all (published before this existed,
+ * or whose journal write failed). Expensive: it reads every published course,
+ * so it is opt-in rather than the default path.
+ */
+async function deepScan(rigobotToken, publishedStorage) {
+    const s3 = new client_s3_1.S3Client({
+        region: process.env.AWS_REGION || "us-east-1",
+    });
+    const bucket = process.env.S3_PACKAGES_BUCKET || "learnpack-paquetes";
+    const limit = Number.parseInt(process.env.DESCRIPTIONS_SWEEP_LIMIT || "", 10) || 25;
+    const slugs = await (0, packageManifestBackfill_1.listPublishedSlugs)(s3, bucket);
+    console.log(`[sweep] Deep scan over ${slugs.length} published course(s), limit ${limit}`);
+    let processed = 0;
+    for (const slug of slugs) {
+        if (processed >= limit) {
+            break;
+        }
+        try {
+            const result = await (0, generateCourseDescriptions_1.generateCourseDescriptions)(publishedStorage, slug, {
+                token: rigobotToken,
+                reconcile: true,
+            });
+            if (result.status === "skipped") {
+                continue;
+            }
+            processed += 1;
+            console.log(`[sweep] "${slug}": ${result.generated} description(s), ${result.failed} failed`);
+        }
+        catch (error) {
+            processed += 1;
+            console.error(`[sweep] Failed on "${slug}":`, error.message);
+        }
+    }
+    console.log(`[sweep] Deep scan done. ${processed} course(s) touched.`);
+}
+main()
+    .then(() => process.exit(0))
+    .catch(error => {
+    console.error("[sweep] Fatal error:", error);
+    process.exit(1);
+});

package/lib/utils/api.d.ts

@@ -1,3 +1,10 @@
+/**
+ * Student telemetry, and publication events. Both share the endpoint: a body
+ * carrying a top-level `event` key is dispatched as an event, anything else
+ * keeps being processed as telemetry.
+ */
+export declare const BREATHECODE_TELEMETRY_URL = "https://breathecode.herokuapp.com/v1/assignment/me/telemetry";
+export declare const BREATHECODE_EVENTS_URL = "https://breathecode.herokuapp.com/v1/assignment/me/telemetry";
 export declare const RIGOBOT_HOST: string;
 export declare const RIGOBOT_REALTIME_HOST = "https://ai.4geeks.com";
 type TConsumableSlug = "ai-conversation-message" | "ai-compilation" | "ai-tutorial-generation" | "ai-generation" | "learnpack-publish";

package/lib/utils/api.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getCurrentTechnologies = exports.fetchTechnologies = exports.doesAssetExists = exports.createAsset = exports.validateToken = exports.listUserAcademies = exports.getConsumable = exports.countConsumables = exports.RIGOBOT_REALTIME_HOST = exports.RIGOBOT_HOST = void 0;
+exports.getCurrentTechnologies = exports.fetchTechnologies = exports.doesAssetExists = exports.createAsset = exports.validateToken = exports.listUserAcademies = exports.getConsumable = exports.countConsumables = exports.RIGOBOT_REALTIME_HOST = exports.RIGOBOT_HOST = exports.BREATHECODE_EVENTS_URL = exports.BREATHECODE_TELEMETRY_URL = void 0;
 exports.resolveLearnpackPackageId = resolveLearnpackPackageId;
 const console_1 = require("../utils/console");
 const storage = require("node-persist");
@@ -9,6 +9,13 @@ const axios_1 = require("axios");
 const dotenv = require("dotenv");
 dotenv.config();
 const HOST = "https://breathecode.herokuapp.com";
+/**
+ * Student telemetry, and publication events. Both share the endpoint: a body
+ * carrying a top-level `event` key is dispatched as an event, anything else
+ * keeps being processed as telemetry.
+ */
+exports.BREATHECODE_TELEMETRY_URL = `${HOST}/v1/assignment/me/telemetry`;
+exports.BREATHECODE_EVENTS_URL = exports.BREATHECODE_TELEMETRY_URL;
 exports.RIGOBOT_HOST = process.env.RIGOBOT_HOST || "https://rigobot.herokuapp.com";
 exports.RIGOBOT_REALTIME_HOST = "https://ai.4geeks.com";
 // export const RIGOBOT_REALTIME_HOST = "http://127.0.0.1:8003"

package/lib/utils/awsCredentials.d.ts

@@ -0,0 +1,20 @@
+/**
+ * AWS credentials for the published bucket, required rather than implicit.
+ *
+ * Building an `S3Client` with no credentials succeeds: the SDK defers to its
+ * provider chain and only fails at the first request, with
+ * "Could not load credentials from any providers". Inside a background job that
+ * surfaces minutes later, detached from the cause. Validating up front turns it
+ * into an obvious misconfiguration.
+ *
+ * Note this deliberately rejects the empty-string fallback pattern
+ * (`process.env.X || ""`): passing empty credentials defeats the provider chain
+ * instead of failing, which is how the opaque error appeared in the first place.
+ */
+export type AwsCredentials = {
+    accessKeyId: string;
+    secretAccessKey: string;
+};
+export declare function requireAwsCredentials(): AwsCredentials;
+export declare function requireS3PackagesBucket(): string;
+export declare function awsRegion(): string;

package/lib/utils/awsCredentials.js

@@ -0,0 +1,43 @@
+"use strict";
+/**
+ * AWS credentials for the published bucket, required rather than implicit.
+ *
+ * Building an `S3Client` with no credentials succeeds: the SDK defers to its
+ * provider chain and only fails at the first request, with
+ * "Could not load credentials from any providers". Inside a background job that
+ * surfaces minutes later, detached from the cause. Validating up front turns it
+ * into an obvious misconfiguration.
+ *
+ * Note this deliberately rejects the empty-string fallback pattern
+ * (`process.env.X || ""`): passing empty credentials defeats the provider chain
+ * instead of failing, which is how the opaque error appeared in the first place.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.requireAwsCredentials = requireAwsCredentials;
+exports.requireS3PackagesBucket = requireS3PackagesBucket;
+exports.awsRegion = awsRegion;
+function requireAwsCredentials() {
+    const accessKeyId = (process.env.AWS_ACCESS_KEY_ID || "").trim();
+    const secretAccessKey = (process.env.AWS_SECRET_ACCESS_KEY || "").trim();
+    const missing = [];
+    if (!accessKeyId) {
+        missing.push("AWS_ACCESS_KEY_ID");
+    }
+    if (!secretAccessKey) {
+        missing.push("AWS_SECRET_ACCESS_KEY");
+    }
+    if (missing.length > 0) {
+        throw new Error(`${missing.join(" and ")} (env) ${missing.length > 1 ? "are" : "is"} required to reach the published package bucket`);
+    }
+    return { accessKeyId, secretAccessKey };
+}
+function requireS3PackagesBucket() {
+    const bucket = (process.env.S3_PACKAGES_BUCKET || "").trim();
+    if (!bucket) {
+        throw new Error("S3_PACKAGES_BUCKET (env) is required: it names the bucket holding published packages");
+    }
+    return bucket;
+}
+function awsRegion() {
+    return (process.env.AWS_REGION || "").trim() || "us-east-1";
+}

package/lib/utils/creatorUtilities.js

@@ -17,6 +17,7 @@ exports.countSentences = countSentences;
 exports.howManyDifficultParagraphs = howManyDifficultParagraphs;
 exports.insertStepInCorrectPosition = insertStepInCorrectPosition;
 const console_1 = require("./console");
+const api_1 = require("./api");
 const frontMatter = require("front-matter");
 const MarkdownIt = require("markdown-it");
 const syllable_1 = require("syllable");
@@ -146,7 +147,7 @@ const makePackageInfo = (choices) => {
             .replace(/ /g, "-")
             .replace(/[^\w-]+/g, ""),
         telemetry: {
-            batch: "https://breathecode.herokuapp.com/v1/assignment/me/telemetry",
+            batch: api_1.BREATHECODE_TELEMETRY_URL,
         },
     };
     return packageInfo;

package/lib/utils/descriptionHash.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Utilities to detect whether a step description is stale relative to its
+ * README content.
+ *
+ * Two signals are stored per (step, lang) at generation time:
+ *  - sha256(normalizeReadme(content))  -> exact fast-path + content-addressing
+ *  - simhash64(normalizeReadme(content)) -> near-duplicate significance filter
+ *
+ * The sweep re-hashes the current README and compares against the stored
+ * values: exact SHA-256 match => no-op; otherwise if the SimHash Hamming
+ * distance to the *generation baseline* is <= k the change is cosmetic and is
+ * ignored, else the description is regenerated. The H1 title is part of the
+ * normalized content (a deliberate product decision).
+ */
+/**
+ * Remove every image from markdown: inline `![alt](url)`, reference-style
+ * `![alt][ref]` and raw `<img>` tags. Regular links `[text](url)` are kept.
+ *
+ * Images are dropped from BOTH sides of the description pipeline — here for
+ * fingerprinting, and again before sending the README to the LLM. Adding,
+ * replacing or removing an illustration does not change what the lesson
+ * teaches, so it must not trigger a regeneration; and the alt text of generated
+ * illustrations is a full image-generation prompt (hundreds of tokens) that
+ * only invites the model to describe the picture instead of the lesson.
+ *
+ * Note the alt text may legitimately contain parentheses, so the alt is matched
+ * as "anything up to the closing bracket" rather than as balanced text.
+ */
+export declare function stripMarkdownImages(content: string): string;
+/**
+ * Canonicalize a README for hashing. Strips the frontmatter (metadata such as
+ * tutorial/intro video URLs, which do not describe the lesson), removes images
+ * (see `stripMarkdownImages`) and collapses all whitespace runs so that
+ * reformatting, blank lines and trailing spaces do not register as content
+ * changes. The H1 and the prose are preserved.
+ */
+export declare function normalizeReadme(content: string): string;
+export declare function sha256(input: string): string;
+/**
+ * 64-bit SimHash (Charikar) over whitespace-delimited tokens, returned as a
+ * zero-padded 16-char hex string.
+ */
+export declare function simhash64(input: string): string;
+/** Hamming distance between two 64-bit hex fingerprints. */
+export declare function hammingHex(a: string, b: string): number;
+export type ContentFingerprint = {
+    sha256: string;
+    simhash: string;
+};
+/** Convenience: normalize + both fingerprints in one call. */
+export declare function fingerprintReadme(content: string): ContentFingerprint;
+/**
+ * Default significance threshold (Hamming distance over the 64-bit SimHash).
+ * k=3 is the classic near-duplicate threshold for 64-bit SimHash
+ * (Charikar 2002; Manku, Jain & Das Sarma, WWW 2007). Overridable via env.
+ */
+export declare function getSimhashThreshold(): number;
+/**
+ * Decide whether a description generated from `previous` is stale given the
+ * `current` README content. Exact match => not stale; else stale only if the
+ * SimHash Hamming distance exceeds the threshold (cosmetic edits are ignored).
+ */
+export declare function isDescriptionStale(previous: {
+    sha256?: string;
+    simhash?: string;
+} | undefined, current: ContentFingerprint, threshold?: number): boolean;