@learnpack/learnpack 5.0.355 → 5.0.357

package/src/services/ingest/core/resolveSlug.ts

@@ -0,0 +1,119 @@
+import { PackageRegistry } from "../ports/packageRegistry"
+import { IngestError } from "../errors"
+
+/**
+ * Room to spare under Rigobot's 100-character SlugField, so a suffixed
+ * suggestion still fits.
+ */
+export const MAX_SLUG_LENGTH = 80
+
+/** How many alternatives to offer when a slug is taken. */
+const SUGGESTION_COUNT = 3
+
+/**
+ * The strictest of the slug rules in play, on purpose.
+ *
+ * This repo's own `slugify` keeps dots, and two things break when one survives:
+ * Rigobot stores `package_slug` in a Django SlugField whose validator rejects
+ * them (it persists anyway, since the view never calls full_clean), and the
+ * published course is served at `{slug}.learn-pack.com`, where a dot adds a
+ * subdomain level the wildcard certificate does not cover.
+ * @param raw - Candidate slug or title.
+ * @returns Lowercase, hyphen-separated, alphanumeric-only slug.
+ */
+export function normalizeSlug(raw: string): string {
+  return raw
+    .normalize("NFD")
+    .replace(/[\u0300-\u036F]/g, "")
+    .toLowerCase()
+    .replace(/[^\da-z]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, MAX_SLUG_LENGTH)
+    .replace(/-+$/g, "")
+}
+
+/**
+ * Proposes alternatives for a slug that is taken.
+ *
+ * Suggested rather than applied: the slug is the public URL and, in practice,
+ * permanent — the rename endpoint does not update Rigobot's copy. Auto-suffixing
+ * is how a `solid-principles-7` ends up shipped. This mirrors the creator's own
+ * publish dialog, which checks availability and lets the human choose.
+ * @param base - The slug that was taken.
+ * @param registry - Used to check each candidate.
+ * @returns Up to three available slugs.
+ */
+export async function suggestSlugs(
+  base: string,
+  registry: PackageRegistry
+): Promise<string[]> {
+  const suggestions: string[] = []
+
+  for (let suffix = 2; suggestions.length < SUGGESTION_COUNT; suffix++) {
+    // Trim the base, not the suffix, so the candidate stays under the cap.
+    const tail = `-${suffix}`
+    const candidate = `${base.slice(0, MAX_SLUG_LENGTH - tail.length)}${tail}`
+
+    // eslint-disable-next-line no-await-in-loop
+    if (await registry.isSlugAvailable(candidate)) {
+      suggestions.push(candidate)
+    }
+
+    if (suffix > SUGGESTION_COUNT + 10) {
+      break
+    }
+  }
+
+  return suggestions
+}
+
+export type ResolveSlugInput = {
+  /** Slug the caller asked for, if any. */
+  requestedSlug?: string;
+  /** Course title, used to derive a slug when none was requested. */
+  courseTitle?: string;
+};
+
+/**
+ * Settles which slug the course will live under.
+ *
+ * An explicit slug is validated rather than repaired: silently reshaping what
+ * somebody typed hands them a different URL than the one they asked for. A
+ * derived slug is only a suggestion, which is why the caller is expected to
+ * show it.
+ *
+ * There is no shared default. An earlier version of the exporter fell back to
+ * "LearnPack Course" for an untitled course, which slugifies to
+ * `learnpack-course` for *every* such course — and since slugs are globally
+ * unique, the second one collides with the first.
+ * @param input - Requested slug and/or course title.
+ * @returns The slug to use.
+ * @throws IngestError `INVALID_SLUG` when nothing usable was given.
+ */
+export function resolveSlug(input: ResolveSlugInput): string {
+  const requested = (input.requestedSlug || "").trim()
+
+  if (requested) {
+    const normalized = normalizeSlug(requested)
+
+    if (normalized !== requested) {
+      throw new IngestError(
+        "INVALID_SLUG",
+        `"${requested}" is not a usable slug. Did you mean "${normalized}"? Slugs are lowercase letters, digits and hyphens, up to ${MAX_SLUG_LENGTH} characters.`
+      )
+    }
+
+    return normalized
+  }
+
+  const derived = normalizeSlug((input.courseTitle || "").trim())
+
+  if (!derived) {
+    throw new IngestError(
+      "INVALID_SLUG",
+      "The course has no title to derive a slug from; pass one explicitly"
+    )
+  }
+
+  return derived
+}

package/src/services/ingest/core/types.ts

@@ -0,0 +1,57 @@
+import { LessonType } from "../../../utils/packageManifest"
+
+/**
+ * A lesson entry as the generator plans it, before any content exists.
+ *
+ * Mirrors `files/lesson_plan.json`. Everything past `id`, `title` and `type` is
+ * optional because an earlier schema (ADR-0001 in learnpack-evals) had neither
+ * `lesson_focus` nor `concepts_covered`, and packages generated back then are
+ * still ingestable.
+ */
+export type PlannedLesson = {
+  id: string;
+  title: string;
+  type?: string;
+  module_number?: number;
+  duration_minutes?: number;
+  concepts_covered?: string[];
+  lesson_focus?: string | null;
+  key_insight?: string | null;
+};
+
+/** The generator's `files/lesson_plan.json`. */
+export type LessonPlan = {
+  course_title?: string;
+  course_duration_minutes?: number;
+  language?: string;
+  lessons: PlannedLesson[];
+};
+
+/**
+ * A planned lesson matched to the exercise folder that actually holds it.
+ *
+ * `exerciseSlug` comes from the package, never from re-slugifying the title:
+ * the generator's Python slugify and this repo's TypeScript one disagree (one
+ * collapses every non-alphanumeric run to a hyphen, the other keeps dots), and
+ * real titles contain typographic apostrophes that make the difference
+ * visible. A recomputed slug would key the sidebar to a folder that does not
+ * exist, and the lesson would render empty.
+ */
+export type IngestLesson = {
+  id: string;
+  exerciseSlug: string;
+  title: string;
+  type: LessonType;
+  durationMinutes: number;
+  /** The generation topic, from `lesson_focus`, falling back to the title. */
+  description: string;
+  /** Language codes with a README present for this lesson. */
+  languages: string[];
+};
+
+/** A plan entry that had no content in the package. */
+export type SkippedLesson = {
+  id: string;
+  title: string;
+  reason: "no-content";
+};

package/src/services/ingest/core/validatePackage.ts

@@ -0,0 +1,132 @@
+import { IngestError } from "../errors"
+
+/** Path of the lesson plan inside the package, relative to its root. */
+export const LESSON_PLAN_PATH = ".learn/artifacts/files/lesson_plan.json"
+
+/** Path of the course manifest inside the package. */
+export const LEARN_JSON_PATH = "learn.json"
+
+export type PackageFiles = Record<string, string>;
+
+const normalize = (path: string): string => path.replace(/\\/g, "/")
+
+/**
+ * Rejects a package the bucket consumers could not read correctly.
+ *
+ * Two of these rules look arbitrary and are not — both come from how
+ * `buildConfig` (`utils/configBuilder.ts`) derives a course from whatever files
+ * it finds:
+ *
+ * - It picks the manifest with `files.find(f => f.name.endsWith("learn.json"))`,
+ *   and GCS lists lexicographically, where `.learn/…` sorts *before*
+ *   `learn.json`. A second file ending in that name would shadow the real one
+ *   and the course would be configured from the wrong manifest.
+ * - It reads the exercise slug as the segment after `exercises`, so a nested
+ *   folder by that name would mint phantom lessons.
+ *
+ * Neither is hypothetical enough to skip: they cost one pass over the entries
+ * and fail with a message that names the offending path.
+ * @param files - Package entries, keyed by path relative to the package root.
+ * @returns Nothing; a valid package simply returns.
+ * @throws IngestError `INVALID_PACKAGE` describing the first problem found.
+ */
+export function validatePackage(files: PackageFiles): void {
+  const paths = Object.keys(files).map(path => normalize(path))
+
+  if (!paths.includes(LEARN_JSON_PATH)) {
+    throw new IngestError(
+      "INVALID_PACKAGE",
+      `The package has no ${LEARN_JSON_PATH} at its root`
+    )
+  }
+
+  if (!paths.includes(LESSON_PLAN_PATH)) {
+    throw new IngestError(
+      "INVALID_PACKAGE",
+      `The package has no ${LESSON_PLAN_PATH}; the syllabus cannot be built without it`
+    )
+  }
+
+  const shadowingManifest = paths.find(
+    path => path !== LEARN_JSON_PATH && path.endsWith("learn.json")
+  )
+  if (shadowingManifest) {
+    throw new IngestError(
+      "INVALID_PACKAGE",
+      `"${shadowingManifest}" would shadow the course manifest: no other file may end in learn.json`
+    )
+  }
+
+  const nestedExercises = paths.find(path => {
+    const segments = path.split("/")
+    // lastIndexOf, not indexOf: the root folder legitimately sits at 0, so
+    // asking where the *first* one is would always answer 0 and never flag the
+    // nested one this rule exists to catch.
+    return segments.lastIndexOf("exercises") > 0
+  })
+  if (nestedExercises) {
+    throw new IngestError(
+      "INVALID_PACKAGE",
+      `"${nestedExercises}" nests a folder named "exercises"; only the package root may have one`
+    )
+  }
+
+  const hasLesson = paths.some(path => path.startsWith("exercises/"))
+  if (!hasLesson) {
+    throw new IngestError(
+      "INVALID_PACKAGE",
+      "The package has no exercises/ folder, so it holds no lessons"
+    )
+  }
+}
+
+const README_PATTERN = /^readme(?:\.([a-z]{2}))?\.md$/i
+
+/**
+ * Reads which exercise folders exist and which languages each one has.
+ *
+ * A folder counts as a lesson only once it holds a README: an exercise with
+ * supporting files but no prose is not something the IDE can render.
+ * @param files - Package entries.
+ * @returns Folder slugs in listing order, and their language codes.
+ */
+export function readExercises(files: PackageFiles): {
+  slugs: string[];
+  languagesBySlug: Record<string, string[]>;
+} {
+  const languagesBySlug: Record<string, string[]> = {}
+
+  for (const rawPath of Object.keys(files)) {
+    const path = normalize(rawPath)
+    if (!path.startsWith("exercises/")) {
+      continue
+    }
+
+    const [, slug, fileName, ...rest] = path.split("/")
+    if (!slug || !fileName || rest.length > 0) {
+      continue
+    }
+
+    const match = fileName.match(README_PATTERN)
+    if (!match) {
+      continue
+    }
+
+    // `README.md` with no language segment is English, matching
+    // `getReadmeExtension` in utils/creatorUtilities.
+    const lang = (match[1] || "en").toLowerCase()
+    if (!languagesBySlug[slug]) {
+      languagesBySlug[slug] = []
+    }
+
+    if (!languagesBySlug[slug].includes(lang)) {
+      languagesBySlug[slug].push(lang)
+    }
+  }
+
+  for (const langs of Object.values(languagesBySlug)) {
+    langs.sort()
+  }
+
+  return { slugs: Object.keys(languagesBySlug), languagesBySlug }
+}

package/src/services/ingest/createIngestService.ts

@@ -0,0 +1,66 @@
+import { Storage } from "@google-cloud/storage"
+import { requireGcpCredentials } from "../../utils/gcpCredentials"
+import { requireGcsBucketName } from "../../utils/gcsBucketName"
+import { CourseStorage } from "./ports/courseStorage"
+import { PackageRegistry } from "./ports/packageRegistry"
+import { RequestAuthenticator } from "./ports/requestAuthenticator"
+import { createGcsCourseStorage } from "./adapters/gcsCourseStorage"
+import { createRigobotPackageRegistry } from "./adapters/rigobotPackageRegistry"
+import { createRigoUserTokenAuth } from "./adapters/rigoUserTokenAuth"
+
+export type IngestService = {
+  storage: CourseStorage;
+  auth: RequestAuthenticator;
+  /**
+   * Organization every ingested package must end up under.
+   *
+   * Rigobot attaches one automatically when the owner belongs to exactly one,
+   * so the ingest verifies the result rather than assuming it: landing in the
+   * wrong organization cannot be undone through the API.
+   */
+  organization: string;
+  /**
+   * Registry scoped to the caller's token.
+   *
+   * Built per request because the package's owner is whoever the token belongs
+   * to — a registry fixed at boot would silently file every ingested course
+   * under the server's identity.
+   */
+  registryFor(rigoToken: string): PackageRegistry;
+};
+
+export function requireIngestOrganization(): string {
+  const organization = (process.env.INGEST_ORGANIZATION || "").trim()
+
+  if (!organization) {
+    throw new Error(
+      "INGEST_ORGANIZATION (env) is required: it names the Rigobot organization ingested packages belong to"
+    )
+  }
+
+  return organization
+}
+
+/**
+ * Composition root: the only place in the ingest service that reads env vars.
+ *
+ * Everything below this line receives its dependencies already built, which is
+ * what keeps the ports testable without credentials and lets the whole service
+ * move behind a different transport later.
+ * @returns The assembled service.
+ * @throws Error naming the missing variable when configuration is incomplete.
+ */
+export function createIngestServiceFromEnv(): IngestService {
+  const credentials = requireGcpCredentials()
+  const bucketName = requireGcsBucketName()
+  const organization = requireIngestOrganization()
+
+  const bucket = new Storage({ credentials }).bucket(bucketName)
+
+  return {
+    storage: createGcsCourseStorage(bucket),
+    auth: createRigoUserTokenAuth(),
+    organization,
+    registryFor: createRigobotPackageRegistry,
+  }
+}

package/src/services/ingest/errors.ts

@@ -0,0 +1,45 @@
+/**
+ * Typed failures for the ingest service.
+ *
+ * The HTTP layer has to answer a slug collision with 409, a package owned by
+ * someone else with 403, and an unreachable Rigobot with 502 — three outcomes
+ * that arrive as the same `AxiosError`. Discriminating them by message
+ * substring would break the first time Rigobot rewords an error, so adapters
+ * name the failure where they still have the response in hand and everything
+ * downstream switches on `code`.
+ */
+
+export type IngestErrorCode =
+  | "INVALID_PACKAGE"
+  | "INVALID_SLUG"
+  | "SLUG_TAKEN"
+  | "SLUG_OWNED_BY_OTHER"
+  | "ORGANIZATION_MISMATCH"
+  | "INVALID_TOKEN"
+  | "REGISTRY_UNAVAILABLE"
+  | "STORAGE_FAILURE";
+
+export class IngestError extends Error {
+  readonly code: IngestErrorCode
+
+  readonly cause?: unknown
+
+  constructor(code: IngestErrorCode, message: string, cause?: unknown) {
+    super(message)
+    this.name = "IngestError"
+    this.code = code
+    this.cause = cause
+    // Restores the prototype chain, which extending a built-in breaks when the
+    // TypeScript target is ES5 — without it `instanceof IngestError` is false.
+    Object.setPrototypeOf(this, IngestError.prototype)
+  }
+}
+
+/**
+ * Narrows an unknown catch binding to an IngestError.
+ * @param error - Value caught from a throwing call.
+ * @returns True when the value is an IngestError.
+ */
+export function isIngestError(error: unknown): error is IngestError {
+  return error instanceof IngestError
+}

package/src/services/ingest/http/errorMapping.ts

@@ -0,0 +1,49 @@
+import {
+  ConflictError,
+  ForbiddenError,
+  InternalServerError,
+  OperationalError,
+  UnauthorizedError,
+  ValidationError,
+} from "../../../utils/errorHandler"
+import { IngestError, IngestErrorCode } from "../errors"
+
+type ErrorFactory = (message: string) => OperationalError;
+
+/**
+ * Which HTTP failure each ingest failure is.
+ *
+ * The line between 403 and 409 is the one worth reading twice: a **409** means
+ * the slug is yours and the caller can act — retry with overwrite, or pick
+ * another. A **403** means it is not yours, and no retry will help. Collapsing
+ * the two would leave the CLI unable to tell "try again differently" from
+ * "stop".
+ *
+ * `REGISTRY_UNAVAILABLE` would be a 502 in a vacuum. `errorHandler` has no
+ * class for that, and adding one to shade a single case is not worth touching a
+ * module the rest of the server depends on; the message already says Rigobot
+ * was unreachable.
+ *
+ * Declared as a total Record, not a switch with a default: adding a code to the
+ * union without deciding its status then fails the build instead of quietly
+ * becoming a 500.
+ */
+const BY_CODE: Record<IngestErrorCode, ErrorFactory> = {
+  INVALID_PACKAGE: message => new ValidationError(message),
+  INVALID_SLUG: message => new ValidationError(message),
+  INVALID_TOKEN: message => new UnauthorizedError(message),
+  SLUG_OWNED_BY_OTHER: message => new ForbiddenError(message),
+  ORGANIZATION_MISMATCH: message => new ForbiddenError(message),
+  SLUG_TAKEN: message => new ConflictError(message),
+  REGISTRY_UNAVAILABLE: message => new InternalServerError(message),
+  STORAGE_FAILURE: message => new InternalServerError(message),
+}
+
+/**
+ * Translates a domain failure into the HTTP error the global handler renders.
+ * @param error - The failure raised by the ingest service.
+ * @returns An operational error carrying the right status code.
+ */
+export function toHttpError(error: IngestError): OperationalError {
+  return BY_CODE[error.code](error.message)
+}

package/src/services/ingest/http/router.ts

@@ -0,0 +1,156 @@
+import * as express from "express"
+import { Request, Response, Router } from "express"
+import { asyncHandler, ValidationError } from "../../../utils/errorHandler"
+import { createUploadMiddleware } from "../../../utils/misc"
+import {
+  createIngestServiceFromEnv,
+  IngestService,
+} from "../createIngestService"
+import { ingestCoursePackage } from "../core/ingestCoursePackage"
+import { isIngestError } from "../errors"
+import { toHttpError } from "./errorMapping"
+import { readPackageZip } from "./zipPackageReader"
+
+const FILE_FIELD = "file"
+
+let cached: IngestService | undefined
+
+/**
+ * Builds the ingest service once, on the first request that needs it.
+ *
+ * Deliberately not built at boot. `INGEST_ORGANIZATION` is a new variable that
+ * no existing deployment sets, and the composition root throws without it — so
+ * constructing it during startup would stop every current server from booting
+ * over a capability it does not use. The repo never lets a new capability do
+ * that: a missing `RIGOBOT_SYSTEM_TOKEN` skips the descriptions stage, a
+ * missing `REDIS_URL` disables history, and unset GitHub credentials fail only
+ * the GitHub endpoints.
+ *
+ * The failure therefore belongs to the request, not the process, and arrives as
+ * a 400 naming the variable — the same shape the GitHub endpoints use.
+ * @returns The assembled service.
+ * @throws ValidationError when configuration is incomplete.
+ */
+function getService(): IngestService {
+  if (cached) {
+    return cached
+  }
+
+  try {
+    cached = createIngestServiceFromEnv()
+    return cached
+  } catch (error) {
+    throw new ValidationError(
+      `Course ingest is not configured. ${(error as Error).message}`
+    )
+  }
+}
+
+/**
+ * Reads a form field as a boolean.
+ *
+ * Multer parses the text fields alongside the file into `req.body`, where they
+ * arrive as strings.
+ * @param value - Raw field value.
+ * @returns Whether the flag was set.
+ */
+const readFlag = (value: unknown): boolean =>
+  value === true || value === "true" || value === "1"
+
+/**
+ * Converts an ingest failure into its HTTP equivalent.
+ *
+ * Registered on the router rather than inside the handler so every route added
+ * later inherits the mapping instead of re-implementing it.
+ * @param error - Whatever the handler rejected with.
+ * @param _req - Unused.
+ * @param _res - Unused.
+ * @param next - Passes the translated error to the global handler.
+ * @returns Nothing.
+ */
+export function ingestErrorTranslator(
+  error: unknown,
+  _req: Request,
+  _res: Response,
+  next: express.NextFunction
+): void {
+  next(isIngestError(error) ? toHttpError(error) : error)
+}
+
+/**
+ * The ingest endpoint.
+ *
+ * Accepts `multipart/form-data` rather than a raw zip body: multer with memory
+ * storage is already wired in this repo, no raw-body middleware is, and the
+ * client side of every existing zip upload already builds a FormData with a
+ * `file` field. Matching that means the producer has nothing new to learn.
+ * @returns A router to mount under a path prefix.
+ */
+export function createIngestRouter(): Router {
+  const router = express.Router()
+  const upload = createUploadMiddleware()
+
+  router.post(
+    "/packages",
+    upload.single(FILE_FIELD),
+    asyncHandler(async (req: Request, res: Response) => {
+      const service = getService()
+
+      // eslint-disable-next-line
+      // @ts-ignore multer augments the request; the Express types do not know
+      const uploaded = req.file
+
+      if (!uploaded) {
+        throw new ValidationError(
+          `A "${FILE_FIELD}" field holding the course zip is required`
+        )
+      }
+
+      // Shape first, credentials second. Authenticating costs a round trip to
+      // Rigobot, and spending it on a request that was never well-formed only
+      // buys a less accurate answer — a malformed call would come back 401
+      // instead of being told what it actually got wrong. Multer has already
+      // buffered the upload by this point either way, so nothing is saved by
+      // rejecting earlier.
+      const identity = await service.auth.authenticate(req.headers)
+
+      const body = (req.body || {}) as Record<string, unknown>
+      const files = await readPackageZip(uploaded.buffer)
+
+      const report = await ingestCoursePackage(
+        {
+          files,
+          requestedSlug: typeof body.slug === "string" ? body.slug : undefined,
+          overwrite: readFlag(body.overwrite),
+        },
+        {
+          storage: service.storage,
+          registry: service.registryFor(identity.rigoToken),
+          organization: service.organization,
+        }
+      )
+
+      // 200 even when lessons were left out. A partial course is a course, and
+      // publish already answers this way: the caller gets the result plus what
+      // degraded, and decides how loudly to say it.
+      res.json({ message: "Course ingested", ...report })
+    })
+  )
+
+  // Must come after the routes: Express only reaches an error middleware for
+  // failures raised by handlers registered before it.
+  router.use(ingestErrorTranslator)
+
+  return router
+}
+
+/**
+ * Drops the memoized service.
+ *
+ * Exposed for tests, which change the environment between cases and need the
+ * next request to rebuild rather than reuse.
+ * @returns Nothing.
+ */
+export function resetIngestServiceCache(): void {
+  cached = undefined
+}