@learnpack/learnpack 5.0.355 → 5.0.357

package/src/services/ingest/core/ingestCoursePackage.ts

@@ -0,0 +1,343 @@
+import Console from "../../../utils/console"
+import { mapWithConcurrency } from "../../../utils/s3/packageManifestBackfill"
+import { CourseStorage } from "../ports/courseStorage"
+import { PackageRegistry, RigoPackage } from "../ports/packageRegistry"
+import { IngestError } from "../errors"
+import { buildIngestConfig } from "./buildIngestConfig"
+import { buildInitialSyllabus } from "./buildInitialSyllabus"
+import { buildSidebar } from "./buildSidebar"
+import { configPath, coursePrefix } from "./coursePaths"
+import { normalizeLearnJson } from "./normalizeLearnJson"
+import { pairLessons } from "./pairLessons"
+import { planDeletions, planWrites } from "./planWrites"
+import { resolveSlug, suggestSlugs } from "./resolveSlug"
+import {
+  LEARN_JSON_PATH,
+  LESSON_PLAN_PATH,
+  PackageFiles,
+  readExercises,
+  validatePackage,
+} from "./validatePackage"
+import { LessonPlan, SkippedLesson } from "./types"
+
+/** Enough parallelism to hide the round trip without opening a socket per file. */
+const WRITE_CONCURRENCY = 10
+
+export type IngestInput = {
+  files: PackageFiles;
+  /** Slug the caller asked for; derived from the title when absent. */
+  requestedSlug?: string;
+  /** Whether the caller accepted overwriting a course they already own. */
+  overwrite: boolean;
+};
+
+export type IngestDeps = {
+  storage: CourseStorage;
+  /** Registry already scoped to the calling user's token. */
+  registry: PackageRegistry;
+  /** Organization every ingested package must end up under. */
+  organization: string;
+  now?: number;
+};
+
+export type IngestReport = {
+  slug: string;
+  packageId: number;
+  organization: string;
+  lessons: number;
+  /** Planned lessons the package held no content for. */
+  skipped: SkippedLesson[];
+  written: number;
+  deleted: number;
+  /**
+   * True when the slug was already registered but its bucket was empty, so this
+   * run completed an earlier attempt instead of overwriting anything.
+   */
+  recoveredEmptyPackage: boolean;
+};
+
+const parseJson = <T>(raw: string, path: string): T => {
+  try {
+    return JSON.parse(raw) as T
+  } catch (error) {
+    throw new IngestError(
+      "INVALID_PACKAGE",
+      `${path} is not valid JSON: ${(error as Error).message}`,
+      error
+    )
+  }
+}
+
+/**
+ * Reads the GitHub connection a previous config may hold.
+ *
+ * A user can link a course to a repository from the creator, and that link
+ * lives in `config.github`. Re-ingesting rewrites `config.json` wholesale, so
+ * without carrying this across, an overwrite would quietly disconnect a
+ * repository nobody asked to disconnect. `GET /config` guards the same field
+ * the same way.
+ * @param storage - Course storage.
+ * @param slug - Course slug.
+ * @returns The stored github block, or undefined.
+ */
+async function readGithubConnection(
+  storage: CourseStorage,
+  slug: string
+): Promise<unknown> {
+  const raw = await storage.read(configPath(slug))
+  if (!raw) {
+    return undefined
+  }
+
+  try {
+    const existing = JSON.parse(raw)
+    return existing?.config?.github
+  } catch {
+    // A config we cannot parse has nothing to preserve; the fresh one wins.
+    return undefined
+  }
+}
+
+/**
+ * Turns the configured organization into the id everything else speaks.
+ *
+ * Configuration names an organization the way a person would, but a package
+ * reports its organization as a numeric id: the field is a plain ForeignKey on
+ * a ModelSerializer, so it serializes to the primary key, and no endpoint ever
+ * returns a slug. Comparing the two directly is how a correctly configured
+ * ingest ends up refused with "belongs to organization 105, not learnpack-labs".
+ *
+ * Accepts an id as-is, and otherwise matches case-insensitively against the
+ * names of the organizations the caller belongs to. A miss lists them, since
+ * nothing else in the flow would ever show the reader an id to use.
+ * @param configured - Value of INGEST_ORGANIZATION.
+ * @param registry - Used to list the caller's organizations.
+ * @returns The organization id.
+ * @throws IngestError `ORGANIZATION_MISMATCH` listing the available options.
+ */
+async function resolveOrganizationId(
+  configured: string,
+  registry: PackageRegistry
+): Promise<number> {
+  const asId = Number(configured)
+  if (Number.isInteger(asId) && asId > 0) {
+    return asId
+  }
+
+  const organizations = await registry.listOrganizations()
+  const wanted = configured.trim().toLowerCase()
+  const match = organizations.find(
+    org => (org.name || "").trim().toLowerCase() === wanted
+  )
+
+  if (match) {
+    return match.id
+  }
+
+  const available = organizations
+    .map(org => `${org.id} (${org.name})`)
+    .join(", ")
+
+  throw new IngestError(
+    "ORGANIZATION_MISMATCH",
+    available ?
+      `INGEST_ORGANIZATION is "${configured}", which matches none of your organizations: ${available}. Use one of those ids or its exact name.` :
+      `INGEST_ORGANIZATION is "${configured}", but this token belongs to no organization`
+  )
+}
+
+/**
+ * Settles the package registration for a slug the caller may already own.
+ *
+ * Registering before writing is deliberate: it is the cheap check that catches
+ * a slug collision or a wrong organization while the bucket is still untouched.
+ * @param slug - Resolved slug.
+ * @param learnJson - Normalized learn.json, stored as the package config.
+ * @param deps - Registry and expected organization.
+ * @param overwrite - Whether the caller accepted overwriting their own course.
+ * @param storage - Used to tell an abandoned registration from a real course.
+ * @returns The package and whether this run is completing an earlier attempt.
+ */
+async function registerPackage(
+  slug: string,
+  learnJson: Record<string, unknown>,
+  deps: IngestDeps,
+  overwrite: boolean,
+  storage: CourseStorage
+): Promise<{ pkg: RigoPackage; recovered: boolean }> {
+  const available = await deps.registry.isSlugAvailable(slug)
+
+  if (available) {
+    return {
+      pkg: await deps.registry.createPackage(slug, learnJson),
+      recovered: false,
+    }
+  }
+
+  const existing = await deps.registry.getPackageBySlug(slug)
+
+  if (!existing) {
+    // Taken, and the registry will not show it to us: it belongs to another
+    // user. No suggestion is offered here — the caller cannot fix this by
+    // retrying, only by choosing a different slug.
+    throw new IngestError(
+      "SLUG_OWNED_BY_OTHER",
+      `The slug "${slug}" belongs to another user`
+    )
+  }
+
+  if (!overwrite) {
+    const contents = await storage.list(coursePrefix(slug))
+
+    // A registered slug with an empty bucket is the fingerprint of a run that
+    // registered and then failed before writing. Refusing it would strand the
+    // user behind a 409 for a course that, as far as they can tell, does not
+    // exist.
+    if (contents.length === 0) {
+      Console.info(
+        `[ingest] "${slug}" was registered but never written; completing that attempt`
+      )
+      return { pkg: existing, recovered: true }
+    }
+
+    const suggestions = await suggestSlugs(slug, deps.registry)
+    throw new IngestError(
+      "SLUG_TAKEN",
+      `You already have a course at "${slug}". Re-run with overwrite to replace it, or use one of: ${suggestions.join(
+        ", "
+      )}`
+    )
+  }
+
+  return { pkg: existing, recovered: false }
+}
+
+/**
+ * Ingests a finished course package: registers it and writes it to the bucket.
+ *
+ * The order is validate → resolve slug → register → write → prune, and it is
+ * not arbitrary. Validation and slug resolution are free and catch most bad
+ * requests; registration is the first thing that can conflict, and doing it
+ * before any write means a rejected course leaves nothing behind.
+ *
+ * Failures throw `IngestError` rather than being collected into the report. A
+ * partial ingest has no useful middle state to report — unlike publishing,
+ * nothing here runs detached after the response, so there is nothing for a
+ * later sweep to resume. Re-running is the recovery path, and the empty-package
+ * case above is what keeps that from being blocked.
+ * @param input - The extracted package and the caller's choices.
+ * @param deps - Storage, registry and expected organization.
+ * @returns What was written.
+ */
+export async function ingestCoursePackage(
+  input: IngestInput,
+  deps: IngestDeps
+): Promise<IngestReport> {
+  validatePackage(input.files)
+
+  const rawLearnJson = parseJson<Record<string, unknown>>(
+    input.files[LEARN_JSON_PATH],
+    LEARN_JSON_PATH
+  )
+  const plan = parseJson<LessonPlan>(
+    input.files[LESSON_PLAN_PATH],
+    LESSON_PLAN_PATH
+  )
+
+  const slug = resolveSlug({
+    requestedSlug: input.requestedSlug,
+    courseTitle: plan.course_title,
+  })
+
+  const learnJson = normalizeLearnJson({ learnJson: rawLearnJson, slug })
+
+  const { pkg, recovered } = await registerPackage(
+    slug,
+    learnJson,
+    deps,
+    input.overwrite,
+    deps.storage
+  )
+
+  const organizationId = await resolveOrganizationId(
+    deps.organization,
+    deps.registry
+  )
+
+  if (pkg.organization === null) {
+    await deps.registry.assignOrganization(pkg.id, organizationId)
+  } else if (pkg.organization !== organizationId) {
+    // There is no way to detach an organization through the API, so landing in
+    // the wrong one is only fixable by hand. Better to stop before writing.
+    throw new IngestError(
+      "ORGANIZATION_MISMATCH",
+      `"${slug}" belongs to organization ${pkg.organization}, not ${organizationId} (${deps.organization})`
+    )
+  }
+
+  const { slugs, languagesBySlug } = readExercises(input.files)
+  const { lessons, skipped } = pairLessons(plan, slugs, languagesBySlug)
+
+  const language = (plan.language || "en").trim().toLowerCase() || "en"
+  const syllabus = buildInitialSyllabus({
+    slug,
+    language,
+    lessons,
+    learnJson,
+    now: deps.now,
+  })
+  const sidebar = buildSidebar(lessons)
+
+  const github = input.overwrite ?
+    await readGithubConnection(deps.storage, slug) :
+    undefined
+  const projected = buildIngestConfig({ lessons, learnJson })
+  const config = github ?
+    { ...projected, config: { ...projected.config, github } } :
+    projected
+
+  const writes = planWrites({
+    slug,
+    files: input.files,
+    lessons,
+    learnJson,
+    syllabus,
+    sidebar,
+    config,
+  })
+
+  let deleted = 0
+  if (input.overwrite) {
+    const existing = await deps.storage.list(coursePrefix(slug))
+    const stale = planDeletions(existing, new Set(writes.keys()), slug)
+    await deps.storage.deleteMany(stale)
+    deleted = stale.length
+  }
+
+  // Written in parallel, with a cap. A course is around fifty small objects —
+  // one README per lesson plus the generation artifacts — and one round trip at
+  // a time turned a few seconds of work into most of a minute, long enough for
+  // the uploading client to give up while the server was still writing.
+  await mapWithConcurrency(
+    [...writes.entries()],
+    WRITE_CONCURRENCY,
+    ([path, contents]) => deps.storage.write(path, contents)
+  )
+
+  if (skipped.length > 0) {
+    Console.warning(
+      `[ingest] "${slug}": ${skipped.length} planned lesson(s) had no content and were left out`
+    )
+  }
+
+  return {
+    slug,
+    packageId: pkg.id,
+    organization: deps.organization,
+    lessons: lessons.length,
+    skipped,
+    written: writes.size,
+    deleted,
+    recoveredEmptyPackage: recovered,
+  }
+}

package/src/services/ingest/core/normalizeLearnJson.ts

@@ -0,0 +1,47 @@
+import { BREATHECODE_TELEMETRY_URL } from "../../../utils/api"
+
+export type NormalizeLearnJsonInput = {
+  /** The `learn.json` shipped inside the package. */
+  learnJson: Record<string, unknown>;
+  /** The slug the ingest resolved, which wins over whatever the package says. */
+  slug: string;
+};
+
+/**
+ * Fills in the fields the generator's exporter omits.
+ *
+ * The generator writes a `learn.json` good enough to run locally, but three
+ * fields only matter once the course lives in the bucket, and each fails
+ * quietly when missing:
+ *
+ * - `telemetry.batch` — without it student telemetry is never sent.
+ * - `preview` — publish falls back to the generic LearnPack logo.
+ * - `technologies` — the organization package listing filters on it.
+ *
+ * The slug is overwritten rather than trusted: it is resolved against the
+ * registry before anything is written, and the copy inside the package predates
+ * that. Leaving them to disagree puts the bucket path and `learn.json` out of
+ * sync, which `change-slug` would then fail to reconcile.
+ * @param input - The package's learn.json and the resolved slug.
+ * @returns A learn.json ready to be written to the bucket.
+ */
+export function normalizeLearnJson(
+  input: NormalizeLearnJsonInput
+): Record<string, unknown> {
+  const { learnJson, slug } = input
+  const telemetry =
+    (learnJson.telemetry as Record<string, unknown> | undefined) || {}
+
+  return {
+    ...learnJson,
+    slug,
+    technologies: Array.isArray(learnJson.technologies) ?
+      learnJson.technologies :
+      [],
+    telemetry: {
+      ...telemetry,
+      batch: telemetry.batch || BREATHECODE_TELEMETRY_URL,
+    },
+    preview: learnJson.preview || `https://${slug}.learn-pack.com/preview.png`,
+  }
+}

package/src/services/ingest/core/pairLessons.ts

@@ -0,0 +1,132 @@
+import Console from "../../../utils/console"
+import { LessonType } from "../../../utils/packageManifest"
+import {
+  IngestLesson,
+  LessonPlan,
+  PlannedLesson,
+  SkippedLesson,
+} from "./types"
+
+const VALID_LESSON_TYPES = new Set<LessonType>(["READ", "CODE", "QUIZ"])
+
+/**
+ * The exercise folder a lesson id owns, e.g. "01.1" for "01.1-split-a-class".
+ *
+ * Parsed by splitting rather than by a two-digit regex: the generator's own id
+ * parser accepts "1.0", so a stricter pattern here would reject a plan the
+ * generator considers valid.
+ * @param exerciseSlug - Folder name inside `exercises/`.
+ * @returns The leading id, or the whole slug when there is no hyphen.
+ */
+export function lessonIdFromSlug(exerciseSlug: string): string {
+  const [head] = exerciseSlug.split("-")
+  return head
+}
+
+/**
+ * Reads the lesson type, falling back rather than failing.
+ *
+ * The generator declares this as a Pydantic Literal, so a successfully
+ * generated course cannot carry anything else — but the package is consumed
+ * long after the fact, and a course that renders every lesson as READ beats one
+ * that refuses to ingest.
+ * @param type - Raw value from the plan.
+ * @param lessonId - Used to make the warning actionable.
+ * @returns A valid lesson type.
+ */
+function coerceLessonType(
+  type: string | undefined,
+  lessonId: string
+): LessonType {
+  if (type && VALID_LESSON_TYPES.has(type as LessonType)) {
+    return type as LessonType
+  }
+
+  if (type) {
+    Console.warning(
+      `[ingest] Lesson ${lessonId} has unknown type "${type}"; treating it as READ`
+    )
+  }
+
+  return "READ"
+}
+
+/**
+ * The generation topic for a lesson.
+ *
+ * `lesson_focus` is absent in packages from the pre-ADR-0001 schema, and this
+ * value is not what students read — the student-facing description lives per
+ * language in `translations[lang].description`, which the descriptions backfill
+ * fills in later. The title is a fine stand-in.
+ * @param lesson - Planned lesson.
+ * @returns The focus, or the title.
+ */
+function describeLesson(lesson: PlannedLesson): string {
+  const focus = (lesson.lesson_focus || "").trim()
+  return focus || lesson.title
+}
+
+export type PairLessonsResult = {
+  lessons: IngestLesson[];
+  skipped: SkippedLesson[];
+};
+
+/**
+ * Matches planned lessons to the exercise folders that actually hold content.
+ *
+ * A plan lists every lesson that was *planned*; module-batched generation can
+ * stop early, leaving later modules without files. Pairing against the folders
+ * present is what makes a partial course ingest as the eight lessons it has
+ * rather than the twelve it intended — and the leftovers are reported instead
+ * of dropped, so nobody discovers the gap by scrolling the IDE.
+ * @param plan - The generator's lesson plan.
+ * @param exerciseSlugs - Folder names found under `exercises/` in the package.
+ * @param languagesBySlug - Language codes with a README, per exercise slug.
+ * @returns Paired lessons in plan order, plus the entries that had no content.
+ */
+export function pairLessons(
+  plan: LessonPlan,
+  exerciseSlugs: string[],
+  languagesBySlug: Record<string, string[]>
+): PairLessonsResult {
+  const slugsById = new Map<string, string>()
+  for (const slug of exerciseSlugs) {
+    const id = lessonIdFromSlug(slug)
+    // First folder wins: a duplicate id means the package is malformed, and
+    // picking arbitrarily is no worse than picking the last one.
+    if (!slugsById.has(id)) {
+      slugsById.set(id, slug)
+    }
+  }
+
+  const lessons: IngestLesson[] = []
+  const skipped: SkippedLesson[] = []
+
+  for (const planned of plan.lessons || []) {
+    const exerciseSlug = slugsById.get(planned.id)
+
+    if (!exerciseSlug) {
+      skipped.push({
+        id: planned.id,
+        title: planned.title,
+        reason: "no-content",
+      })
+      continue
+    }
+
+    lessons.push({
+      id: planned.id,
+      exerciseSlug,
+      title: planned.title,
+      type: coerceLessonType(planned.type, planned.id),
+      durationMinutes:
+        typeof planned.duration_minutes === "number" ?
+          planned.duration_minutes :
+          0,
+      description: describeLesson(planned),
+      languages: languagesBySlug[exerciseSlug] || [],
+    })
+  }
+
+  return { lessons, skipped }
+}

package/src/services/ingest/core/planWrites.ts

@@ -0,0 +1,91 @@
+import {
+  configPath,
+  coursePrefix,
+  learnJsonPath,
+  prunablePrefixes,
+  sidebarPath,
+  syllabusPath,
+} from "./coursePaths"
+import { PackageFiles } from "./validatePackage"
+import { IngestLesson } from "./types"
+
+/** Package paths copied through untouched, keyed by their prefix. */
+const PASSTHROUGH_PREFIXES = [".learn/artifacts/", "exercises/"]
+
+export type PlanWritesInput = {
+  slug: string;
+  files: PackageFiles;
+  lessons: IngestLesson[];
+  learnJson: Record<string, unknown>;
+  syllabus: unknown;
+  sidebar: unknown;
+  config: unknown;
+};
+
+const normalize = (path: string): string => path.replace(/\\/g, "/")
+
+const stringify = (value: unknown): string => JSON.stringify(value, null, 2)
+
+/**
+ * Lays out every object the ingest will write, keyed by its bucket path.
+ *
+ * Returned rather than written so the orchestrator can compare the plan against
+ * what is already in the bucket before touching anything — which is what makes
+ * pruning stale lessons possible, and what lets a test assert the whole tree
+ * without a storage double.
+ *
+ * Content files ride through verbatim; only the four derived documents are
+ * generated. Note `.learn/config.json` from the package is *not* among them:
+ * it is a projection of the other files, and the copy the generator ships
+ * carries localhost URLs and a stale exercise list.
+ * @param input - Slug, package files and the derived documents.
+ * @returns Bucket path → contents.
+ */
+export function planWrites(input: PlanWritesInput): Map<string, string> {
+  const writes = new Map<string, string>()
+  const prefix = coursePrefix(input.slug)
+
+  for (const [rawPath, contents] of Object.entries(input.files)) {
+    const path = normalize(rawPath)
+
+    if (PASSTHROUGH_PREFIXES.some(known => path.startsWith(known))) {
+      writes.set(`${prefix}${path}`, contents)
+    }
+  }
+
+  writes.set(learnJsonPath(input.slug), stringify(input.learnJson))
+  writes.set(syllabusPath(input.slug), stringify(input.syllabus))
+  writes.set(sidebarPath(input.slug), stringify(input.sidebar))
+  writes.set(configPath(input.slug), stringify(input.config))
+
+  return writes
+}
+
+/**
+ * Objects the course still holds that this ingest is not going to rewrite.
+ *
+ * Scoped to the prefixes the ingest owns. Everything else under the course
+ * belongs to somebody else — preview images the user uploaded,
+ * `package-manifest.json`, the publish journal — and deleting those would
+ * destroy work the ingest never created.
+ *
+ * Pruning matters because `buildConfig` derives the exercise list from whatever
+ * files exist: a regenerated course with fewer lessons would otherwise keep
+ * listing the old folders as lessons that open empty.
+ * @param existing - Every object path currently under the course.
+ * @param planned - Paths this ingest is about to write.
+ * @param slug - Course slug, for scoping the prunable prefixes.
+ * @returns Paths safe to delete.
+ */
+export function planDeletions(
+  existing: string[],
+  planned: Set<string>,
+  slug: string
+): string[] {
+  const prunable = prunablePrefixes(slug)
+
+  return existing.filter(
+    path =>
+      !planned.has(path) && prunable.some(prefix => path.startsWith(prefix))
+  )
+}