@learnpack/learnpack 5.0.359 → 5.0.360

package/src/creator/src/utils/courseLanguage.ts

@@ -0,0 +1,173 @@
+import { franc } from "franc";
+
+export type TCourseLanguage = "en" | "es";
+
+/**
+ * Narrows a language code to one the creator can build a course in.
+ * @param value - A code such as "es", "EN" or an i18next resolved language.
+ * @returns "en" or "es", or null for anything else (including "es-MX").
+ */
+export const toCourseLanguage = (
+  value: string | null | undefined
+): TCourseLanguage | null => {
+  const code = value?.trim().toLowerCase();
+  return code === "en" || code === "es" ? code : null;
+};
+
+// franc only reads the first 2048 characters of whatever it is given, so long
+// text is judged window by window.
+const WINDOW_LENGTH = 2048;
+// Enough windows that a cover page or front matter cannot decide alone, few
+// enough that an English bibliography at the end of a Spanish document cannot
+// outvote its body.
+const MAX_WINDOWS = 5;
+// Below this many letters, titles, code and tables misclassify. Digits and
+// symbols are not counted because they carry no language.
+const MIN_LETTERS = 200;
+// A window shorter than this at the end of the text is too small to vote.
+const MIN_WINDOW_LENGTH = 200;
+const MAJORITY = 2 / 3;
+
+const countLetters = (text: string) => (text.match(/\p{L}/gu) || []).length;
+
+/**
+ * Detects whether a piece of seed text is English or Spanish.
+ *
+ * Votes over the first few 2048-character windows and only answers on a
+ * two-thirds majority, so mixed or ambiguous text comes back as null rather
+ * than as a guess. Windows in any other language vote too, so a Portuguese
+ * or French document is not passed off as Spanish or English.
+ * @param text - Text extracted from an uploaded file, pasted text or a
+ * transcript.
+ * @returns "en" or "es", "other" when the text is mostly in a language the
+ * creator does not support, or null when there is too little text or no clear
+ * majority.
+ */
+export const detectTextLanguage = (
+  text: string
+): TCourseLanguage | "other" | null => {
+  if (countLetters(text) < MIN_LETTERS) return null;
+
+  let english = 0;
+  let spanish = 0;
+  let other = 0;
+  let windows = 0;
+  for (
+    let start = 0;
+    start < text.length && windows < MAX_WINDOWS;
+    start += WINDOW_LENGTH
+  ) {
+    const window = text.slice(start, start + WINDOW_LENGTH);
+    if (start > 0 && window.length < MIN_WINDOW_LENGTH) continue;
+    windows++;
+    const lang = franc(window);
+    if (lang === "eng") english++;
+    else if (lang === "spa") spanish++;
+    else if (lang !== "und") other++;
+  }
+
+  const votes = english + spanish + other;
+  if (votes === 0) return null;
+  if (english / votes >= MAJORITY) return "en";
+  if (spanish / votes >= MAJORITY) return "es";
+  if (other / votes >= MAJORITY) return "other";
+  return null;
+};
+
+export type TSeedLanguage = TCourseLanguage | "mixed" | null;
+
+/**
+ * Detects the language of everything the user gave as course content, judging
+ * each source separately so one long file cannot outvote a short one.
+ * @param seeds - The text of each uploaded file, pasted text or transcript.
+ * @returns The shared language, "mixed" when confidently detected sources
+ * disagree, or null when none could be judged or any source is in a language
+ * the creator does not support (the model then picks, as before).
+ */
+export const detectSeedLanguage = (seeds: string[]): TSeedLanguage => {
+  const detected = seeds.map(detectTextLanguage);
+  if (detected.includes("other")) return null;
+  const languages = new Set(
+    detected.filter((lang): lang is TCourseLanguage => lang !== null)
+  );
+  if (languages.size === 0) return null;
+  if (languages.size > 1) return "mixed";
+  return [...languages][0];
+};
+
+// Words that exist in only one of the two languages. Ambiguous ones such as
+// "a", "me", "no" or "son" are left out on purpose.
+const ENGLISH_ONLY_WORDS = new Set(
+  "the to and of for with how what why when your my our this that these those is are was be into from about on in an i we you want learn teach build create make use using write explain help course lesson beginners introduction basics guide".split(
+    " "
+  )
+);
+const SPANISH_ONLY_WORDS = new Set(
+  "el la los las del de para con por sobre cómo como qué que y en un una unos unas mi mis nuestro nuestra nuestros es son está quiero aprender aprende enseña enseñar crear crea construir usar usando escribir explica ayúdame curso lección principiantes introducción básico básica guía desde".split(
+    " "
+  )
+);
+const SPANISH_CHARACTERS = /[ñáéíóú¿¡]/i;
+
+/**
+ * Detects the language a course description was written in.
+ *
+ * Descriptions are a few words long and full of technical terms, which
+ * statistical detection misreads ("Learn TypeScript generics" comes out as
+ * Spanish). Counting words that only exist in one language is reliable at
+ * that length, and answering only when the other language shows no sign at
+ * all keeps mixed descriptions out.
+ * @param description - The course description from the wizard.
+ * @returns "en" or "es", or null when the description is mixed or has no
+ * telling words.
+ */
+export const detectDescriptionLanguage = (
+  description: string
+): TCourseLanguage | null => {
+  const words = description.toLowerCase().match(/\p{L}+/gu) || [];
+  let english = 0;
+  let spanish = 0;
+  for (const word of words) {
+    if (ENGLISH_ONLY_WORDS.has(word)) english++;
+    if (SPANISH_ONLY_WORDS.has(word)) spanish++;
+  }
+  if (SPANISH_CHARACTERS.test(description)) spanish++;
+
+  if (english > 0 && spanish === 0) return "en";
+  if (spanish > 0 && english === 0) return "es";
+  return null;
+};
+
+export type TCourseLanguageDecision =
+  // Nothing settles the language: the request goes out exactly as before and
+  // the model picks.
+  | { kind: "model" }
+  | { kind: "use"; language: TCourseLanguage }
+  // Only the user can say which language they meant.
+  | { kind: "ask" };
+
+/**
+ * Decides the course language before generation.
+ *
+ * A language the user chose always wins. Otherwise the seed's language is used
+ * when the description agrees or cannot be judged, and any disagreement is put
+ * to the user. A misread description can therefore only cause a question,
+ * never a course in the wrong language.
+ */
+export const decideCourseLanguage = ({
+  chosen,
+  seed,
+  description,
+}: {
+  chosen: TCourseLanguage | null;
+  seed: TSeedLanguage;
+  description: TCourseLanguage | null;
+}): TCourseLanguageDecision => {
+  if (chosen) return { kind: "use", language: chosen };
+  if (seed === null) return { kind: "model" };
+  if (seed === "mixed") return { kind: "ask" };
+  if (description === null || description === seed) {
+    return { kind: "use", language: seed };
+  }
+  return { kind: "ask" };
+};

package/src/creator/src/utils/creationRequest.test.ts

@@ -0,0 +1,104 @@
+import { describe, it, expect } from "vitest";
+import type { ParsedFile } from "../components/FileUploader";
+import { buildCourseInfo, toCourseFormState } from "./creationRequest";
+import useStore, { FormState } from "./store";
+
+/**
+ * The courseInfo the wizard sent before the course language could be settled,
+ * copied byte for byte from App.tsx (upstream 68c0bf9). Whitespace inside the
+ * template is part of the prompt, so it must not be reformatted.
+ */
+const legacyCourseInfo = (
+  formState: FormState,
+  techs: { slug: string }[],
+  uploadedFiles: ParsedFile[]
+) =>
+  // prettier-ignore
+  `${JSON.stringify(
+            formState
+          )}. The following technologies are available, choose up to 3 from the following list: <techs>${techs
+            .map((t) => t.slug)
+            .join(", ")}</techs>
+            
+            The following files have been uploaded by the user: ${uploadedFiles.map((f) => `<FILE name="${f.name}">${f.text}</FILE>`).join("\n")}
+            `;
+
+const defaultFormState = () => useStore.getInitialState().formState;
+
+const techs = [{ slug: "python" }, { slug: "sql" }];
+const uploadedFiles: ParsedFile[] = [
+  {
+    name: "notes.pdf",
+    text: "Relational databases",
+    status: "SUCCESS",
+    notificationId: "n1",
+  },
+];
+
+describe("buildCourseInfo", () => {
+  it("sends exactly the previous prompt when no language is settled", () => {
+    const formState = {
+      ...defaultFormState(),
+      description: "Learn SQL joins",
+      purpose: "learnpack-lesson-writer",
+      duration: 30,
+      hasContentIndex: true,
+    };
+    // The wizard state before this change had no languageChosen key.
+    const previousFormState = { ...formState };
+    delete previousFormState.languageChosen;
+
+    expect(
+      buildCourseInfo({ formState, techs, uploadedFiles, language: null })
+    ).toBe(legacyCourseInfo(previousFormState, techs, uploadedFiles));
+  });
+
+  it("asks for the settled language in plain words", () => {
+    const formState = { ...defaultFormState(), language: "en" };
+    const courseInfo = buildCourseInfo({
+      formState,
+      techs,
+      uploadedFiles,
+      language: "es",
+    });
+
+    expect(courseInfo).toContain('"language":"es"');
+    expect(courseInfo).not.toContain('"language":"en"');
+    expect(
+      courseInfo.endsWith(
+        "The user chose Spanish as the course language. This replaces the rule to write in the language of the user messages: write every field in Spanish, including the title, the description, and each step's title and description in listOfSteps, even if the user's messages or the uploaded files are in another language. Set languageCode to \"es\"."
+      )
+    ).toBe(true);
+  });
+
+  it("never sends languageChosen to the model", () => {
+    const formState = {
+      ...defaultFormState(),
+      language: "es",
+      languageChosen: true,
+    };
+    for (const language of [null, "es"] as const) {
+      expect(
+        buildCourseInfo({ formState, techs, uploadedFiles, language })
+      ).not.toContain("languageChosen");
+    }
+  });
+});
+
+describe("toCourseFormState", () => {
+  it("drops languageChosen and keeps every other key in order", () => {
+    const formState = { ...defaultFormState(), languageChosen: true };
+    const courseFormState = toCourseFormState(formState);
+
+    expect("languageChosen" in courseFormState).toBe(false);
+    expect(Object.keys(courseFormState)).toEqual(
+      Object.keys(formState).filter((key) => key !== "languageChosen")
+    );
+  });
+
+  it("does not change the form state it was given", () => {
+    const formState = { ...defaultFormState(), languageChosen: true };
+    toCourseFormState(formState);
+    expect(formState.languageChosen).toBe(true);
+  });
+});

package/src/creator/src/utils/creationRequest.ts

@@ -0,0 +1,63 @@
+import type { ParsedFile } from "../components/FileUploader";
+import type { TCourseLanguage } from "./courseLanguage";
+import type { FormState } from "./store";
+
+const LANGUAGE_NAMES: Record<TCourseLanguage, string> = {
+  en: "English",
+  es: "Spanish",
+};
+
+/**
+ * Drops the wizard-only fields from the form state before it leaves the
+ * wizard, in the prompt or in the syllabus sent to the server.
+ * @param formState - The wizard's form state.
+ * @returns A copy without `languageChosen`, keys otherwise in the same order.
+ */
+export const toCourseFormState = (formState: FormState): FormState => {
+  const courseFormState = { ...formState };
+  delete courseFormState.languageChosen;
+  return courseFormState;
+};
+
+/**
+ * Builds the `courseInfo` text sent to the model to generate the syllabus.
+ *
+ * Without a settled language the text is exactly what the wizard sent before
+ * the course language could be settled, so the model keeps choosing.
+ * @param language - The course language settled before generation, or null.
+ */
+export const buildCourseInfo = ({
+  formState,
+  techs,
+  uploadedFiles,
+  language,
+}: {
+  formState: FormState;
+  techs: { slug: string }[];
+  uploadedFiles: ParsedFile[];
+  language: TCourseLanguage | null;
+}): string => {
+  const courseFormState = toCourseFormState(formState);
+
+  // Kept byte for byte from the original request, including the spaces on the
+  // blank line, which .editorconfig would trim. creationRequest.test.ts checks
+  // it against a copy of the original.
+  const courseInfo = `${JSON.stringify(
+    language ? { ...courseFormState, language } : courseFormState
+  )}. The following technologies are available, choose up to 3 from the following list: <techs>${techs
+    .map((t) => t.slug)
+    .join(", ")}</techs>
+            
+            The following files have been uploaded by the user: ${uploadedFiles
+              .map((f) => `<FILE name="${f.name}">${f.text}</FILE>`)
+              .join("\n")}
+            `;
+
+  if (!language) return courseInfo;
+
+  const name = LANGUAGE_NAMES[language];
+  // Prompt 390 keys on the first sentence to override its rule of writing in
+  // the language of the user messages, so keep its wording in sync.
+  return `${courseInfo}
+The user chose ${name} as the course language. This replaces the rule to write in the language of the user messages: write every field in ${name}, including the title, the description, and each step's title and description in listOfSteps, even if the user's messages or the uploaded files are in another language. Set languageCode to "${language}".`;
+};

package/src/creator/src/utils/store.ts

@@ -15,6 +15,10 @@ export type FormState = {
   difficulty: TDifficulty;
   slug: string;
   language?: string;
+  // True once the user picked the course language themselves (the language
+  // picker, the `language` URL param or a language modal). Never sent to the
+  // model or the server.
+  languageChosen?: boolean;
   isCompleted: boolean;
   variables: string[];
   currentStep: string;
@@ -93,6 +97,7 @@ const useStore = create<Store>()(
         contentIndex: "",
         purpose: "",
         language: "en",
+        languageChosen: false,
         technologies: [],
         difficulty: "beginner",
         // sources: [],
@@ -129,6 +134,7 @@ const useStore = create<Store>()(
             description: "",
             duration: 0,
             language: "en",
+            languageChosen: false,
             difficulty: "beginner",
             technologies: [],
             hasContentIndex: false,
@@ -179,6 +185,7 @@ const useStore = create<Store>()(
             slug: "",
             difficulty: "beginner",
             language: "en",
+            languageChosen: false,
             technologies: [],
             hasContentIndex: false,
             contentIndex: "",

package/src/creatorDist/assets/{index-zrPponAn.css → index-B7s5Nmq8.css}

@@ -65,6 +65,7 @@
     --color-blue-50: oklch(97% 0.014 254.604);
     --color-blue-100: oklch(93.2% 0.032 255.585);
     --color-blue-200: oklch(88.2% 0.059 254.128);
+    --color-blue-300: oklch(80.9% 0.105 251.813);
     --color-blue-400: oklch(70.7% 0.165 254.624);
     --color-blue-500: oklch(62.3% 0.214 259.815);
     --color-blue-600: oklch(54.6% 0.245 262.881);
@@ -439,6 +440,12 @@
   .z-1000 {
     z-index: 1000;
   }
+  .col-span-2 {
+    grid-column: span 2 / span 2;
+  }
+  .row-start-2 {
+    grid-row-start: 2;
+  }
   .container {
     width: 100%;
   }
@@ -536,6 +543,9 @@
   .flex {
     display: flex;
   }
+  .grid {
+    display: grid;
+  }
   .hidden {
     display: none;
   }
@@ -548,6 +558,9 @@
   .table {
     display: table;
   }
+  .field-sizing-content {
+    field-sizing: content;
+  }
   .h-2 {
     height: calc(var(--spacing) * 2);
   }
@@ -644,6 +657,9 @@
   .w-full {
     width: 100%;
   }
+  .max-w-64 {
+    max-width: calc(var(--spacing) * 64);
+  }
   .max-w-\[700px\] {
     max-width: 700px;
   }
@@ -705,6 +721,9 @@
   .list-disc {
     list-style-type: disc;
   }
+  .grid-cols-2 {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+  }
   .flex-col {
     flex-direction: column;
   }
@@ -780,6 +799,15 @@
       calc(var(--spacing) * 2) * calc(1 - var(--tw-space-x-reverse))
     );
   }
+  .justify-self-center {
+    justify-self: center;
+  }
+  .justify-self-end {
+    justify-self: flex-end;
+  }
+  .justify-self-start {
+    justify-self: flex-start;
+  }
   .truncate {
     text-overflow: ellipsis;
     white-space: nowrap;
@@ -905,6 +933,9 @@
   .bg-red-100 {
     background-color: var(--color-red-100);
   }
+  .bg-transparent {
+    background-color: #0000;
+  }
   .bg-white {
     background-color: var(--color-white);
   }
@@ -1046,6 +1077,9 @@
   .break-words {
     overflow-wrap: break-word;
   }
+  .whitespace-nowrap {
+    white-space: nowrap;
+  }
   .whitespace-pre-line {
     white-space: pre-line;
   }
@@ -1244,6 +1278,19 @@
   .focus\:ring-blue-200:focus {
     --tw-ring-color: var(--color-blue-200);
   }
+  .focus\:outline-none:focus {
+    --tw-outline-style: none;
+    outline-style: none;
+  }
+  .focus-visible\:ring-2:focus-visible {
+    --tw-ring-shadow: var(--tw-ring-inset) 0 0 0
+      calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color, currentcolor);
+    box-shadow: var(--tw-inset-shadow), var(--tw-inset-ring-shadow),
+      var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);
+  }
+  .focus-visible\:ring-blue-300:focus-visible {
+    --tw-ring-color: var(--color-blue-300);
+  }
   .disabled\:cursor-not-allowed:disabled {
     cursor: not-allowed;
   }
@@ -1257,9 +1304,21 @@
     opacity: 0.5;
   }
   @media (min-width: 40rem) {
+    .sm\:col-span-1 {
+      grid-column: span 1 / span 1;
+    }
+    .sm\:col-start-2 {
+      grid-column-start: 2;
+    }
+    .sm\:row-start-1 {
+      grid-row-start: 1;
+    }
     .sm\:w-auto {
       width: auto;
     }
+    .sm\:grid-cols-\[1fr_auto_1fr\] {
+      grid-template-columns: 1fr auto 1fr;
+    }
     .sm\:flex-row {
       flex-direction: row;
     }