brightspace-mcp-server 3.13.8 → 3.13.9

package/build/utils/transcript/captions.js

@@ -12,6 +12,121 @@ function parseTimestamp(raw) {
     return ((Number(hours ?? 0) * 3600 + Number(minutes) * 60 + Number(seconds)) * 1000 +
         Number(millis));
 }
+const ENTITIES = {
+    amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: "\u00a0", "#39": "'", "#34": '"',
+};
+/**
+ * Decode the XML/HTML entities caption payloads carry. YouTube's tracks are
+ * full of `&#39;` and `&amp;`, which used to reach the user verbatim.
+ */
+function decodeEntities(text) {
+    return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, body) => {
+        const key = body.toLowerCase();
+        if (ENTITIES[key] !== undefined)
+            return ENTITIES[key];
+        if (key.startsWith("#x")) {
+            const code = Number.parseInt(key.slice(2), 16);
+            return Number.isFinite(code) && code > 0 ? String.fromCodePoint(code) : whole;
+        }
+        if (key.startsWith("#")) {
+            const code = Number.parseInt(key.slice(1), 10);
+            return Number.isFinite(code) && code > 0 ? String.fromCodePoint(code) : whole;
+        }
+        return whole;
+    });
+}
+/*
+ * Only strip the tags captions actually use. The old `/<[^>]+>/g` ate
+ * everything between any two angle brackets, so a lecture line like
+ * "if x < y and y > z" silently lost the middle -- and maths and inequalities
+ * are exactly what a course recording says out loud.
+ */
+const CAPTION_TAG = /<\/?(?:[0-9:.]+|c(?:\.[^>\s]*)?|v(?:\s[^>]*)?|b|i|u|ruby|rt|lang(?:\s[^>]*)?)>/gi;
+function stripCaptionTags(text) {
+    return text.replace(CAPTION_TAG, "");
+}
+/**
+ * YouTube's auto-generated tracks roll: each cue repeats the tail of the one
+ * before it, so a naive join says every line two or three times. Drop a cue
+ * whose text the previous cue already ended with.
+ */
+function dropRollingRepeats(cues) {
+    const out = [];
+    for (const cue of cues) {
+        const previous = out[out.length - 1];
+        if (previous && (previous.text === cue.text || previous.text.endsWith(cue.text)))
+            continue;
+        if (previous && cue.text.startsWith(previous.text) && cue.text.length > previous.text.length) {
+            out[out.length - 1] = { ...previous, endMs: cue.endMs, text: cue.text };
+            continue;
+        }
+        out.push(cue);
+    }
+    return out;
+}
+/**
+ * A TTML/DFXP time expression: a clock value ("00:00:01.500", optionally with
+ * frames) or an offset ("1.5s", "150ms", "90f" treated as seconds-less).
+ */
+function parseTimeExpression(raw) {
+    const value = raw.trim();
+    const offset = /^(\d+(?:\.\d+)?)(h|min|m|s|ms)?$/.exec(value);
+    if (offset) {
+        const n = Number(offset[1]);
+        const unit = offset[2] ?? "s";
+        const scale = unit === "h" ? 3600_000 : unit === "min" || unit === "m" ? 60_000 : unit === "ms" ? 1 : 1000;
+        return Math.round(n * scale);
+    }
+    const clock = /^(?:(\d+):)?(\d{1,2}):(\d{2})(?:[.,](\d{1,3}))?$/.exec(value);
+    if (!clock)
+        return NaN;
+    const [, h, m, sec, frac] = clock;
+    const millis = frac ? Number(frac.padEnd(3, "0")) : 0;
+    return (Number(h ?? 0) * 3600 + Number(m) * 60 + Number(sec)) * 1000 + millis;
+}
+/**
+ * Timed-text XML. Three shapes reach us: YouTube srv3 (`<p t= d=>`), YouTube
+ * srv1 (`<text start= dur=>`), and TTML/DFXP (`<p begin= end=>`), which is
+ * what Kaltura serves for many caption assets. TTML used to parse as zero
+ * cues and be reported as a track with nothing readable in it.
+ */
+function parseTimedText(xml) {
+    const cues = [];
+    const element = /<(p|text)\b([^>]*)>([\s\S]*?)<\/\1>/g;
+    for (const match of xml.matchAll(element)) {
+        const attrs = match[2];
+        const numericStart = /\b(?:t|start)="([\d.]+)"/.exec(attrs)?.[1];
+        const numericDur = /\b(?:d|dur)="([\d.]+)"/.exec(attrs)?.[1];
+        const beginRaw = /\bbegin="([^"]+)"/.exec(attrs)?.[1];
+        const endRaw = /\bend="([^"]+)"/.exec(attrs)?.[1];
+        const ttmlDur = /\bdur="([^"]+)"/.exec(attrs)?.[1];
+        let startMs;
+        let durMs;
+        if (beginRaw !== undefined) {
+            startMs = parseTimeExpression(beginRaw);
+            const endMs = endRaw !== undefined ? parseTimeExpression(endRaw) : NaN;
+            durMs = Number.isFinite(endMs)
+                ? endMs - startMs
+                : ttmlDur !== undefined ? parseTimeExpression(ttmlDur) : 0;
+        }
+        else if (numericStart !== undefined) {
+            // srv3 counts milliseconds, srv1 counts fractional seconds.
+            const seconds = match[1] === "text";
+            startMs = Math.round(Number(numericStart) * (seconds ? 1000 : 1));
+            durMs = numericDur === undefined ? 0 : Math.round(Number(numericDur) * (seconds ? 1000 : 1));
+        }
+        else {
+            continue;
+        }
+        if (!Number.isFinite(startMs) || !Number.isFinite(durMs))
+            continue;
+        // <s> word segments inside a <p> carry the words for ASR tracks.
+        const text = decodeEntities(match[3].replace(/<[^>]*>/g, " ")).replace(/\s+/g, " ").trim();
+        if (text)
+            cues.push({ startMs, endMs: startMs + Math.max(0, durMs), text });
+    }
+    return cues;
+}
 /**
  * Parse WebVTT or SRT text into cues. Format is detected from the WEBVTT
  * header rather than taken on faith, since a caller only knows what the
@@ -19,7 +134,13 @@ function parseTimestamp(raw) {
  */
 export function parseCaptions(raw) {
     const normalized = raw.replace(/^/, "").replace(/\r\n?/g, "\n");
-    const format = /^WEBVTT/.test(normalized.trimStart()) ? "vtt" : "srt";
+    const head = normalized.trimStart();
+    // Dispatch on what actually arrived. Calling every non-VTT payload "srt" and
+    // then finding no cues is how an XML or HTML body got reported as a caption
+    // track with nothing readable in it.
+    if (head.startsWith("<"))
+        return { format: "timedtext", cues: dropRollingRepeats(parseTimedText(normalized)) };
+    const format = /^WEBVTT\b/.test(head) ? "vtt" : "srt";
     const cues = [];
     for (const block of normalized.split(/\n{2,}/)) {
         const lines = block.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
@@ -31,15 +152,13 @@ export function parseCaptions(raw) {
         const endMs = parseTimestamp((endRaw ?? "").trim().split(/\s+/)[0] ?? "");
         if (Number.isNaN(startMs) || Number.isNaN(endMs))
             continue;
-        const text = lines
-            .slice(timingIndex + 1)
-            .join(" ")
-            .replace(/<[^>]+>/g, "")
+        const text = decodeEntities(stripCaptionTags(lines.slice(timingIndex + 1).join(" ")))
+            .replace(/\s+/g, " ")
             .trim();
         if (text)
             cues.push({ startMs, endMs, text });
     }
-    return { format, cues };
+    return { format, cues: dropRollingRepeats(cues) };
 }
 /** "H:MM:SS", matching how course-length recordings are usually captioned. */
 export function formatTimestamp(ms) {

package/build/utils/transcript/kaltura.js

@@ -21,6 +21,14 @@
 import { parseCaptions } from "./captions.js";
 import { NoTranscriptError, TranscriptFetchError } from "./errors.js";
 const KALTURA_API_BASE = "https://cdnapisec.kaltura.com/api_v3/service";
+/** The API's own refusal, in either envelope, or undefined when it did not refuse. */
+function apiException(data) {
+    if (data.error)
+        return data.error.message ?? data.error.code ?? "unknown error";
+    if (data.objectType === "KalturaAPIException")
+        return data.message ?? data.code ?? "unknown error";
+    return undefined;
+}
 async function startWidgetSession(partnerId, fetchImpl) {
     const res = await fetchImpl(`${KALTURA_API_BASE}/session/action/startWidgetSession?widgetId=_${encodeURIComponent(partnerId)}&format=1`);
     if (!res.ok) {
@@ -42,10 +50,16 @@ async function listCaptionAssets(ks, entryId, fetchImpl) {
         throw new TranscriptFetchError(`Kaltura caption list request failed (HTTP ${res.status}).`);
     }
     const data = (await res.json());
-    if (data.error) {
-        throw new TranscriptFetchError(`Kaltura rejected the caption list request: ${data.error.message ?? "unknown error"}.`);
+    const refusal = apiException(data);
+    if (refusal) {
+        throw new TranscriptFetchError(`Kaltura rejected the caption list request: ${refusal}.`);
+    }
+    // A missing `objects` is a malformed answer, not an empty list. Only a real
+    // empty array means the entry genuinely has no caption assets.
+    if (!Array.isArray(data.objects)) {
+        throw new TranscriptFetchError("Kaltura returned an unreadable caption list for this video.");
     }
-    return data.objects ?? [];
+    return data.objects;
 }
 async function serveCaption(ks, captionAssetId, fetchImpl) {
     const res = await fetchImpl(`${KALTURA_API_BASE}/caption_captionasset/action/serve?ks=${encodeURIComponent(ks)}` +
@@ -53,7 +67,17 @@ async function serveCaption(ks, captionAssetId, fetchImpl) {
     if (!res.ok) {
         throw new TranscriptFetchError(`Kaltura caption download failed (HTTP ${res.status}).`);
     }
-    return res.text();
+    const body = await res.text();
+    // The asset was listed a moment ago, so nothing coming back is a refusal,
+    // not an absence. An error page arrives as JSON here rather than as cues.
+    if (!body.trim()) {
+        throw new TranscriptFetchError("Kaltura listed a caption asset for this video but served an empty response for it.");
+    }
+    const refusal = body.trimStart().startsWith("{") ? apiException(JSON.parse(body)) : undefined;
+    if (refusal) {
+        throw new TranscriptFetchError(`Kaltura refused to serve the caption asset: ${refusal}.`);
+    }
+    return body;
 }
 /** Best-effort title/duration — a caption-only entry still has a usable transcript without this. */
 async function fetchMediaInfo(ks, entryId, fetchImpl) {
@@ -76,12 +100,30 @@ export async function getKalturaTranscript(partnerId, entryId, fetchImpl) {
     if (assets.length === 0) {
         throw new NoTranscriptError("This Kaltura video has no captions available.");
     }
-    const asset = assets.find((a) => a.isDefault) ?? assets[0];
-    const raw = await serveCaption(ks, asset.id, fetchImpl);
-    const { format, cues } = parseCaptions(raw);
-    if (cues.length === 0) {
-        throw new NoTranscriptError("This Kaltura video has a caption track, but it contained no readable cues.");
+    // Try the default asset first, then the rest: an entry can carry a stale or
+    // empty asset alongside a good one, and giving up on the first was enough to
+    // report a captioned lecture as having no transcript.
+    const ordered = [...assets].sort((a, b) => Number(Boolean(b.isDefault)) - Number(Boolean(a.isDefault)));
+    let asset = ordered[0];
+    let parsed;
+    let lastError;
+    for (const candidate of ordered) {
+        try {
+            const attempt = parseCaptions(await serveCaption(ks, candidate.id, fetchImpl));
+            if (attempt.cues.length > 0) {
+                asset = candidate;
+                parsed = attempt;
+                break;
+            }
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : undefined;
+        }
+    }
+    if (!parsed) {
+        throw lastError ?? new TranscriptFetchError("Kaltura served this video's caption assets in a format that could not be read.");
     }
+    const { format, cues } = parsed;
     const media = await fetchMediaInfo(ks, entryId, fetchImpl);
     return {
         cues,

package/build/utils/transcript/platform.js

@@ -43,7 +43,12 @@ export function detectVideoPlatform(url) {
     // query params or path segments before falling back to "unknown".
     if (hasHostLabel(host, "kaltura"))
         return "kaltura";
-    if (isDomainOrSubdomain(host, "youtube.com") || isDomainOrSubdomain(host, "youtu.be"))
+    if (isDomainOrSubdomain(host, "youtube.com")
+        || isDomainOrSubdomain(host, "youtu.be")
+        // Privacy-mode embeds are the same video. extractYouTubeVideoId has always
+        // accepted this host, so leaving it out here made the tool answer
+        // "unsupported platform" for a link it could in fact read.
+        || isDomainOrSubdomain(host, "youtube-nocookie.com"))
         return "youtube";
     if (hasHostLabel(host, "panopto"))
         return "panopto";
@@ -102,9 +107,12 @@ export function extractYouTubeVideoId(url) {
         const v = parsed.searchParams.get("v");
         if (v)
             return v;
-        const embedMatch = parsed.pathname.match(/\/embed\/([\w-]+)/);
-        if (embedMatch)
-            return embedMatch[1];
+        // /embed/, /shorts/, /live/ and the old /v/ all name the video in the path.
+        // Only /embed/ was handled, so a Shorts or a premiere link reported that no
+        // video id could be found.
+        const pathMatch = parsed.pathname.match(/\/(?:embed|shorts|live|v)\/([\w-]+)/);
+        if (pathMatch)
+            return pathMatch[1];
     }
     return null;
 }

package/build/utils/transcript/youtube.js

@@ -4,53 +4,166 @@
  * Licensed under MIT — see LICENSE file for details.
  */
 /**
- * YouTube transcript retrieval via its public timedtext endpoint. No
- * authentication is needed — this is the same caption track a signed-out
- * browser can fetch — so unlike Kaltura there is no session to reuse or fail
- * to obtain.
+ * YouTube transcript retrieval. Still no authentication: this is the caption
+ * track a signed-out viewer can read.
+ *
+ * It no longer uses `api/timedtext?type=list`. That endpoint now answers
+ * HTTP 200 with an empty body for every video, captioned or not, so listing
+ * tracks through it reported "this video has no captions" universally.
+ *
+ * The track list comes from YouTube's InnerTube player endpoint instead,
+ * asked as one of the mobile clients. The client identity matters: the signed
+ * caption URLs a browser page hands out are stamped `exp=xpe` and require a
+ * proof-of-origin token this server cannot mint, while the mobile clients'
+ * URLs are stamped `exp=xpo` and serve captions as-is. Both were measured;
+ * see the markers below.
  */
 import { parseCaptions } from "./captions.js";
 import { NoTranscriptError, TranscriptFetchError } from "./errors.js";
-const TIMEDTEXT_BASE = "https://www.youtube.com/api/timedtext";
-async function listCaptionTracks(videoId, fetchImpl) {
-    const res = await fetchImpl(`${TIMEDTEXT_BASE}?type=list&v=${encodeURIComponent(videoId)}`);
-    if (!res.ok) {
-        throw new TranscriptFetchError(`YouTube caption list request failed (HTTP ${res.status}).`);
+const PLAYER_ENDPOINT = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
+/**
+ * Client versions rot: YouTube eventually refuses one outright, and the
+ * symptom is a non-OK playabilityStatus for every video at once. Bump these.
+ */
+const CLIENTS = [
+    {
+        name: "ANDROID",
+        userAgent: "com.google.android.youtube/20.10.38 (Linux; U; Android 14) gzip",
+        context: { clientName: "ANDROID", clientVersion: "20.10.38", androidSdkVersion: 34, hl: "en", gl: "US" },
+    },
+    {
+        name: "IOS",
+        userAgent: "com.google.ios.youtube/21.26.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)",
+        context: {
+            clientName: "IOS", clientVersion: "21.26.4", deviceMake: "Apple", deviceModel: "iPhone16,2",
+            osName: "iPhone", osVersion: "18.3.2.22D82", hl: "en", gl: "US",
+        },
+    },
+];
+function trackLabel(track) {
+    return track.name?.runs?.[0]?.text ?? track.name?.simpleText;
+}
+/**
+ * What the player said about a video that will not play for us. These are
+ * distinct from "has no captions", and saying so is the whole point: a
+ * bot check or an age gate used to be reported as an absent transcript.
+ */
+function playabilityFailure(status, reason) {
+    const detail = reason ? ` YouTube said: ${reason}` : "";
+    if (status === "LOGIN_REQUIRED") {
+        return new TranscriptFetchError(`YouTube would not serve this video without a sign-in.${detail}`);
+    }
+    if (status === "UNPLAYABLE" || status === "ERROR") {
+        return new TranscriptFetchError(`YouTube reports this video as unavailable.${detail}`);
     }
-    const xml = await res.text();
-    return [...xml.matchAll(/<track\b[^>]*\blang_code="([^"]+)"/g)].map((m) => m[1]);
+    return new TranscriptFetchError(`YouTube refused to describe this video (${status}).${detail}`);
 }
-async function fetchTitle(videoId, fetchImpl) {
-    try {
-        const watchUrl = `https://www.youtube.com/watch?v=${encodeURIComponent(videoId)}`;
-        const res = await fetchImpl(`https://www.youtube.com/oembed?url=${encodeURIComponent(watchUrl)}&format=json`);
-        if (!res.ok)
-            return null;
-        const data = (await res.json());
-        return data.title ?? null;
-    }
-    catch {
-        return null;
+async function requestPlayer(videoId, fetchImpl) {
+    let lastError;
+    for (const client of CLIENTS) {
+        let parsed;
+        try {
+            const res = await fetchImpl(PLAYER_ENDPOINT, {
+                method: "POST",
+                headers: { "Content-Type": "application/json", "User-Agent": client.userAgent },
+                body: JSON.stringify({ videoId, contentCheckOk: true, racyCheckOk: true, context: { client: client.context } }),
+            });
+            if (!res.ok) {
+                lastError = new TranscriptFetchError(`YouTube player request failed (HTTP ${res.status}).`);
+                continue;
+            }
+            parsed = (await res.json());
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new TranscriptFetchError("YouTube player request failed.");
+            continue;
+        }
+        const status = parsed.playabilityStatus?.status;
+        // A client that has aged out answers non-OK for everything, so try the
+        // next one before believing what it says about the video.
+        if (status && status !== "OK") {
+            lastError = playabilityFailure(status, parsed.playabilityStatus?.reason);
+            continue;
+        }
+        return parsed;
     }
+    throw lastError ?? new TranscriptFetchError("YouTube did not describe this video.");
+}
+/**
+ * Which track to read. Auto-dubbed videos are the reason this is not simply
+ * "the first English one": dubbing adds *audio* tracks, all pointing back at
+ * the same caption list, and the dub a viewer hears decides which caption
+ * track YouTube shows by default. So the default audio track's own choice
+ * wins, then a manual track in the wanted language, then anything manual,
+ * then auto-generated. There is no transcript of synthesised dub speech --
+ * only the original language's captions, hence no attempt to find one.
+ */
+export function selectCaptionTrack(tracks, audio, defaultAudioIndex, preferred) {
+    if (tracks.length === 0)
+        return undefined;
+    const manual = (t) => t.kind !== "asr";
+    const wanted = (t) => (t.languageCode ?? "").toLowerCase().startsWith(preferred);
+    const chosen = audio?.[defaultAudioIndex ?? 0]?.defaultCaptionTrackIndex;
+    if (chosen !== undefined && tracks[chosen])
+        return tracks[chosen];
+    return tracks.find((t) => manual(t) && wanted(t))
+        ?? tracks.find(wanted)
+        ?? tracks.find(manual)
+        ?? tracks[0];
+}
+/**
+ * Ask a signed caption URL for WebVTT. `fmt` is already present on the URL
+ * (as `srv3`), and a second one is ignored, so it has to be replaced rather
+ * than appended. The signature covers only the parameters named in
+ * `sparams`, which `fmt` is not, so rewriting it keeps the URL valid.
+ */
+export function captionUrlAsVtt(baseUrl) {
+    return /[?&]fmt=/.test(baseUrl)
+        ? baseUrl.replace(/([?&])fmt=[^&]*/, "$1fmt=vtt")
+        : `${baseUrl}${baseUrl.includes("?") ? "&" : "?"}fmt=vtt`;
+}
+/** True for the browser-issued URLs that need a proof-of-origin token. */
+function needsProofOfOrigin(baseUrl) {
+    const exp = /[?&]exp=([^&]*)/.exec(baseUrl)?.[1] ?? "";
+    return exp.split(",").some((value) => value === "xpe" || value === "xpv");
 }
-export async function getYouTubeTranscript(videoId, fetchImpl) {
-    const tracks = await listCaptionTracks(videoId, fetchImpl);
+export async function getYouTubeTranscript(videoId, fetchImpl, language = "en") {
+    const player = await requestPlayer(videoId, fetchImpl);
+    const renderer = player.captions?.playerCaptionsTracklistRenderer;
+    const tracks = (renderer?.captionTracks ?? []).filter((t) => typeof t?.baseUrl === "string" && t.baseUrl);
     if (tracks.length === 0) {
         throw new NoTranscriptError("This YouTube video has no caption tracks available.");
     }
-    const lang = tracks.find((t) => t.startsWith("en")) ?? tracks[0];
-    const res = await fetchImpl(`${TIMEDTEXT_BASE}?lang=${encodeURIComponent(lang)}&v=${encodeURIComponent(videoId)}&fmt=vtt`);
+    const track = selectCaptionTrack(tracks, renderer?.audioTracks, renderer?.defaultAudioTrackIndex, language);
+    if (!track) {
+        throw new NoTranscriptError("This YouTube video has no caption tracks available.");
+    }
+    if (needsProofOfOrigin(track.baseUrl)) {
+        throw new TranscriptFetchError("YouTube issued this caption track behind a proof-of-origin check this server cannot answer, " +
+            "so the captions could not be downloaded. The video may still have captions.");
+    }
+    const res = await fetchImpl(captionUrlAsVtt(track.baseUrl));
     if (!res.ok) {
         throw new TranscriptFetchError(`YouTube caption download failed (HTTP ${res.status}).`);
     }
-    const vtt = await res.text();
-    if (!vtt.trim()) {
-        throw new NoTranscriptError("This YouTube video has no caption tracks available.");
+    const body = await res.text();
+    // A caption URL that answers 200 with nothing is a refusal, not an absence:
+    // the track was listed a moment ago. Saying "no captions" here is what sent
+    // users looking for a transcript that was there all along.
+    if (!body.trim()) {
+        throw new TranscriptFetchError("YouTube listed a caption track for this video but served an empty response for it. " +
+            "This is usually a temporary block rather than a missing transcript; try again.");
     }
-    const { format, cues } = parseCaptions(vtt);
+    const { format, cues } = parseCaptions(body);
     if (cues.length === 0) {
-        throw new NoTranscriptError("This YouTube video has a caption track, but it contained no readable cues.");
+        throw new TranscriptFetchError("YouTube served a caption track for this video in a format that could not be read.");
     }
-    const title = await fetchTitle(videoId, fetchImpl);
-    return { cues, format, language: lang, title, durationSeconds: null };
+    const lengthSeconds = Number(player.videoDetails?.lengthSeconds);
+    return {
+        cues,
+        format,
+        language: trackLabel(track) ?? track.languageCode ?? null,
+        title: player.videoDetails?.title ?? null,
+        durationSeconds: Number.isFinite(lengthSeconds) && lengthSeconds > 0 ? lengthSeconds : null,
+    };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "brightspace-mcp-server",
-  "version": "3.13.8",
+  "version": "3.13.9",
   "mcpName": "io.github.RohanMuppa/brightspace",
   "description": "MCP server for Brightspace (D2L). Check grades, due dates, assignments, announcements, syllabus, rosters and more via Claude, ChatGPT, Cursor, Windsurf, or any MCP client.",
   "type": "module",