@ivanbard/dd2l 0.0.0-stage → 0.1.1

package/src/pdf-worker.mjs

@@ -0,0 +1,26 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { getDocumentProxy, extractText, renderPageAsImage } from 'unpdf';
+
+let document;
+try {
+  document = await getDocumentProxy(new Uint8Array(workerData));
+  if (document.numPages > 250) throw new Error('page limit');
+  const { text: pageTexts, totalPages } = await extractText(document, { mergePages: false });
+  const images = [];
+  const links = [];
+  let ocrLimited = false;
+  for (let index = 0; index < pageTexts.length; index++) {
+    const page = await document.getPage(index + 1);
+    for (const annotation of await page.getAnnotations()) if (annotation.url && links.length < 200) links.push({ url: annotation.url, label: '', page: index + 1 });
+    if (pageTexts[index].trim()) continue;
+    if (!(await page.getOperatorList()).fnArray.length) continue;
+    const viewport = page.getViewport({ scale: 1.5 });
+    if (images.length === 10 || viewport.width * viewport.height > 16_000_000) { ocrLimited = true; continue; }
+    images.push({ index, bytes: await renderPageAsImage(document, index + 1, { scale: 1.5, canvasImport: () => import('@napi-rs/canvas') }) });
+  }
+  const text = pageTexts.join('\n\n');
+  parentPort.postMessage({ text: text.slice(0, 1_000_000), pageTexts: pageTexts.map(value => value.slice(0, 1_000_000)),
+    pages: totalPages, truncated: text.length > 1_000_000 || ocrLimited, ocrLimited, images, links });
+} catch {
+  parentPort.postMessage({ error: 'PDF text extraction failed or exceeded the 250-page limit; bytes remain available.' });
+} finally { await document?.loadingTask?.destroy(); }

package/src/research.mjs

@@ -0,0 +1,70 @@
+import { dirname, basename } from 'node:path';
+import { Connector } from './connector.mjs';
+import { BrightspaceClient, SCHOOL } from './client.mjs';
+import { Session } from './session.mjs';
+import { Store, lock } from './store.mjs';
+import { matches } from '../ui/answer-view.js';
+
+export const researchActions = { type: 'array', minItems: 0, maxItems: 2, items: { anyOf: [
+  { type: 'object', additionalProperties: false, required: ['action', 'course', 'section', 'query'], properties: {
+    action: { type: 'string', enum: ['browse'] }, course: { type: 'string', maxLength: 20 },
+    section: { type: 'string', enum: ['courses', 'content', 'assignments', 'quizzes', 'announcements', 'grades', 'calendar', 'overview', 'discussions'] },
+    query: { type: 'string', maxLength: 200 },
+  } },
+  { type: 'object', additionalProperties: false, required: ['action', 'resource'], properties: {
+    action: { type: 'string', enum: ['read'] }, resource: { type: 'string', maxLength: 24 },
+  } },
+] } };
+
+async function localBrowse(path, action, options) {
+  if (basename(path) !== 'inventory.sqlite') throw new Error('Live reading requires the standard inventory cache.');
+  const directory = dirname(path);
+  let release;
+  try { release = await lock(directory); }
+  catch { throw Object.assign(new Error('Another dd2l command owns the inventory. Use the UI opened by dd2l start to share its connector.'), { code: 'busy' }); }
+  let store;
+  try {
+    store = new Store(directory);
+    const origin = store.getMeta('school_origin') || SCHOOL;
+    const connector = new Connector(new BrightspaceClient(new Session(directory, { origin }), { origin }), store);
+    return await connector.browse(action, options);
+  } finally { store?.close(); await release(); }
+}
+
+// A request owns its discovered handles and budget; the model never supplies a network URL.
+export function createResearch({ connector, inventoryPath, scope = '', accountScope = '', signal }) {
+  const resources = new Map(), history = [], keys = new Set();
+  let available = true, used = 0;
+  return {
+    history, keys,
+    get available() { return available && used < 8; },
+    get remaining() { return Math.max(0, 8 - used); },
+    async execute(action) {
+      signal?.throwIfAborted();
+      if (!available || used >= 8) throw new Error('Live read budget is exhausted.');
+      if (!matches([action], researchActions)) throw new Error('Invalid Brightspace read request.');
+      used++;
+      let result;
+      try {
+        const options = { scope, accountScope, resources, signal };
+        result = connector ? await connector.browse(action, options) : await localBrowse(inventoryPath, action, options);
+        for (const key of result.resourceKeys || []) keys.add(key);
+        accountScope ||= result.accountScope || '';
+        const { resourceKeys, accountScope: account, ...safe } = result;
+        result = safe;
+      } catch (error) {
+        signal?.throwIfAborted();
+        const stopped = !['denied', 'restricted', 'invalid_link', 'unavailable', 'partial', 'external_auth_required', 'unsupported_link'].includes(error.code);
+        if (stopped) available = false;
+        result = { outcome: error.code || 'busy', message: error.code === 'needs_login'
+          ? 'School sign-in is required. Run setup again; cached sources may still be used.'
+          : error.code === 'retry_wait' ? 'Brightspace is temporarily unavailable or rate limited. Further live reads stopped.'
+            : error.code === 'busy' ? error.message
+              : !stopped ? error.message : 'Live reading is unavailable. Cached evidence may still be used.',
+          retryAfter: error.retryAfter || 0, resources: [] };
+      }
+      history.push({ action, result });
+      return result;
+    },
+  };
+}

package/src/session.mjs

@@ -0,0 +1,142 @@
+import { execFile } from 'node:child_process';
+import { readFile, writeFile, rename, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { AccessError, BrightspaceClient, SCHOOL, schoolOrigin, request, readJson } from './client.mjs';
+
+// Secrets go through stdin, never command arguments or a plaintext temp file.
+export function protect(bytes, decrypt = false) {
+  if (process.platform !== 'win32') throw new Error('Protected session storage currently requires Windows.');
+  const method = decrypt ? 'Unprotect' : 'Protect';
+  const script = `Add-Type -AssemblyName System.Security; try { $b=[Convert]::FromBase64String([Console]::In.ReadToEnd()); $r=[Security.Cryptography.ProtectedData]::${method}($b,$null,[Security.Cryptography.DataProtectionScope]::CurrentUser); [Console]::Out.Write([Convert]::ToBase64String($r)) } catch { exit 1 }`;
+  return new Promise((resolve, reject) => {
+    const child = execFile('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script],
+      { windowsHide: true, timeout: 30_000, maxBuffer: 8 * 1024 * 1024 }, (error, stdout) => {
+        if (error) reject(new Error('Windows could not protect or unlock this session. Sign in again if it belongs to another Windows account.'));
+        else resolve(Buffer.from(stdout.trim(), 'base64'));
+      });
+    child.stdin.on('error', () => {});
+    child.stdin.end(Buffer.from(bytes).toString('base64'));
+  });
+}
+
+export async function atomicWrite(path, value) {
+  const temp = `${path}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(temp, value, { mode: 0o600, flag: 'wx' });
+    await rename(temp, path);
+  } finally { await rm(temp, { force: true }); }
+}
+
+export class Session {
+  constructor(directory, { origin = SCHOOL, fetchImpl = fetch, crypt = protect } = {}) {
+    this.directory = directory;
+    this.path = join(directory, 'session.dpapi');
+    this.origin = schoolOrigin(origin);
+    this.fetch = fetchImpl;
+    this.crypt = crypt;
+  }
+
+  async load() {
+    if (this.value) return this.value;
+    let encrypted;
+    try { encrypted = await readFile(this.path); }
+    catch (error) {
+      if (error.code === 'ENOENT') throw new AccessError('needs_login', 'Run npx @ivanbard/dd2l setup (npm run setup from source) to sign in.');
+      throw error;
+    }
+    let value;
+    try { value = JSON.parse((await this.crypt(encrypted, true)).toString()); }
+    catch { throw new AccessError('needs_login', 'Saved session could not be unlocked. Run npx @ivanbard/dd2l setup (npm run setup from source).'); }
+    if (value.origin !== this.origin || !Array.isArray(value.cookies) || !value.csrf || !value.identity?.Identifier) {
+      throw new AccessError('needs_login', 'Saved session does not match this school or is incomplete.');
+    }
+    this.value = value;
+    return value;
+  }
+
+  async save(value) {
+    await mkdir(this.directory, { recursive: true });
+    await atomicWrite(this.path, await this.crypt(Buffer.from(JSON.stringify(value))));
+    this.value = value;
+  }
+
+  async token(rejected) {
+    const value = await this.load();
+    if (value.token && value.expiresAt > Date.now() + 60_000 && value.token !== rejected) return value.token;
+    if (!this.renewing) this.renewing = this.renew(value).finally(() => { this.renewing = null; });
+    return this.renewing;
+  }
+
+  async renew(value) {
+    const updated = await mintSession(this.origin, value, this.fetch);
+    await this.save(updated);
+    return updated.token;
+  }
+}
+
+async function mintSession(origin, value, fetchImpl) {
+    const url = new URL(origin);
+    const cookies = value.cookies.filter(cookie => {
+      const domain = String(cookie.domain).replace(/^\./, '');
+      return (url.hostname === domain || url.hostname.endsWith(`.${domain}`)) &&
+        (!cookie.expires || cookie.expires === -1 || cookie.expires > Date.now() / 1000) &&
+        (cookie.path === '/' || '/d2l/lp/auth/oauth2/token'.startsWith(cookie.path));
+    });
+    if (!cookies.length) throw new AccessError('needs_login', 'School session has expired. Run npx @ivanbard/dd2l setup (npm run setup from source).');
+    const payload = await readJson(await request(`${origin}/d2l/lp/auth/oauth2/token`, {
+      method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded',
+        cookie: cookies.map(c => `${c.name}=${c.value}`).join('; '), 'x-csrf-token': value.csrf },
+      body: 'scope=*:*:*',
+    }, fetchImpl));
+    if (typeof payload.access_token !== 'string' || !payload.access_token) throw new AccessError('invalid_response', 'Token exchange did not return an API token.');
+    let exp;
+    try { exp = JSON.parse(Buffer.from(payload.access_token.split('.')[1], 'base64url')).exp * 1000; } catch {}
+    const ttl = Number(payload.expires_in);
+    const expiresAt = Number.isFinite(exp) && exp > Date.now() ? exp : Date.now() + (ttl > 0 && Number.isFinite(ttl) ? ttl : 300) * 1000;
+    return { ...value, token: payload.access_token, expiresAt };
+}
+
+export async function login(session, client, { onProgress = console.error, expectedAccount } = {}) {
+  const { chromium } = await import('playwright-core');
+  const versions = await client.versions();
+  onProgress('Opening Chrome. Complete your school sign-in and MFA in the browser.');
+  const browser = await chromium.launch({ channel: 'chrome', headless: false, timeout: 30_000 });
+  try {
+    const context = await browser.newContext();
+    const page = await context.newPage();
+    await page.goto(`${session.origin}/d2l/home`, { waitUntil: 'domcontentloaded', timeout: 60_000 });
+    const deadline = Date.now() + 5 * 60_000;
+    while (Date.now() < deadline) {
+      if (page.isClosed()) throw new AccessError('needs_login', 'Sign-in browser was closed before verification.');
+      if (new URL(page.url()).origin === session.origin && /\/d2l\/(?:home|le\b|lms\b)/.test(new URL(page.url()).pathname)) {
+        let csrf;
+        try {
+          csrf = await page.evaluate(() =>
+            window.D2L?.LP?.Web?.Authentication?.Xsrf?.GetXsrfToken?.() ||
+            localStorage.getItem('XSRF.Token') || document.querySelector('meta[name="d2l-xsrf-token"]')?.content);
+        } catch { /* SSO navigation can replace the page during this read. */ }
+        if (typeof csrf === 'string' && csrf) {
+          const candidate = { origin: session.origin, csrf, cookies: await context.cookies(session.origin) };
+          try {
+            const verified = await mintSession(session.origin, candidate, session.fetch);
+            const candidateClient = new BrightspaceClient({ token: async () => verified.token }, { origin: session.origin, fetchImpl: session.fetch });
+            const identity = await candidateClient.get(`/d2l/api/lp/${versions.lp}/users/whoami`);
+            if (!identity?.Identifier) throw new AccessError('invalid_response', 'Could not verify the signed-in identity.');
+            if (expectedAccount && expectedAccount !== `${session.origin}:${identity.Identifier}`) {
+              throw new AccessError('account_mismatch', 'Inventory belongs to another school account. Run logout before switching accounts.');
+            }
+            // Verify before persisting; a failed login must not overwrite a working session.
+            await session.save({ ...verified, identity });
+            onProgress('School identity verified; session saved with Windows protection.');
+            return;
+          } catch (error) {
+            if (error.code !== 'needs_login') throw error;
+          }
+        }
+      }
+      await new Promise(resolve => setTimeout(resolve, 1500));
+    }
+    throw new AccessError('needs_login', 'Sign-in did not complete within five minutes.');
+  } finally { await browser.close(); }
+}

package/src/store.mjs

@@ -0,0 +1,108 @@
+import { DatabaseSync } from 'node:sqlite';
+import { mkdir, open, readFile, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+
+export function withoutCredentials(value) {
+  return JSON.parse(JSON.stringify(value, (key, item) => {
+    if (/^(?:access_token|refresh_token|authorization|cookie|csrfToken|xsrfToken|password|passcode)$/i.test(key)) return '[redacted]';
+    if (typeof item === 'string') return item.replace(/([?&](?:amp;)?(?:d2lSessionVal|access_token|refresh_token|csrf|xsrf|token)=)[^&\s"'<>]*/gi, '$1[redacted]');
+    return item;
+  }));
+}
+
+export async function lock(directory) {
+  await mkdir(directory, { recursive: true });
+  const path = join(directory, 'worker.lock');
+  const id = randomUUID();
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const handle = await open(path, 'wx', 0o600);
+      try { await handle.writeFile(JSON.stringify({ pid: process.pid, id })); } finally { await handle.close(); }
+      return async () => {
+        const current = JSON.parse(await readFile(path, 'utf8'));
+        if (current.id === id) await rm(path);
+      };
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      let owner;
+      try { owner = JSON.parse(await readFile(path, 'utf8')); } catch {}
+      if (!owner?.pid || attempt) throw new Error('Another dd2l command owns this data directory.');
+      try { process.kill(owner.pid, 0); }
+      catch (error) {
+        if (error.code === 'ESRCH') { await rm(path, { force: true }); continue; }
+      }
+      throw new Error('Another dd2l command is running. Stop it before starting this command.');
+    }
+  }
+}
+
+export class Store {
+  constructor(directory) {
+    this.path = join(directory, 'inventory.sqlite');
+    this.db = new DatabaseSync(this.path);
+    this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;
+      CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+      CREATE TABLE IF NOT EXISTS resources (
+        key TEXT PRIMARY KEY, course TEXT, category TEXT NOT NULL,
+        data TEXT, last_success TEXT, checked_at TEXT NOT NULL, outcome TEXT NOT NULL,
+        observation TEXT NOT NULL, last_source TEXT
+      );`);
+    // Preserve inventories created before source-window tracking was added.
+    if (!this.db.prepare('PRAGMA table_info(resources)').all().some(column => column.name === 'last_source')) {
+      this.db.exec('ALTER TABLE resources ADD COLUMN last_source TEXT');
+    }
+  }
+
+  getMeta(key) {
+    const row = this.db.prepare('SELECT value FROM meta WHERE key=?').get(key);
+    return row ? JSON.parse(row.value) : null;
+  }
+  meta(key, value) { this.db.prepare('INSERT OR REPLACE INTO meta VALUES (?, ?)').run(key, JSON.stringify(value)); }
+  resource(key) {
+    const row = this.db.prepare('SELECT * FROM resources WHERE key=?').get(key);
+    return row ? { ...row, data: row.data ? JSON.parse(row.data) : null, observation: JSON.parse(row.observation) } : null;
+  }
+
+  commit(identity, origin, observations, summary, enrollments) {
+    const account = `${origin}:${identity.Identifier}`;
+    const previous = this.getMeta('account');
+    if (previous && previous !== account) throw new Error('This data directory belongs to another school account. Sign out before switching accounts.');
+    const stamp = new Date().toISOString();
+    this.db.exec('BEGIN IMMEDIATE');
+    try {
+      this.meta('account', account);
+      this.meta('school_origin', origin);
+      for (const raw of observations) {
+        const observation = withoutCredentials(raw);
+        const successful = observation.outcome === 'available' && observation.complete !== false;
+        const cleared = ['denied', 'not_configured'].includes(observation.outcome);
+        const prior = this.db.prepare('SELECT data,last_success,last_source FROM resources WHERE key=?').get(observation.key);
+        this.db.prepare('INSERT OR REPLACE INTO resources (key,course,category,data,last_success,checked_at,outcome,observation,last_source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)').run(
+          observation.key, observation.course ?? null, observation.category,
+          successful ? JSON.stringify(observation.data) : cleared ? null : prior?.data ?? null,
+          successful ? observation.cached ? prior?.last_success ?? observation.retrievedAt : stamp : cleared ? null : prior?.last_success ?? null,
+          stamp, observation.outcome, JSON.stringify(successful ? { ...observation, data: undefined } : observation),
+          successful ? observation.source ?? null : cleared ? null : prior?.last_source ?? null);
+      }
+      // Only a complete, unfiltered enrollment read can establish loss of course access.
+      if (enrollments?.complete) {
+        const allowed = new Set(enrollments.data.filter(x => x.Access?.CanAccess !== false).map(x => String(x.OrgUnit.Id)));
+        for (const row of this.db.prepare('SELECT DISTINCT course FROM resources WHERE course IS NOT NULL').all()) {
+          if (!allowed.has(row.course)) this.db.prepare("UPDATE resources SET data=NULL,last_success=NULL,last_source=NULL,outcome='denied',observation=? WHERE course=?")
+            .run(JSON.stringify({ outcome: 'denied', message: 'Course access is no longer confirmed by complete enrollments.' }), row.course);
+        }
+      }
+      this.meta('last_run', summary);
+      this.meta('state', { state: summary.retryAfter ? 'retry_wait' : 'ready', updatedAt: stamp });
+      this.db.exec('COMMIT');
+    } catch (error) { this.db.exec('ROLLBACK'); throw error; }
+  }
+
+  status() {
+    return { state: this.getMeta('state'), lastRun: this.getMeta('last_run'),
+      resources: this.db.prepare('SELECT category,outcome,checked_at,last_success FROM resources ORDER BY category').all() };
+  }
+  clear() { this.db.exec('BEGIN; DELETE FROM resources; DELETE FROM meta; COMMIT;'); }
+  close() { this.db.close(); }
+}

package/src/syllabus.mjs

@@ -0,0 +1,191 @@
+import { AccessError, apiUrl } from './client.mjs';
+import { descriptionContent, readDocument } from './documents.mjs';
+import { readLinkedDocument, actionLink } from './links.mjs';
+
+const syllabusTitle = /syllab|course\s+(?:outline|overview|information|polic|guide)|\boutline\b|plan de cours/i;
+const explicitTitle = /syllab|course\s+outline|plan de cours/i;
+
+export function findSyllabusSources(tree, now = Date.now()) {
+  const sources = [];
+  function visit(node, parents = [], inSyllabusModule = false, ancestorUnavailable = false) {
+    if (!node || typeof node !== 'object') return;
+    const title = String(node.Title || '');
+    const description = descriptionContent(node.Description);
+    const directMatch = syllabusTitle.test(title);
+    const matched = directMatch || inSyllabusModule || explicitTitle.test(description.text);
+    const unavailable = ancestorUnavailable || node.IsHidden === true || node.IsLocked === true ||
+      Date.parse(node.StartDateTime) > now || Date.parse(node.EndDateTime) < now;
+    const common = { title, parents, topicId: node.TopicId, moduleId: node.ModuleId, activityType: node.ActivityType, lastModifiedDate: node.LastModifiedDate,
+      url: node.Url, unavailable, role: explicitTitle.test(title) ? 'explicit_syllabus' : 'course_information',
+      matchedBy: directMatch ? 'title' : inSyllabusModule ? 'parent_module' : 'description' };
+    if (matched && description.text) sources.push({ ...common, kind: node.TopicId != null ? 'topic_description' : 'module_description', description: description.text, links: description.links });
+    if (matched && node.TopicId != null) sources.push({ ...common, kind: node.ActivityType === 1 ? 'content_file' : 'linked_topic', description: '' });
+    for (const child of node.Modules || []) visit(child, [...parents, title].filter(Boolean), inSyllabusModule || directMatch, unavailable);
+    for (const child of node.Topics || []) visit(child, [...parents, title].filter(Boolean), inSyllabusModule || directMatch, unavailable);
+  }
+  visit(tree);
+  return sources;
+}
+
+export function classifyOverview(observation) {
+  if (observation.status === 404) {
+    observation.outcome = 'not_configured';
+    observation.complete = true;
+    observation.message = 'Course content overview is not configured; other syllabus locations still need checking.';
+    delete observation.error;
+  }
+  return observation;
+}
+
+export async function collectSyllabus(client, { le, courseId, content, overview }) {
+  const descriptors = content.data ? findSyllabusSources(content.data) : [];
+  const observations = [];
+  const catalog = [];
+  let documentReads = 0;
+  let limited = false;
+  const resolved = new Map();
+  const source = async (descriptor, action) => {
+    const identity = descriptor.kind === 'overview' ? 'overview' : descriptor.kind === 'overview_attachment' ? 'overview_attachment' :
+      `${descriptor.kind}:${descriptor.kind === 'linked_document' ? descriptor.url : descriptor.topicId ?? descriptor.moduleId ?? JSON.stringify([...(descriptor.parents || []), descriptor.title])}`;
+    const key = `syllabus:${courseId}:${identity}`;
+    const record = { key, category: 'syllabus_source', course: courseId, source: descriptor.source || `${le}/content/toc`,
+      retrievedAt: new Date().toISOString(), outcome: 'available', complete: true };
+    let entry = { ...descriptor, key, retrievedAt: record.retrievedAt };
+    delete entry.description;
+    try {
+      if (descriptor.unavailable) throw new AccessError('restricted', 'Source is hidden, locked, or outside its release window.');
+      const data = await action();
+      const role = descriptor.kind === 'overview_attachment' && explicitTitle.test(data.text?.slice(0, 2000) || '') ? 'explicit_syllabus' : descriptor.role;
+      record.data = { ...descriptor, ...data, role };
+      entry = { ...entry, outcome: 'available', textStatus: data.textStatus, textCharacters: data.text?.trim().length || 0, format: data.format,
+        pages: data.pages, links: data.links || [], linksLimited: data.linksLimited, source: record.source, role };
+      if (data.textStatus === 'failed' || data.textStatus === 'partial') {
+        record.complete = false;
+        record.outcome = 'partial';
+        entry.outcome = 'partial';
+      }
+    } catch (error) {
+      if (!(error instanceof AccessError) || error.code === 'needs_login') throw error;
+      if (error.status === 429) throw error;
+      record.outcome = ['denied', 'restricted'].includes(error.code) ? 'denied' : error.code === 'partial' ? 'partial' : 'failed';
+      Object.assign(record, { complete: false, error: error.code, message: error.message, status: error.status, retryAfter: error.retryAfter });
+      entry = { ...entry, outcome: record.outcome, error: error.code, message: error.message, textCharacters: 0 };
+    }
+    observations.push(record);
+    catalog.push(entry);
+    if (entry.textStatus === 'available') {
+      for (const candidate of [descriptor.url, record.source]) if (candidate) {
+        try { resolved.set(new URL(candidate, client.origin).href.split('#')[0], entry.key); } catch {}
+      }
+    }
+  };
+
+  if (overview.outcome === 'not_configured') {
+    for (const kind of ['overview', 'overview_attachment']) {
+      observations.push({ key: `syllabus:${courseId}:${kind}`, category: 'syllabus_source', course: courseId,
+        source: kind === 'overview' ? `${le}/overview` : `${le}/overview/attachment`, outcome: 'not_configured', complete: true,
+        retrievedAt: new Date().toISOString(), status: 404 });
+    }
+  } else if (overview.outcome === 'available') {
+    await source({ kind: 'overview', title: 'Course overview', role: 'course_information', source: `${le}/overview` }, async () => {
+      const value = descriptionContent(overview.data?.Description);
+      return { ...value, format: 'inline', textStatus: value.text ? 'available' : 'empty' };
+    });
+    if (overview.data?.HasAttachment === true) {
+      documentReads++;
+      await source({ kind: 'overview_attachment', title: 'Course overview attachment', role: 'course_information', source: `${le}/overview/attachment` }, () => readDocument(client, `${le}/overview/attachment`));
+    }
+  }
+
+  for (const descriptor of descriptors.slice(0, 100)) {
+    if (descriptor.unavailable) {
+      await source(descriptor, async () => ({}));
+      continue;
+    }
+    if (descriptor.kind.endsWith('_description')) {
+      await source(descriptor, async () => ({ text: descriptor.description, links: descriptor.links || [], format: 'inline', textStatus: descriptor.description ? 'available' : 'empty' }));
+      continue;
+    }
+    if (!/^\d+$/.test(String(descriptor.topicId))) {
+      catalog.push({ ...descriptor, outcome: 'unresolved', textCharacters: 0 });
+      continue;
+    }
+    const topicPath = `${le}/content/topics/${descriptor.topicId}`;
+    if (descriptor.kind === 'content_file') {
+      if (documentReads >= 20) { limited = true; catalog.push({ ...descriptor, outcome: 'not_read_limit', textCharacters: 0 }); continue; }
+      documentReads++;
+      await source({ ...descriptor, source: `${topicPath}/file` }, () => readDocument(client, `${topicPath}/file`));
+    } else {
+      await source({ ...descriptor, source: topicPath }, async () => {
+        const metadata = await client.get(topicPath);
+        const value = descriptionContent(metadata.Description);
+        const target = metadata.Url || descriptor.url;
+        // Link targets are handled below with separate school/public trust boundaries.
+        let linkStatus = 'unresolved';
+        if (target) {
+          try { apiUrl(client.origin, target); linkStatus = 'api_link_not_followed'; }
+          catch {
+            try { linkStatus = new URL(target, client.origin).origin === client.origin ? 'school_link_not_followed' : 'external'; }
+            catch { linkStatus = 'invalid'; }
+          }
+        }
+        return { ...value, format: 'link', metadata, links: target ? [...value.links, { url: target, label: descriptor.title, status: linkStatus }] : value.links,
+          textStatus: value.text ? 'available' : target && actionLink(target, client.origin) ? 'not_applicable' : 'unresolved' };
+      });
+    }
+  }
+  // Breadth-first, deduplicated document links; never activate surveys or enrollment links.
+  const seen = new Set();
+  for (let index = 0; index < catalog.length && index < 200; index++) {
+    const entry = catalog[index];
+    for (const link of entry.links || []) {
+      if (actionLink(link.url, client.origin)) { link.status = 'not_content'; continue; }
+      let target;
+      try { target = new URL(link.url, entry.url ? new URL(entry.url, client.origin) : client.origin); target.hash = ''; }
+      catch { link.status = 'invalid'; continue; }
+      // Read source documents, not an unbounded crawl of their citations and site navigation.
+      if (target.origin !== client.origin && ((entry.linkDepth || 0) > 0 ||
+          (entry.format === 'pdf' && !/\.(?:pdf|docx|pptx|xlsx)$/i.test(target.pathname) && !syllabusTitle.test(link.label || '')))) {
+        link.status = 'reference';
+        continue;
+      }
+      if (resolved.has(target.href)) { link.status = 'read'; link.sourceKey = resolved.get(target.href); continue; }
+      if (seen.has(target.href)) { link.status = 'already_attempted'; continue; }
+      seen.add(target.href);
+      if ((entry.linkDepth || 0) >= 2 || documentReads >= 20) { limited = true; link.status = 'not_read_limit'; continue; }
+      documentReads++;
+      await source({ kind: 'linked_document', title: link.label || 'Linked course document', url: target.href,
+        parents: [...(entry.parents || []), entry.title].filter(Boolean), role: 'course_information',
+        source: target.href, linkDepth: (entry.linkDepth || 0) + 1 }, () => readLinkedDocument(client, target.href, { le, courseId, tree: content.data }));
+      const linked = catalog.at(-1);
+      link.status = linked.textStatus === 'available' ? 'read' : linked.error || linked.textStatus || linked.outcome;
+      link.sourceKey = linked.key;
+      if (link.status === 'read') resolved.set(target.href, linked.key);
+    }
+    if (entry.kind === 'linked_topic' && entry.textStatus === 'unresolved' && entry.links?.length &&
+        entry.links.every(link => ['read', 'not_content', 'reference'].includes(link.status))) {
+      entry.textStatus = 'linked';
+      const observation = observations.find(item => item.key === entry.key);
+      if (observation?.data) observation.data.textStatus = 'linked';
+    }
+  }
+  limited ||= descriptors.length > 100 || catalog.some(item => item.linksLimited);
+  const readable = catalog.filter(item => item.textStatus === 'available' && item.textCharacters > 0);
+  const explicit = readable.filter(item => item.role === 'explicit_syllabus');
+  const discoveryComplete = content.complete === true && overview.complete === true && !limited;
+  const unreadLinkCount = catalog.reduce((sum, item) => sum + (item.links?.filter(link => !['read', 'not_content', 'reference'].includes(link.status)).length || 0), 0);
+  const textCoverageComplete = discoveryComplete && !unreadLinkCount && catalog.every(item => item.outcome === 'available' && ['available', 'empty', 'not_applicable', 'linked'].includes(item.textStatus));
+  const summary = {
+    result: explicit.length ? 'explicit_sources' : readable.length ? 'course_information' : 'unresolved',
+    overview: overview.outcome === 'available' ? descriptionContent(overview.data?.Description).text ? 'readable' : 'empty' : overview.outcome,
+    candidateCount: descriptors.length, readableSourceCount: readable.length, explicitSourceCount: explicit.length,
+    textCharacters: readable.reduce((sum, item) => sum + item.textCharacters, 0),
+    discoveryComplete, textCoverageComplete, limited, unreadLinkCount,
+    unresolvedSourceCount: catalog.filter(item => item.outcome !== 'available' || !['available', 'empty', 'not_applicable', 'linked'].includes(item.textStatus)).length,
+    formats: [...new Set(readable.map(item => item.format))].sort(),
+    retryAfter: Math.max(0, ...observations.filter(item => item.error === 'retry_wait').map(item => item.retryAfter || 60_000)),
+  };
+  observations.push({ key: `${le}/syllabus`, category: 'syllabus', course: courseId, source: `${le}/content/toc`,
+    retrievedAt: new Date().toISOString(), outcome: 'available', complete: discoveryComplete, data: { ...summary, sources: catalog } });
+  return { observations, summary };
+}