@ivanbard/dd2l 0.1.0 → 0.1.2

package/src/store.mjs

@@ -1,106 +1,108 @@
-import { DatabaseSync } from 'node:sqlite';
-import { mkdir, open, readFile, rm } from 'node:fs/promises';
-import { join } from 'node:path';
-import { randomUUID } from 'node:crypto';
-
-export function withoutCredentials(value) {
-  return JSON.parse(JSON.stringify(value, (key, item) => {
-    if (/^(?:access_token|refresh_token|authorization|cookie|csrfToken|xsrfToken|password|passcode)$/i.test(key)) return '[redacted]';
-    if (typeof item === 'string') return item.replace(/([?&](?:amp;)?(?:d2lSessionVal|access_token|refresh_token|csrf|xsrf|token)=)[^&\s"'<>]*/gi, '$1[redacted]');
-    return item;
-  }));
-}
-
-export async function lock(directory) {
-  await mkdir(directory, { recursive: true });
-  const path = join(directory, 'worker.lock');
-  const id = randomUUID();
-  for (let attempt = 0; attempt < 2; attempt++) {
-    try {
-      const handle = await open(path, 'wx', 0o600);
-      try { await handle.writeFile(JSON.stringify({ pid: process.pid, id })); } finally { await handle.close(); }
-      return async () => {
-        const current = JSON.parse(await readFile(path, 'utf8'));
-        if (current.id === id) await rm(path);
-      };
-    } catch (error) {
-      if (error.code !== 'EEXIST') throw error;
-      let owner;
-      try { owner = JSON.parse(await readFile(path, 'utf8')); } catch {}
-      if (!owner?.pid || attempt) throw new Error('Another dd2l command owns this data directory.');
-      try { process.kill(owner.pid, 0); }
-      catch (error) {
-        if (error.code === 'ESRCH') { await rm(path, { force: true }); continue; }
-      }
-      throw new Error('Another dd2l command is running. Stop it before starting this command.');
-    }
-  }
-}
-
-export class Store {
-  constructor(directory) {
-    this.db = new DatabaseSync(join(directory, 'inventory.sqlite'));
-    this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;
-      CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
-      CREATE TABLE IF NOT EXISTS resources (
-        key TEXT PRIMARY KEY, course TEXT, category TEXT NOT NULL,
-        data TEXT, last_success TEXT, checked_at TEXT NOT NULL, outcome TEXT NOT NULL,
-        observation TEXT NOT NULL, last_source TEXT
-      );`);
-    // Preserve inventories created before source-window tracking was added.
-    if (!this.db.prepare('PRAGMA table_info(resources)').all().some(column => column.name === 'last_source')) {
-      this.db.exec('ALTER TABLE resources ADD COLUMN last_source TEXT');
-    }
-  }
-
-  getMeta(key) {
-    const row = this.db.prepare('SELECT value FROM meta WHERE key=?').get(key);
-    return row ? JSON.parse(row.value) : null;
-  }
-  meta(key, value) { this.db.prepare('INSERT OR REPLACE INTO meta VALUES (?, ?)').run(key, JSON.stringify(value)); }
-  resource(key) {
-    const row = this.db.prepare('SELECT * FROM resources WHERE key=?').get(key);
-    return row ? { ...row, data: row.data ? JSON.parse(row.data) : null, observation: JSON.parse(row.observation) } : null;
-  }
-
-  commit(identity, origin, observations, summary, enrollments) {
-    const account = `${origin}:${identity.Identifier}`;
-    const previous = this.getMeta('account');
-    if (previous && previous !== account) throw new Error('This data directory belongs to another school account. Sign out before switching accounts.');
-    const stamp = new Date().toISOString();
-    this.db.exec('BEGIN IMMEDIATE');
-    try {
-      this.meta('account', account);
-      for (const raw of observations) {
-        const observation = withoutCredentials(raw);
-        const successful = observation.outcome === 'available' && observation.complete !== false;
-        const cleared = ['denied', 'not_configured'].includes(observation.outcome);
-        const prior = this.db.prepare('SELECT data,last_success,last_source FROM resources WHERE key=?').get(observation.key);
-        this.db.prepare('INSERT OR REPLACE INTO resources (key,course,category,data,last_success,checked_at,outcome,observation,last_source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)').run(
-          observation.key, observation.course ?? null, observation.category,
-          successful ? JSON.stringify(observation.data) : cleared ? null : prior?.data ?? null,
-          successful ? observation.cached ? prior?.last_success ?? observation.retrievedAt : stamp : cleared ? null : prior?.last_success ?? null,
-          stamp, observation.outcome, JSON.stringify(successful ? { ...observation, data: undefined } : observation),
-          successful ? observation.source ?? null : cleared ? null : prior?.last_source ?? null);
-      }
-      // Only a complete, unfiltered enrollment read can establish loss of course access.
-      if (enrollments?.complete) {
-        const allowed = new Set(enrollments.data.filter(x => x.Access?.CanAccess !== false).map(x => String(x.OrgUnit.Id)));
-        for (const row of this.db.prepare('SELECT DISTINCT course FROM resources WHERE course IS NOT NULL').all()) {
-          if (!allowed.has(row.course)) this.db.prepare("UPDATE resources SET data=NULL,last_success=NULL,last_source=NULL,outcome='denied',observation=? WHERE course=?")
-            .run(JSON.stringify({ outcome: 'denied', message: 'Course access is no longer confirmed by complete enrollments.' }), row.course);
-        }
-      }
-      this.meta('last_run', summary);
-      this.meta('state', { state: summary.retryAfter ? 'retry_wait' : 'ready', updatedAt: stamp });
-      this.db.exec('COMMIT');
-    } catch (error) { this.db.exec('ROLLBACK'); throw error; }
-  }
-
-  status() {
-    return { state: this.getMeta('state'), lastRun: this.getMeta('last_run'),
-      resources: this.db.prepare('SELECT category,outcome,checked_at,last_success FROM resources ORDER BY category').all() };
-  }
-  clear() { this.db.exec('BEGIN; DELETE FROM resources; DELETE FROM meta; COMMIT;'); }
-  close() { this.db.close(); }
-}
+import { DatabaseSync } from 'node:sqlite';
+import { mkdir, open, readFile, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+
+export function withoutCredentials(value) {
+  return JSON.parse(JSON.stringify(value, (key, item) => {
+    if (/^(?:access_token|refresh_token|authorization|cookie|csrfToken|xsrfToken|password|passcode)$/i.test(key)) return '[redacted]';
+    if (typeof item === 'string') return item.replace(/([?&](?:amp;)?(?:d2lSessionVal|access_token|refresh_token|csrf|xsrf|token)=)[^&\s"'<>]*/gi, '$1[redacted]');
+    return item;
+  }));
+}
+
+export async function lock(directory) {
+  await mkdir(directory, { recursive: true });
+  const path = join(directory, 'worker.lock');
+  const id = randomUUID();
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const handle = await open(path, 'wx', 0o600);
+      try { await handle.writeFile(JSON.stringify({ pid: process.pid, id })); } finally { await handle.close(); }
+      return async () => {
+        const current = JSON.parse(await readFile(path, 'utf8'));
+        if (current.id === id) await rm(path);
+      };
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      let owner;
+      try { owner = JSON.parse(await readFile(path, 'utf8')); } catch {}
+      if (!owner?.pid || attempt) throw new Error('Another dd2l command owns this data directory.');
+      try { process.kill(owner.pid, 0); }
+      catch (error) {
+        if (error.code === 'ESRCH') { await rm(path, { force: true }); continue; }
+      }
+      throw new Error('Another dd2l command is running. Stop it before starting this command.');
+    }
+  }
+}
+
+export class Store {
+  constructor(directory) {
+    this.path = join(directory, 'inventory.sqlite');
+    this.db = new DatabaseSync(this.path);
+    this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;
+      CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+      CREATE TABLE IF NOT EXISTS resources (
+        key TEXT PRIMARY KEY, course TEXT, category TEXT NOT NULL,
+        data TEXT, last_success TEXT, checked_at TEXT NOT NULL, outcome TEXT NOT NULL,
+        observation TEXT NOT NULL, last_source TEXT
+      );`);
+    // Preserve inventories created before source-window tracking was added.
+    if (!this.db.prepare('PRAGMA table_info(resources)').all().some(column => column.name === 'last_source')) {
+      this.db.exec('ALTER TABLE resources ADD COLUMN last_source TEXT');
+    }
+  }
+
+  getMeta(key) {
+    const row = this.db.prepare('SELECT value FROM meta WHERE key=?').get(key);
+    return row ? JSON.parse(row.value) : null;
+  }
+  meta(key, value) { this.db.prepare('INSERT OR REPLACE INTO meta VALUES (?, ?)').run(key, JSON.stringify(value)); }
+  resource(key) {
+    const row = this.db.prepare('SELECT * FROM resources WHERE key=?').get(key);
+    return row ? { ...row, data: row.data ? JSON.parse(row.data) : null, observation: JSON.parse(row.observation) } : null;
+  }
+
+  commit(identity, origin, observations, summary, enrollments) {
+    const account = `${origin}:${identity.Identifier}`;
+    const previous = this.getMeta('account');
+    if (previous && previous !== account) throw new Error('This data directory belongs to another school account. Sign out before switching accounts.');
+    const stamp = new Date().toISOString();
+    this.db.exec('BEGIN IMMEDIATE');
+    try {
+      this.meta('account', account);
+      this.meta('school_origin', origin);
+      for (const raw of observations) {
+        const observation = withoutCredentials(raw);
+        const successful = observation.outcome === 'available' && observation.complete !== false;
+        const cleared = ['denied', 'not_configured'].includes(observation.outcome);
+        const prior = this.db.prepare('SELECT data,last_success,last_source FROM resources WHERE key=?').get(observation.key);
+        this.db.prepare('INSERT OR REPLACE INTO resources (key,course,category,data,last_success,checked_at,outcome,observation,last_source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)').run(
+          observation.key, observation.course ?? null, observation.category,
+          successful ? JSON.stringify(observation.data) : cleared ? null : prior?.data ?? null,
+          successful ? observation.cached ? prior?.last_success ?? observation.retrievedAt : stamp : cleared ? null : prior?.last_success ?? null,
+          stamp, observation.outcome, JSON.stringify(successful ? { ...observation, data: undefined } : observation),
+          successful ? observation.source ?? null : cleared ? null : prior?.last_source ?? null);
+      }
+      // Only a complete, unfiltered enrollment read can establish loss of course access.
+      if (enrollments?.complete) {
+        const allowed = new Set(enrollments.data.filter(x => x.Access?.CanAccess !== false).map(x => String(x.OrgUnit.Id)));
+        for (const row of this.db.prepare('SELECT DISTINCT course FROM resources WHERE course IS NOT NULL').all()) {
+          if (!allowed.has(row.course)) this.db.prepare("UPDATE resources SET data=NULL,last_success=NULL,last_source=NULL,outcome='denied',observation=? WHERE course=?")
+            .run(JSON.stringify({ outcome: 'denied', message: 'Course access is no longer confirmed by complete enrollments.' }), row.course);
+        }
+      }
+      this.meta('last_run', summary);
+      this.meta('state', { state: summary.retryAfter ? 'retry_wait' : 'ready', updatedAt: stamp });
+      this.db.exec('COMMIT');
+    } catch (error) { this.db.exec('ROLLBACK'); throw error; }
+  }
+
+  status() {
+    return { state: this.getMeta('state'), lastRun: this.getMeta('last_run'),
+      resources: this.db.prepare('SELECT category,outcome,checked_at,last_success FROM resources ORDER BY category').all() };
+  }
+  clear() { this.db.exec('BEGIN; DELETE FROM resources; DELETE FROM meta; COMMIT;'); }
+  close() { this.db.close(); }
+}

package/src/syllabus.mjs

@@ -1,191 +1,191 @@
-import { AccessError, apiUrl } from './client.mjs';
-import { descriptionContent, readDocument } from './documents.mjs';
-import { readLinkedDocument, actionLink } from './links.mjs';
-
-const syllabusTitle = /syllab|course\s+(?:outline|overview|information|polic|guide)|\boutline\b|plan de cours/i;
-const explicitTitle = /syllab|course\s+outline|plan de cours/i;
-
-export function findSyllabusSources(tree, now = Date.now()) {
-  const sources = [];
-  function visit(node, parents = [], inSyllabusModule = false, ancestorUnavailable = false) {
-    if (!node || typeof node !== 'object') return;
-    const title = String(node.Title || '');
-    const description = descriptionContent(node.Description);
-    const directMatch = syllabusTitle.test(title);
-    const matched = directMatch || inSyllabusModule || explicitTitle.test(description.text);
-    const unavailable = ancestorUnavailable || node.IsHidden === true || node.IsLocked === true ||
-      Date.parse(node.StartDateTime) > now || Date.parse(node.EndDateTime) < now;
-    const common = { title, parents, topicId: node.TopicId, moduleId: node.ModuleId, activityType: node.ActivityType, lastModifiedDate: node.LastModifiedDate,
-      url: node.Url, unavailable, role: explicitTitle.test(title) ? 'explicit_syllabus' : 'course_information',
-      matchedBy: directMatch ? 'title' : inSyllabusModule ? 'parent_module' : 'description' };
-    if (matched && description.text) sources.push({ ...common, kind: node.TopicId != null ? 'topic_description' : 'module_description', description: description.text, links: description.links });
-    if (matched && node.TopicId != null) sources.push({ ...common, kind: node.ActivityType === 1 ? 'content_file' : 'linked_topic', description: '' });
-    for (const child of node.Modules || []) visit(child, [...parents, title].filter(Boolean), inSyllabusModule || directMatch, unavailable);
-    for (const child of node.Topics || []) visit(child, [...parents, title].filter(Boolean), inSyllabusModule || directMatch, unavailable);
-  }
-  visit(tree);
-  return sources;
-}
-
-export function classifyOverview(observation) {
-  if (observation.status === 404) {
-    observation.outcome = 'not_configured';
-    observation.complete = true;
-    observation.message = 'Course content overview is not configured; other syllabus locations still need checking.';
-    delete observation.error;
-  }
-  return observation;
-}
-
-export async function collectSyllabus(client, { le, courseId, content, overview }) {
-  const descriptors = content.data ? findSyllabusSources(content.data) : [];
-  const observations = [];
-  const catalog = [];
-  let documentReads = 0;
-  let limited = false;
-  const resolved = new Map();
-  const source = async (descriptor, action) => {
-    const identity = descriptor.kind === 'overview' ? 'overview' : descriptor.kind === 'overview_attachment' ? 'overview_attachment' :
-      `${descriptor.kind}:${descriptor.kind === 'linked_document' ? descriptor.url : descriptor.topicId ?? descriptor.moduleId ?? JSON.stringify([...(descriptor.parents || []), descriptor.title])}`;
-    const key = `syllabus:${courseId}:${identity}`;
-    const record = { key, category: 'syllabus_source', course: courseId, source: descriptor.source || `${le}/content/toc`,
-      retrievedAt: new Date().toISOString(), outcome: 'available', complete: true };
-    let entry = { ...descriptor, key, retrievedAt: record.retrievedAt };
-    delete entry.description;
-    try {
-      if (descriptor.unavailable) throw new AccessError('restricted', 'Source is hidden, locked, or outside its release window.');
-      const data = await action();
-      const role = descriptor.kind === 'overview_attachment' && explicitTitle.test(data.text?.slice(0, 2000) || '') ? 'explicit_syllabus' : descriptor.role;
-      record.data = { ...descriptor, ...data, role };
-      entry = { ...entry, outcome: 'available', textStatus: data.textStatus, textCharacters: data.text?.trim().length || 0, format: data.format,
-        pages: data.pages, links: data.links || [], linksLimited: data.linksLimited, source: record.source, role };
-      if (data.textStatus === 'failed' || data.textStatus === 'partial') {
-        record.complete = false;
-        record.outcome = 'partial';
-        entry.outcome = 'partial';
-      }
-    } catch (error) {
-      if (!(error instanceof AccessError) || error.code === 'needs_login') throw error;
-      if (error.status === 429) throw error;
-      record.outcome = ['denied', 'restricted'].includes(error.code) ? 'denied' : error.code === 'partial' ? 'partial' : 'failed';
-      Object.assign(record, { complete: false, error: error.code, message: error.message, status: error.status, retryAfter: error.retryAfter });
-      entry = { ...entry, outcome: record.outcome, error: error.code, message: error.message, textCharacters: 0 };
-    }
-    observations.push(record);
-    catalog.push(entry);
-    if (entry.textStatus === 'available') {
-      for (const candidate of [descriptor.url, record.source]) if (candidate) {
-        try { resolved.set(new URL(candidate, client.origin).href.split('#')[0], entry.key); } catch {}
-      }
-    }
-  };
-
-  if (overview.outcome === 'not_configured') {
-    for (const kind of ['overview', 'overview_attachment']) {
-      observations.push({ key: `syllabus:${courseId}:${kind}`, category: 'syllabus_source', course: courseId,
-        source: kind === 'overview' ? `${le}/overview` : `${le}/overview/attachment`, outcome: 'not_configured', complete: true,
-        retrievedAt: new Date().toISOString(), status: 404 });
-    }
-  } else if (overview.outcome === 'available') {
-    await source({ kind: 'overview', title: 'Course overview', role: 'course_information', source: `${le}/overview` }, async () => {
-      const value = descriptionContent(overview.data?.Description);
-      return { ...value, format: 'inline', textStatus: value.text ? 'available' : 'empty' };
-    });
-    if (overview.data?.HasAttachment === true) {
-      documentReads++;
-      await source({ kind: 'overview_attachment', title: 'Course overview attachment', role: 'course_information', source: `${le}/overview/attachment` }, () => readDocument(client, `${le}/overview/attachment`));
-    }
-  }
-
-  for (const descriptor of descriptors.slice(0, 100)) {
-    if (descriptor.unavailable) {
-      await source(descriptor, async () => ({}));
-      continue;
-    }
-    if (descriptor.kind.endsWith('_description')) {
-      await source(descriptor, async () => ({ text: descriptor.description, links: descriptor.links || [], format: 'inline', textStatus: descriptor.description ? 'available' : 'empty' }));
-      continue;
-    }
-    if (!/^\d+$/.test(String(descriptor.topicId))) {
-      catalog.push({ ...descriptor, outcome: 'unresolved', textCharacters: 0 });
-      continue;
-    }
-    const topicPath = `${le}/content/topics/${descriptor.topicId}`;
-    if (descriptor.kind === 'content_file') {
-      if (documentReads >= 20) { limited = true; catalog.push({ ...descriptor, outcome: 'not_read_limit', textCharacters: 0 }); continue; }
-      documentReads++;
-      await source({ ...descriptor, source: `${topicPath}/file` }, () => readDocument(client, `${topicPath}/file`));
-    } else {
-      await source({ ...descriptor, source: topicPath }, async () => {
-        const metadata = await client.get(topicPath);
-        const value = descriptionContent(metadata.Description);
-        const target = metadata.Url || descriptor.url;
-        // Link targets are handled below with separate school/public trust boundaries.
-        let linkStatus = 'unresolved';
-        if (target) {
-          try { apiUrl(client.origin, target); linkStatus = 'api_link_not_followed'; }
-          catch {
-            try { linkStatus = new URL(target, client.origin).origin === client.origin ? 'school_link_not_followed' : 'external'; }
-            catch { linkStatus = 'invalid'; }
-          }
-        }
-        return { ...value, format: 'link', metadata, links: target ? [...value.links, { url: target, label: descriptor.title, status: linkStatus }] : value.links,
-          textStatus: value.text ? 'available' : target && actionLink(target, client.origin) ? 'not_applicable' : 'unresolved' };
-      });
-    }
-  }
-  // Breadth-first, deduplicated document links; never activate surveys or enrollment links.
-  const seen = new Set();
-  for (let index = 0; index < catalog.length && index < 200; index++) {
-    const entry = catalog[index];
-    for (const link of entry.links || []) {
-      if (actionLink(link.url, client.origin)) { link.status = 'not_content'; continue; }
-      let target;
-      try { target = new URL(link.url, entry.url ? new URL(entry.url, client.origin) : client.origin); target.hash = ''; }
-      catch { link.status = 'invalid'; continue; }
-      // Read source documents, not an unbounded crawl of their citations and site navigation.
-      if (target.origin !== client.origin && ((entry.linkDepth || 0) > 0 ||
-          (entry.format === 'pdf' && !/\.(?:pdf|docx|pptx|xlsx)$/i.test(target.pathname) && !syllabusTitle.test(link.label || '')))) {
-        link.status = 'reference';
-        continue;
-      }
-      if (resolved.has(target.href)) { link.status = 'read'; link.sourceKey = resolved.get(target.href); continue; }
-      if (seen.has(target.href)) { link.status = 'already_attempted'; continue; }
-      seen.add(target.href);
-      if ((entry.linkDepth || 0) >= 2 || documentReads >= 20) { limited = true; link.status = 'not_read_limit'; continue; }
-      documentReads++;
-      await source({ kind: 'linked_document', title: link.label || 'Linked course document', url: target.href,
-        parents: [...(entry.parents || []), entry.title].filter(Boolean), role: 'course_information',
-        source: target.href, linkDepth: (entry.linkDepth || 0) + 1 }, () => readLinkedDocument(client, target.href, { le, courseId, tree: content.data }));
-      const linked = catalog.at(-1);
-      link.status = linked.textStatus === 'available' ? 'read' : linked.error || linked.textStatus || linked.outcome;
-      link.sourceKey = linked.key;
-      if (link.status === 'read') resolved.set(target.href, linked.key);
-    }
-    if (entry.kind === 'linked_topic' && entry.textStatus === 'unresolved' && entry.links?.length &&
-        entry.links.every(link => ['read', 'not_content', 'reference'].includes(link.status))) {
-      entry.textStatus = 'linked';
-      const observation = observations.find(item => item.key === entry.key);
-      if (observation?.data) observation.data.textStatus = 'linked';
-    }
-  }
-  limited ||= descriptors.length > 100 || catalog.some(item => item.linksLimited);
-  const readable = catalog.filter(item => item.textStatus === 'available' && item.textCharacters > 0);
-  const explicit = readable.filter(item => item.role === 'explicit_syllabus');
-  const discoveryComplete = content.complete === true && overview.complete === true && !limited;
-  const unreadLinkCount = catalog.reduce((sum, item) => sum + (item.links?.filter(link => !['read', 'not_content', 'reference'].includes(link.status)).length || 0), 0);
-  const textCoverageComplete = discoveryComplete && !unreadLinkCount && catalog.every(item => item.outcome === 'available' && ['available', 'empty', 'not_applicable', 'linked'].includes(item.textStatus));
-  const summary = {
-    result: explicit.length ? 'explicit_sources' : readable.length ? 'course_information' : 'unresolved',
-    overview: overview.outcome === 'available' ? descriptionContent(overview.data?.Description).text ? 'readable' : 'empty' : overview.outcome,
-    candidateCount: descriptors.length, readableSourceCount: readable.length, explicitSourceCount: explicit.length,
-    textCharacters: readable.reduce((sum, item) => sum + item.textCharacters, 0),
-    discoveryComplete, textCoverageComplete, limited, unreadLinkCount,
-    unresolvedSourceCount: catalog.filter(item => item.outcome !== 'available' || !['available', 'empty', 'not_applicable', 'linked'].includes(item.textStatus)).length,
-    formats: [...new Set(readable.map(item => item.format))].sort(),
-    retryAfter: Math.max(0, ...observations.filter(item => item.error === 'retry_wait').map(item => item.retryAfter || 60_000)),
-  };
-  observations.push({ key: `${le}/syllabus`, category: 'syllabus', course: courseId, source: `${le}/content/toc`,
-    retrievedAt: new Date().toISOString(), outcome: 'available', complete: discoveryComplete, data: { ...summary, sources: catalog } });
-  return { observations, summary };
-}
+import { AccessError, apiUrl } from './client.mjs';
+import { descriptionContent, readDocument } from './documents.mjs';
+import { readLinkedDocument, actionLink } from './links.mjs';
+
+const syllabusTitle = /syllab|course\s+(?:outline|overview|information|polic|guide)|\boutline\b|plan de cours/i;
+const explicitTitle = /syllab|course\s+outline|plan de cours/i;
+
+export function findSyllabusSources(tree, now = Date.now()) {
+  const sources = [];
+  function visit(node, parents = [], inSyllabusModule = false, ancestorUnavailable = false) {
+    if (!node || typeof node !== 'object') return;
+    const title = String(node.Title || '');
+    const description = descriptionContent(node.Description);
+    const directMatch = syllabusTitle.test(title);
+    const matched = directMatch || inSyllabusModule || explicitTitle.test(description.text);
+    const unavailable = ancestorUnavailable || node.IsHidden === true || node.IsLocked === true ||
+      Date.parse(node.StartDateTime) > now || Date.parse(node.EndDateTime) < now;
+    const common = { title, parents, topicId: node.TopicId, moduleId: node.ModuleId, activityType: node.ActivityType, lastModifiedDate: node.LastModifiedDate,
+      url: node.Url, unavailable, role: explicitTitle.test(title) ? 'explicit_syllabus' : 'course_information',
+      matchedBy: directMatch ? 'title' : inSyllabusModule ? 'parent_module' : 'description' };
+    if (matched && description.text) sources.push({ ...common, kind: node.TopicId != null ? 'topic_description' : 'module_description', description: description.text, links: description.links });
+    if (matched && node.TopicId != null) sources.push({ ...common, kind: node.ActivityType === 1 ? 'content_file' : 'linked_topic', description: '' });
+    for (const child of node.Modules || []) visit(child, [...parents, title].filter(Boolean), inSyllabusModule || directMatch, unavailable);
+    for (const child of node.Topics || []) visit(child, [...parents, title].filter(Boolean), inSyllabusModule || directMatch, unavailable);
+  }
+  visit(tree);
+  return sources;
+}
+
+export function classifyOverview(observation) {
+  if (observation.status === 404) {
+    observation.outcome = 'not_configured';
+    observation.complete = true;
+    observation.message = 'Course content overview is not configured; other syllabus locations still need checking.';
+    delete observation.error;
+  }
+  return observation;
+}
+
+export async function collectSyllabus(client, { le, courseId, content, overview }) {
+  const descriptors = content.data ? findSyllabusSources(content.data) : [];
+  const observations = [];
+  const catalog = [];
+  let documentReads = 0;
+  let limited = false;
+  const resolved = new Map();
+  const source = async (descriptor, action) => {
+    const identity = descriptor.kind === 'overview' ? 'overview' : descriptor.kind === 'overview_attachment' ? 'overview_attachment' :
+      `${descriptor.kind}:${descriptor.kind === 'linked_document' ? descriptor.url : descriptor.topicId ?? descriptor.moduleId ?? JSON.stringify([...(descriptor.parents || []), descriptor.title])}`;
+    const key = `syllabus:${courseId}:${identity}`;
+    const record = { key, category: 'syllabus_source', course: courseId, source: descriptor.source || `${le}/content/toc`,
+      retrievedAt: new Date().toISOString(), outcome: 'available', complete: true };
+    let entry = { ...descriptor, key, retrievedAt: record.retrievedAt };
+    delete entry.description;
+    try {
+      if (descriptor.unavailable) throw new AccessError('restricted', 'Source is hidden, locked, or outside its release window.');
+      const data = await action();
+      const role = descriptor.kind === 'overview_attachment' && explicitTitle.test(data.text?.slice(0, 2000) || '') ? 'explicit_syllabus' : descriptor.role;
+      record.data = { ...descriptor, ...data, role };
+      entry = { ...entry, outcome: 'available', textStatus: data.textStatus, textCharacters: data.text?.trim().length || 0, format: data.format,
+        pages: data.pages, links: data.links || [], linksLimited: data.linksLimited, source: record.source, role };
+      if (data.textStatus === 'failed' || data.textStatus === 'partial') {
+        record.complete = false;
+        record.outcome = 'partial';
+        entry.outcome = 'partial';
+      }
+    } catch (error) {
+      if (!(error instanceof AccessError) || error.code === 'needs_login') throw error;
+      if (error.status === 429) throw error;
+      record.outcome = ['denied', 'restricted'].includes(error.code) ? 'denied' : error.code === 'partial' ? 'partial' : 'failed';
+      Object.assign(record, { complete: false, error: error.code, message: error.message, status: error.status, retryAfter: error.retryAfter });
+      entry = { ...entry, outcome: record.outcome, error: error.code, message: error.message, textCharacters: 0 };
+    }
+    observations.push(record);
+    catalog.push(entry);
+    if (entry.textStatus === 'available') {
+      for (const candidate of [descriptor.url, record.source]) if (candidate) {
+        try { resolved.set(new URL(candidate, client.origin).href.split('#')[0], entry.key); } catch {}
+      }
+    }
+  };
+
+  if (overview.outcome === 'not_configured') {
+    for (const kind of ['overview', 'overview_attachment']) {
+      observations.push({ key: `syllabus:${courseId}:${kind}`, category: 'syllabus_source', course: courseId,
+        source: kind === 'overview' ? `${le}/overview` : `${le}/overview/attachment`, outcome: 'not_configured', complete: true,
+        retrievedAt: new Date().toISOString(), status: 404 });
+    }
+  } else if (overview.outcome === 'available') {
+    await source({ kind: 'overview', title: 'Course overview', role: 'course_information', source: `${le}/overview` }, async () => {
+      const value = descriptionContent(overview.data?.Description);
+      return { ...value, format: 'inline', textStatus: value.text ? 'available' : 'empty' };
+    });
+    if (overview.data?.HasAttachment === true) {
+      documentReads++;
+      await source({ kind: 'overview_attachment', title: 'Course overview attachment', role: 'course_information', source: `${le}/overview/attachment` }, () => readDocument(client, `${le}/overview/attachment`));
+    }
+  }
+
+  for (const descriptor of descriptors.slice(0, 100)) {
+    if (descriptor.unavailable) {
+      await source(descriptor, async () => ({}));
+      continue;
+    }
+    if (descriptor.kind.endsWith('_description')) {
+      await source(descriptor, async () => ({ text: descriptor.description, links: descriptor.links || [], format: 'inline', textStatus: descriptor.description ? 'available' : 'empty' }));
+      continue;
+    }
+    if (!/^\d+$/.test(String(descriptor.topicId))) {
+      catalog.push({ ...descriptor, outcome: 'unresolved', textCharacters: 0 });
+      continue;
+    }
+    const topicPath = `${le}/content/topics/${descriptor.topicId}`;
+    if (descriptor.kind === 'content_file') {
+      if (documentReads >= 20) { limited = true; catalog.push({ ...descriptor, outcome: 'not_read_limit', textCharacters: 0 }); continue; }
+      documentReads++;
+      await source({ ...descriptor, source: `${topicPath}/file` }, () => readDocument(client, `${topicPath}/file`));
+    } else {
+      await source({ ...descriptor, source: topicPath }, async () => {
+        const metadata = await client.get(topicPath);
+        const value = descriptionContent(metadata.Description);
+        const target = metadata.Url || descriptor.url;
+        // Link targets are handled below with separate school/public trust boundaries.
+        let linkStatus = 'unresolved';
+        if (target) {
+          try { apiUrl(client.origin, target); linkStatus = 'api_link_not_followed'; }
+          catch {
+            try { linkStatus = new URL(target, client.origin).origin === client.origin ? 'school_link_not_followed' : 'external'; }
+            catch { linkStatus = 'invalid'; }
+          }
+        }
+        return { ...value, format: 'link', metadata, links: target ? [...value.links, { url: target, label: descriptor.title, status: linkStatus }] : value.links,
+          textStatus: value.text ? 'available' : target && actionLink(target, client.origin) ? 'not_applicable' : 'unresolved' };
+      });
+    }
+  }
+  // Breadth-first, deduplicated document links; never activate surveys or enrollment links.
+  const seen = new Set();
+  for (let index = 0; index < catalog.length && index < 200; index++) {
+    const entry = catalog[index];
+    for (const link of entry.links || []) {
+      if (actionLink(link.url, client.origin)) { link.status = 'not_content'; continue; }
+      let target;
+      try { target = new URL(link.url, entry.url ? new URL(entry.url, client.origin) : client.origin); target.hash = ''; }
+      catch { link.status = 'invalid'; continue; }
+      // Read source documents, not an unbounded crawl of their citations and site navigation.
+      if (target.origin !== client.origin && ((entry.linkDepth || 0) > 0 ||
+          (entry.format === 'pdf' && !/\.(?:pdf|docx|pptx|xlsx)$/i.test(target.pathname) && !syllabusTitle.test(link.label || '')))) {
+        link.status = 'reference';
+        continue;
+      }
+      if (resolved.has(target.href)) { link.status = 'read'; link.sourceKey = resolved.get(target.href); continue; }
+      if (seen.has(target.href)) { link.status = 'already_attempted'; continue; }
+      seen.add(target.href);
+      if ((entry.linkDepth || 0) >= 2 || documentReads >= 20) { limited = true; link.status = 'not_read_limit'; continue; }
+      documentReads++;
+      await source({ kind: 'linked_document', title: link.label || 'Linked course document', url: target.href,
+        parents: [...(entry.parents || []), entry.title].filter(Boolean), role: 'course_information',
+        source: target.href, linkDepth: (entry.linkDepth || 0) + 1 }, () => readLinkedDocument(client, target.href, { le, courseId, tree: content.data }));
+      const linked = catalog.at(-1);
+      link.status = linked.textStatus === 'available' ? 'read' : linked.error || linked.textStatus || linked.outcome;
+      link.sourceKey = linked.key;
+      if (link.status === 'read') resolved.set(target.href, linked.key);
+    }
+    if (entry.kind === 'linked_topic' && entry.textStatus === 'unresolved' && entry.links?.length &&
+        entry.links.every(link => ['read', 'not_content', 'reference'].includes(link.status))) {
+      entry.textStatus = 'linked';
+      const observation = observations.find(item => item.key === entry.key);
+      if (observation?.data) observation.data.textStatus = 'linked';
+    }
+  }
+  limited ||= descriptors.length > 100 || catalog.some(item => item.linksLimited);
+  const readable = catalog.filter(item => item.textStatus === 'available' && item.textCharacters > 0);
+  const explicit = readable.filter(item => item.role === 'explicit_syllabus');
+  const discoveryComplete = content.complete === true && overview.complete === true && !limited;
+  const unreadLinkCount = catalog.reduce((sum, item) => sum + (item.links?.filter(link => !['read', 'not_content', 'reference'].includes(link.status)).length || 0), 0);
+  const textCoverageComplete = discoveryComplete && !unreadLinkCount && catalog.every(item => item.outcome === 'available' && ['available', 'empty', 'not_applicable', 'linked'].includes(item.textStatus));
+  const summary = {
+    result: explicit.length ? 'explicit_sources' : readable.length ? 'course_information' : 'unresolved',
+    overview: overview.outcome === 'available' ? descriptionContent(overview.data?.Description).text ? 'readable' : 'empty' : overview.outcome,
+    candidateCount: descriptors.length, readableSourceCount: readable.length, explicitSourceCount: explicit.length,
+    textCharacters: readable.reduce((sum, item) => sum + item.textCharacters, 0),
+    discoveryComplete, textCoverageComplete, limited, unreadLinkCount,
+    unresolvedSourceCount: catalog.filter(item => item.outcome !== 'available' || !['available', 'empty', 'not_applicable', 'linked'].includes(item.textStatus)).length,
+    formats: [...new Set(readable.map(item => item.format))].sort(),
+    retryAfter: Math.max(0, ...observations.filter(item => item.error === 'retry_wait').map(item => item.retryAfter || 60_000)),
+  };
+  observations.push({ key: `${le}/syllabus`, category: 'syllabus', course: courseId, source: `${le}/content/toc`,
+    retrievedAt: new Date().toISOString(), outcome: 'available', complete: discoveryComplete, data: { ...summary, sources: catalog } });
+  return { observations, summary };
+}