erdos-problems 0.1.0 → 0.1.2

package/src/commands/scaffold.js

@@ -0,0 +1,60 @@
+import path from 'node:path';
+import { getProblem } from '../atlas/catalog.js';
+import { scaffoldProblem } from '../runtime/problem-artifacts.js';
+import { getWorkspaceProblemScaffoldDir } from '../runtime/paths.js';
+import { readCurrentProblem } from '../runtime/workspace.js';
+
+export function parseScaffoldArgs(args) {
+  const [kind, value, ...rest] = args;
+  if (kind !== 'problem') {
+    return { error: 'Only `erdos scaffold problem <id>` is supported right now.' };
+  }
+  let destination = null;
+  for (let index = 0; index < rest.length; index += 1) {
+    const token = rest[index];
+    if (token === '--dest') {
+      destination = rest[index + 1];
+      if (!destination) {
+        return { error: 'Missing destination path after --dest.' };
+      }
+      index += 1;
+      continue;
+    }
+    return { error: `Unknown scaffold option: ${token}` };
+  }
+  return { problemId: value ?? readCurrentProblem(), destination };
+}
+
+export function runScaffoldCommand(args) {
+  if (args.length === 0 || args[0] === 'help' || args[0] === '--help') {
+    console.log('Usage:');
+    console.log('  erdos scaffold problem <id> [--dest <path>]');
+    return 0;
+  }
+
+  const parsed = parseScaffoldArgs(args);
+  if (parsed.error) {
+    console.error(parsed.error);
+    return 1;
+  }
+  if (!parsed.problemId) {
+    console.error('Missing problem id and no active problem is selected.');
+    return 1;
+  }
+
+  const problem = getProblem(parsed.problemId);
+  if (!problem) {
+    console.error(`Unknown problem: ${parsed.problemId}`);
+    return 1;
+  }
+
+  const destination = parsed.destination
+    ? path.resolve(parsed.destination)
+    : getWorkspaceProblemScaffoldDir(problem.problemId);
+
+  const result = scaffoldProblem(problem, destination);
+  console.log(`Scaffold created: ${result.destination}`);
+  console.log(`Artifacts copied: ${result.copiedArtifacts.length}`);
+  console.log(`Upstream record included: ${result.inventory.upstreamRecordIncluded ? 'yes' : 'no'}`);
+  return 0;
+}

package/src/commands/upstream.js

@@ -0,0 +1,60 @@
+import { buildUpstreamDiff, loadActiveUpstreamSnapshot, syncUpstream, writeDiffArtifacts } from '../upstream/sync.js';
+
+export async function runUpstreamCommand(args) {
+  const [subcommand, ...rest] = args;
+
+  if (!subcommand || subcommand === 'help' || subcommand === '--help') {
+    console.log('Usage:');
+    console.log('  erdos upstream show');
+    console.log('  erdos upstream sync [--write-package-snapshot]');
+    console.log('  erdos upstream diff [--write-package-report]');
+    return 0;
+  }
+
+  if (subcommand === 'show') {
+    const snapshot = loadActiveUpstreamSnapshot();
+    if (!snapshot) {
+      console.log('No upstream snapshot available yet. Run `erdos upstream sync`.');
+      return 0;
+    }
+    console.log(`Snapshot kind: ${snapshot.kind}`);
+    console.log(`Upstream repo: ${snapshot.manifest.upstream_repo}`);
+    console.log(`Upstream commit: ${snapshot.manifest.upstream_commit ?? '(unknown)'}`);
+    console.log(`Fetched at: ${snapshot.manifest.fetched_at}`);
+    console.log(`Entries: ${snapshot.manifest.entry_count}`);
+    return 0;
+  }
+
+  if (subcommand === 'sync') {
+    const writePackageSnapshot = rest.includes('--write-package-snapshot');
+    const result = await syncUpstream({ writePackageSnapshot });
+    console.log(`Fetched upstream commit: ${result.snapshot.manifest.upstream_commit ?? '(unknown)'}`);
+    console.log(`Workspace snapshot: ${result.workspacePaths.manifestPath}`);
+    if (result.bundledPaths) {
+      console.log(`Bundled snapshot: ${result.bundledPaths.manifestPath}`);
+    }
+    const diffArtifacts = writeDiffArtifacts({ writePackageReport: writePackageSnapshot });
+    console.log(`Workspace diff report: ${diffArtifacts.workspaceDiffPath}`);
+    if (diffArtifacts.repoDiffPath) {
+      console.log(`Repo diff report: ${diffArtifacts.repoDiffPath}`);
+    }
+    return 0;
+  }
+
+  if (subcommand === 'diff') {
+    const writePackageReport = rest.includes('--write-package-report');
+    const diffArtifacts = writeDiffArtifacts({ writePackageReport });
+    const diff = buildUpstreamDiff();
+    console.log(`Local seeded problems: ${diff.localProblemCount}`);
+    console.log(`Upstream total problems: ${diff.upstreamProblemCount}`);
+    console.log(`Upstream-only count: ${diff.upstreamOnlyCount}`);
+    console.log(`Workspace diff report: ${diffArtifacts.workspaceDiffPath}`);
+    if (diffArtifacts.repoDiffPath) {
+      console.log(`Repo diff report: ${diffArtifacts.repoDiffPath}`);
+    }
+    return 0;
+  }
+
+  console.error(`Unknown upstream subcommand: ${subcommand}`);
+  return 1;
+}

package/src/commands/workspace.js

@@ -19,6 +19,8 @@ export function runWorkspaceCommand(args) {
   console.log(`State dir: ${summary.stateDir}`);
   console.log(`Initialized: ${summary.hasState ? 'yes' : 'no'}`);
   console.log(`Active problem: ${summary.activeProblem ?? '(none)'}`);
+  console.log(`Workspace upstream dir: ${summary.upstreamDir}`);
+  console.log(`Workspace scaffold dir: ${summary.scaffoldDir}`);
   console.log(`Updated at: ${summary.updatedAt ?? '(none)'}`);
   return 0;
 }

package/src/runtime/files.js

@@ -0,0 +1,37 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+export function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+export function writeJson(filePath, payload) {
+  ensureDir(path.dirname(filePath));
+  fs.writeFileSync(filePath, JSON.stringify(payload, null, 2) + '\n');
+}
+
+export function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+export function writeText(filePath, text) {
+  ensureDir(path.dirname(filePath));
+  fs.writeFileSync(filePath, text);
+}
+
+export function readText(filePath) {
+  return fs.readFileSync(filePath, 'utf8');
+}
+
+export function fileExists(filePath) {
+  return fs.existsSync(filePath);
+}
+
+export function copyFileIfPresent(sourcePath, destinationPath) {
+  if (!fileExists(sourcePath)) {
+    return false;
+  }
+  ensureDir(path.dirname(destinationPath));
+  fs.copyFileSync(sourcePath, destinationPath);
+  return true;
+}

package/src/runtime/paths.js

@@ -20,6 +20,70 @@ export function getCurrentProblemPath() {
   return path.join(getWorkspaceDir(), 'current-problem.json');
 }
 
+export function getWorkspaceReportsDir() {
+  return path.join(getWorkspaceDir(), 'reports');
+}
+
+export function getWorkspaceDiffPath() {
+  return path.join(getWorkspaceReportsDir(), 'UPSTREAM_DIFF.md');
+}
+
+export function getWorkspaceUpstreamDir() {
+  return path.join(getWorkspaceDir(), 'upstream', 'erdosproblems');
+}
+
+export function getWorkspaceUpstreamYamlPath() {
+  return path.join(getWorkspaceUpstreamDir(), 'problems.yaml');
+}
+
+export function getWorkspaceUpstreamIndexPath() {
+  return path.join(getWorkspaceUpstreamDir(), 'PROBLEMS_INDEX.json');
+}
+
+export function getWorkspaceUpstreamManifestPath() {
+  return path.join(getWorkspaceUpstreamDir(), 'SYNC_MANIFEST.json');
+}
+
+export function getWorkspaceScaffoldsDir() {
+  return path.join(getWorkspaceDir(), 'scaffolds');
+}
+
+export function getWorkspaceProblemScaffoldDir(problemId) {
+  return path.join(getWorkspaceScaffoldsDir(), String(problemId));
+}
+
 export function getProblemDir(problemId) {
   return path.join(repoRoot, 'problems', String(problemId));
 }
+
+export function getBundledDataDir() {
+  return path.join(repoRoot, 'data');
+}
+
+export function getBundledUpstreamDir() {
+  return path.join(getBundledDataDir(), 'upstream', 'erdosproblems');
+}
+
+export function getBundledUpstreamYamlPath() {
+  return path.join(getBundledUpstreamDir(), 'problems.yaml');
+}
+
+export function getBundledUpstreamIndexPath() {
+  return path.join(getBundledUpstreamDir(), 'PROBLEMS_INDEX.json');
+}
+
+export function getBundledUpstreamManifestPath() {
+  return path.join(getBundledUpstreamDir(), 'SYNC_MANIFEST.json');
+}
+
+export function getRepoAnalysisDir() {
+  return path.join(repoRoot, 'analysis');
+}
+
+export function getRepoUpstreamDiffPath() {
+  return path.join(getRepoAnalysisDir(), 'UPSTREAM_DIFF.md');
+}
+
+export function getPackDir(packName) {
+  return path.join(repoRoot, 'packs', String(packName));
+}

package/src/runtime/problem-artifacts.js

@@ -0,0 +1,150 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { loadActiveUpstreamSnapshot } from '../upstream/sync.js';
+import { copyFileIfPresent, ensureDir, writeJson, writeText } from './files.js';
+import { getPackDir } from './paths.js';
+
+const DOSSIER_FILES = [
+  ['problem.yaml', 'problemYamlPath', 'problem.yaml'],
+  ['STATEMENT.md', 'statementPath', 'STATEMENT.md'],
+  ['REFERENCES.md', 'referencesPath', 'REFERENCES.md'],
+  ['EVIDENCE.md', 'evidencePath', 'EVIDENCE.md'],
+  ['FORMALIZATION.md', 'formalizationPath', 'FORMALIZATION.md'],
+];
+
+function getPackContextPath(problem) {
+  if (!problem.cluster) {
+    return null;
+  }
+  return path.join(getPackDir(problem.cluster), 'README.md');
+}
+
+export function getProblemArtifactInventory(problem) {
+  const snapshot = loadActiveUpstreamSnapshot();
+  const upstreamRecord = snapshot?.index?.by_number?.[problem.problemId] ?? null;
+  const canonicalArtifacts = DOSSIER_FILES.map(([label, key, destinationName]) => {
+    const filePath = problem[key];
+    return {
+      label,
+      path: filePath,
+      destinationName,
+      exists: fs.existsSync(filePath),
+    };
+  });
+
+  const packContextPath = getPackContextPath(problem);
+  const packContext = packContextPath
+    ? {
+        label: 'PACK_CONTEXT.md',
+        path: packContextPath,
+        destinationName: 'PACK_CONTEXT.md',
+        exists: fs.existsSync(packContextPath),
+      }
+    : null;
+
+  return {
+    generatedAt: new Date().toISOString(),
+    problemId: problem.problemId,
+    displayName: problem.displayName,
+    title: problem.title,
+    sourceUrl: problem.sourceUrl,
+    cluster: problem.cluster,
+    repoStatus: problem.repoStatus,
+    harnessDepth: problem.harnessDepth,
+    problemDir: problem.problemDir,
+    canonicalArtifacts,
+    packContext,
+    upstreamSnapshot: snapshot
+      ? {
+          kind: snapshot.kind,
+          manifestPath: snapshot.manifestPath,
+          indexPath: snapshot.indexPath,
+          yamlPath: snapshot.yamlPath,
+          upstreamCommit: snapshot.manifest.upstream_commit ?? null,
+          fetchedAt: snapshot.manifest.fetched_at,
+        }
+      : null,
+    upstreamRecordIncluded: Boolean(upstreamRecord),
+    upstreamRecord,
+  };
+}
+
+export function scaffoldProblem(problem, destination) {
+  ensureDir(destination);
+  const inventory = getProblemArtifactInventory(problem);
+
+  const copiedArtifacts = [];
+  for (const artifact of inventory.canonicalArtifacts) {
+    const destinationPath = path.join(destination, artifact.destinationName);
+    if (copyFileIfPresent(artifact.path, destinationPath)) {
+      copiedArtifacts.push({
+        label: artifact.label,
+        sourcePath: artifact.path,
+        destinationPath,
+      });
+    }
+  }
+
+  if (inventory.packContext?.exists) {
+    const destinationPath = path.join(destination, inventory.packContext.destinationName);
+    if (copyFileIfPresent(inventory.packContext.path, destinationPath)) {
+      copiedArtifacts.push({
+        label: inventory.packContext.label,
+        sourcePath: inventory.packContext.path,
+        destinationPath,
+      });
+    }
+  }
+
+  if (inventory.upstreamRecord) {
+    writeJson(path.join(destination, 'UPSTREAM_RECORD.json'), inventory.upstreamRecord);
+  }
+
+  const problemRecord = {
+    generatedAt: inventory.generatedAt,
+    problemId: problem.problemId,
+    title: problem.title,
+    cluster: problem.cluster,
+    siteStatus: problem.siteStatus,
+    repoStatus: problem.repoStatus,
+    harnessDepth: problem.harnessDepth,
+    sourceUrl: problem.sourceUrl,
+    activeRoute: problem.researchState?.active_route ?? null,
+  };
+
+  const artifactIndex = {
+    generatedAt: inventory.generatedAt,
+    problemId: problem.problemId,
+    bundledProblemDir: problem.problemDir,
+    copiedArtifacts,
+    packContext: inventory.packContext,
+    canonicalArtifacts: inventory.canonicalArtifacts,
+    upstreamSnapshot: inventory.upstreamSnapshot,
+    includedUpstreamRecord: inventory.upstreamRecordIncluded,
+  };
+
+  writeJson(path.join(destination, 'PROBLEM.json'), problemRecord);
+  writeJson(path.join(destination, 'ARTIFACT_INDEX.json'), artifactIndex);
+  writeText(
+    path.join(destination, 'README.md'),
+    [
+      `# Erdos Problem ${problem.problemId} Scaffold`,
+      '',
+      'This scaffold was generated by the erdos CLI.',
+      '',
+      `- Title: ${problem.title}`,
+      `- Cluster: ${problem.cluster}`,
+      `- Source: ${problem.sourceUrl}`,
+      `- Repo status: ${problem.repoStatus}`,
+      `- Harness depth: ${problem.harnessDepth}`,
+      `- Upstream record included: ${inventory.upstreamRecordIncluded ? 'yes' : 'no'}`,
+      '',
+    ].join('\n'),
+  );
+
+  return {
+    destination,
+    copiedArtifacts,
+    inventory,
+  };
+}

package/src/runtime/workspace.js

@@ -1,16 +1,13 @@
 import fs from 'node:fs';
-import path from 'node:path';
 import {
   getCurrentProblemPath,
   getWorkspaceDir,
+  getWorkspaceProblemScaffoldDir,
   getWorkspaceRoot,
   getWorkspaceStatePath,
+  getWorkspaceUpstreamDir,
 } from './paths.js';
-
-function writeJson(filePath, payload) {
-  fs.mkdirSync(path.dirname(filePath), { recursive: true });
-  fs.writeFileSync(filePath, JSON.stringify(payload, null, 2) + '\n');
-}
+import { writeJson } from './files.js';
 
 export function readWorkspaceState() {
   const filePath = getWorkspaceStatePath();
@@ -61,11 +58,14 @@ export function readCurrentProblem() {
 
 export function getWorkspaceSummary() {
   const state = readWorkspaceState();
+  const activeProblem = readCurrentProblem();
   return {
     workspaceRoot: getWorkspaceRoot(),
     stateDir: getWorkspaceDir(),
     hasState: Boolean(state),
-    activeProblem: readCurrentProblem(),
+    activeProblem,
+    upstreamDir: getWorkspaceUpstreamDir(),
+    scaffoldDir: activeProblem ? getWorkspaceProblemScaffoldDir(activeProblem) : getWorkspaceProblemScaffoldDir('<problem-id>'),
     updatedAt: state?.updatedAt ?? null,
   };
 }

package/src/upstream/sync.js

@@ -0,0 +1,272 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { parse } from 'yaml';
+import { loadLocalProblems } from '../atlas/catalog.js';
+import { ensureDir, readJson, writeJson, writeText } from '../runtime/files.js';
+import {
+  getBundledUpstreamDir,
+  getBundledUpstreamIndexPath,
+  getBundledUpstreamManifestPath,
+  getBundledUpstreamYamlPath,
+  getRepoUpstreamDiffPath,
+  getWorkspaceDiffPath,
+  getWorkspaceUpstreamDir,
+  getWorkspaceUpstreamIndexPath,
+  getWorkspaceUpstreamManifestPath,
+  getWorkspaceUpstreamYamlPath,
+} from '../runtime/paths.js';
+
+const UPSTREAM_REPO_URL = 'https://github.com/teorth/erdosproblems';
+const RAW_PROBLEMS_URL = 'https://raw.githubusercontent.com/teorth/erdosproblems/master/data/problems.yaml';
+const COMMIT_API_URL = 'https://api.github.com/repos/teorth/erdosproblems/commits?path=data/problems.yaml&per_page=1';
+
+function sha256(text) {
+  return crypto.createHash('sha256').update(text).digest('hex');
+}
+
+function normalizeTag(tag) {
+  return String(tag).trim().toLowerCase();
+}
+
+function normalizeStatus(state) {
+  return String(state ?? '').trim().toLowerCase();
+}
+
+function normalizeUpstreamRecord(entry) {
+  return {
+    number: String(entry.number),
+    prize: entry.prize ?? null,
+    status: {
+      state: entry.status?.state ?? 'unknown',
+      last_update: entry.status?.last_update ?? null,
+    },
+    formalized: {
+      state: entry.formalized?.state ?? 'unknown',
+      last_update: entry.formalized?.last_update ?? null,
+    },
+    tags: (entry.tags ?? []).map(normalizeTag),
+    oeis: entry.oeis ?? [],
+  };
+}
+
+async function fetchUpstreamCommit() {
+  const response = await fetch(COMMIT_API_URL, {
+    headers: {
+      'User-Agent': 'erdos-problems-cli',
+      Accept: 'application/vnd.github+json',
+    },
+  });
+  if (!response.ok) {
+    return null;
+  }
+  const payload = await response.json();
+  return payload[0]?.sha ?? null;
+}
+
+export async function fetchUpstreamSnapshot() {
+  const response = await fetch(RAW_PROBLEMS_URL, {
+    headers: {
+      'User-Agent': 'erdos-problems-cli',
+      Accept: 'text/plain',
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`Unable to fetch upstream problems.yaml: ${response.status}`);
+  }
+  const rawYaml = await response.text();
+  const parsed = parse(rawYaml);
+  const records = parsed.map(normalizeUpstreamRecord);
+  const commit = await fetchUpstreamCommit();
+  const manifest = {
+    upstream_repo: UPSTREAM_REPO_URL,
+    source_url: RAW_PROBLEMS_URL,
+    source_file: 'data/problems.yaml',
+    fetched_at: new Date().toISOString(),
+    upstream_commit: commit,
+    raw_sha256: sha256(rawYaml),
+    entry_count: records.length,
+  };
+  const byNumber = Object.fromEntries(records.map((record) => [record.number, record]));
+  return {
+    rawYaml,
+    manifest,
+    index: {
+      generated_at: manifest.fetched_at,
+      entry_count: records.length,
+      records,
+      by_number: byNumber,
+    },
+  };
+}
+
+function loadSnapshotPaths(preferWorkspace) {
+  if (preferWorkspace && fs.existsSync(getWorkspaceUpstreamIndexPath())) {
+    return {
+      manifestPath: getWorkspaceUpstreamManifestPath(),
+      indexPath: getWorkspaceUpstreamIndexPath(),
+      yamlPath: getWorkspaceUpstreamYamlPath(),
+      kind: 'workspace',
+    };
+  }
+  return {
+    manifestPath: getBundledUpstreamManifestPath(),
+    indexPath: getBundledUpstreamIndexPath(),
+    yamlPath: getBundledUpstreamYamlPath(),
+    kind: 'bundled',
+  };
+}
+
+export function loadActiveUpstreamSnapshot() {
+  const paths = loadSnapshotPaths(true);
+  if (!fs.existsSync(paths.indexPath) || !fs.existsSync(paths.manifestPath)) {
+    return null;
+  }
+  return {
+    kind: paths.kind,
+    manifest: readJson(paths.manifestPath),
+    manifestPath: paths.manifestPath,
+    index: readJson(paths.indexPath),
+    indexPath: paths.indexPath,
+    yamlPath: paths.yamlPath,
+  };
+}
+
+function writeSnapshotToDirectory(directory, snapshot) {
+  ensureDir(directory);
+  const yamlPath = path.join(directory, 'problems.yaml');
+  const indexPath = path.join(directory, 'PROBLEMS_INDEX.json');
+  const manifestPath = path.join(directory, 'SYNC_MANIFEST.json');
+  writeText(yamlPath, snapshot.rawYaml);
+  writeJson(indexPath, snapshot.index);
+  writeJson(manifestPath, snapshot.manifest);
+  return { yamlPath, indexPath, manifestPath };
+}
+
+function compareTagSets(localProblem, upstreamRecord) {
+  const localTags = new Set((localProblem.familyTags ?? []).map(normalizeTag));
+  const upstreamTags = new Set((upstreamRecord.tags ?? []).map(normalizeTag));
+  const localOnly = [...localTags].filter((tag) => !upstreamTags.has(tag)).sort();
+  const upstreamOnly = [...upstreamTags].filter((tag) => !localTags.has(tag)).sort();
+  return { localOnly, upstreamOnly };
+}
+
+export function buildUpstreamDiff() {
+  const snapshot = loadActiveUpstreamSnapshot();
+  if (!snapshot) {
+    throw new Error('No upstream snapshot available. Run `erdos upstream sync` first.');
+  }
+
+  const localProblems = loadLocalProblems();
+  const recordsByNumber = snapshot.index.by_number ?? {};
+  const overlaps = [];
+  const localOnly = [];
+
+  for (const problem of localProblems) {
+    const upstream = recordsByNumber[problem.problemId];
+    if (!upstream) {
+      localOnly.push(problem.problemId);
+      continue;
+    }
+
+    const statusMatches = normalizeStatus(problem.siteStatus) === normalizeStatus(upstream.status.state);
+    const formalizedMatches = normalizeStatus(problem.upstreamFormalizedState) === normalizeStatus(upstream.formalized.state);
+    const tagDiff = compareTagSets(problem, upstream);
+
+    overlaps.push({
+      problemId: problem.problemId,
+      title: problem.title,
+      cluster: problem.cluster,
+      localSiteStatus: problem.siteStatus,
+      upstreamSiteStatus: upstream.status.state,
+      statusMatches,
+      localFormalized: problem.upstreamFormalizedState,
+      upstreamFormalized: upstream.formalized.state,
+      formalizedMatches,
+      localPrize: problem.prize,
+      upstreamPrize: upstream.prize,
+      localOnlyTags: tagDiff.localOnly,
+      upstreamOnlyTags: tagDiff.upstreamOnly,
+    });
+  }
+
+  const upstreamOnly = Object.keys(recordsByNumber)
+    .filter((problemId) => !localProblems.some((problem) => problem.problemId === problemId))
+    .sort((a, b) => Number(a) - Number(b));
+
+  return {
+    snapshot,
+    localProblemCount: localProblems.length,
+    upstreamProblemCount: snapshot.index.entry_count,
+    overlaps,
+    localOnly,
+    upstreamOnlyCount: upstreamOnly.length,
+    upstreamOnlyPreview: upstreamOnly.slice(0, 20),
+  };
+}
+
+export function renderUpstreamDiffMarkdown(diff) {
+  const lines = [];
+  lines.push('# Upstream Diff');
+  lines.push('');
+  lines.push(`Snapshot kind: \
+${diff.snapshot.kind}`.replace('\
+', ''));
+  lines.push(`Upstream commit: ${diff.snapshot.manifest.upstream_commit ?? '(unknown)'}`);
+  lines.push(`Fetched at: ${diff.snapshot.manifest.fetched_at}`);
+  lines.push(`Local seeded problems: ${diff.localProblemCount}`);
+  lines.push(`Upstream total problems: ${diff.upstreamProblemCount}`);
+  lines.push(`Upstream-only problems not yet locally seeded: ${diff.upstreamOnlyCount}`);
+  lines.push('');
+  lines.push('## Overlap');
+  lines.push('');
+  lines.push('| ID | Cluster | Local site | Upstream site | Status | Local formalized | Upstream formalized | Tags |');
+  lines.push('| --- | --- | --- | --- | --- | --- | --- | --- |');
+  for (const row of diff.overlaps) {
+    const status = row.statusMatches ? 'match' : 'DIFF';
+    const formalized = row.formalizedMatches ? 'match' : 'DIFF';
+    const tagNotes = [];
+    if (row.localOnlyTags.length > 0) {
+      tagNotes.push(`local-only: ${row.localOnlyTags.join(', ')}`);
+    }
+    if (row.upstreamOnlyTags.length > 0) {
+      tagNotes.push(`upstream-only: ${row.upstreamOnlyTags.join(', ')}`);
+    }
+    lines.push(`| ${row.problemId} | ${row.cluster} | ${row.localSiteStatus} | ${row.upstreamSiteStatus} | ${status} | ${row.localFormalized ?? '(unset)'} | ${row.upstreamFormalized ?? '(unset)'} | ${tagNotes.join('; ') || 'match'} |`);
+  }
+  lines.push('');
+  lines.push('## Upstream-Only Preview');
+  lines.push('');
+  lines.push(diff.upstreamOnlyPreview.join(', ') || '(none)');
+  lines.push('');
+  return lines.join('\n');
+}
+
+export async function syncUpstream({ writePackageSnapshot = false } = {}) {
+  const snapshot = await fetchUpstreamSnapshot();
+  const workspacePaths = writeSnapshotToDirectory(getWorkspaceUpstreamDir(), snapshot);
+  let bundledPaths = null;
+  if (writePackageSnapshot) {
+    bundledPaths = writeSnapshotToDirectory(getBundledUpstreamDir(), snapshot);
+  }
+  return {
+    snapshot,
+    workspacePaths,
+    bundledPaths,
+  };
+}
+
+export function writeDiffArtifacts({ writePackageReport = false } = {}) {
+  const diff = buildUpstreamDiff();
+  const markdown = renderUpstreamDiffMarkdown(diff);
+  writeText(getWorkspaceDiffPath(), markdown);
+  if (writePackageReport) {
+    writeText(getRepoUpstreamDiffPath(), markdown);
+  }
+  return {
+    diff,
+    markdown,
+    workspaceDiffPath: getWorkspaceDiffPath(),
+    repoDiffPath: writePackageReport ? getRepoUpstreamDiffPath() : null,
+  };
+}