erdos-problems 0.1.1 → 0.1.3

package/src/commands/pull.js

@@ -0,0 +1,203 @@
+import path from 'node:path';
+import { getProblem } from '../atlas/catalog.js';
+import { ensureDir, writeJson, writeText } from '../runtime/files.js';
+import { getWorkspaceProblemPullDir } from '../runtime/paths.js';
+import { scaffoldProblem } from '../runtime/problem-artifacts.js';
+import { loadActiveUpstreamSnapshot, syncUpstream } from '../upstream/sync.js';
+import { fetchProblemSiteSnapshot } from '../upstream/site.js';
+
+function parsePullArgs(args) {
+  const [kind, value, ...rest] = args;
+  if (kind !== 'problem') {
+    return { error: 'Only `erdos pull problem <id>` is supported right now.' };
+  }
+
+  let destination = null;
+  let includeSite = false;
+  let refreshUpstream = false;
+
+  for (let index = 0; index < rest.length; index += 1) {
+    const token = rest[index];
+    if (token === '--dest') {
+      destination = rest[index + 1];
+      if (!destination) {
+        return { error: 'Missing destination path after --dest.' };
+      }
+      index += 1;
+      continue;
+    }
+    if (token === '--include-site') {
+      includeSite = true;
+      continue;
+    }
+    if (token === '--refresh-upstream') {
+      refreshUpstream = true;
+      continue;
+    }
+    return { error: `Unknown pull option: ${token}` };
+  }
+
+  return {
+    problemId: value,
+    destination,
+    includeSite,
+    refreshUpstream,
+  };
+}
+
+function writeUpstreamOnlyBundle(problemId, destination, upstreamRecord, snapshot) {
+  ensureDir(destination);
+
+  if (upstreamRecord) {
+    writeJson(path.join(destination, 'UPSTREAM_RECORD.json'), upstreamRecord);
+  }
+
+  const generatedAt = new Date().toISOString();
+  writeJson(path.join(destination, 'PROBLEM.json'), {
+    generatedAt,
+    problemId,
+    title: `Erdos Problem #${problemId}`,
+    cluster: null,
+    siteStatus: upstreamRecord?.status?.state ?? 'unknown',
+    repoStatus: 'upstream-only',
+    harnessDepth: 'unseeded',
+    sourceUrl: `https://www.erdosproblems.com/${problemId}`,
+    activeRoute: null,
+  });
+
+  writeJson(path.join(destination, 'ARTIFACT_INDEX.json'), {
+    generatedAt,
+    problemId,
+    copiedArtifacts: [],
+    canonicalArtifacts: [],
+    upstreamSnapshot: snapshot
+      ? {
+          kind: snapshot.kind,
+          manifestPath: snapshot.manifestPath,
+          indexPath: snapshot.indexPath,
+          yamlPath: snapshot.yamlPath,
+          upstreamCommit: snapshot.manifest.upstream_commit ?? null,
+          fetchedAt: snapshot.manifest.fetched_at,
+        }
+      : null,
+    includedUpstreamRecord: Boolean(upstreamRecord),
+  });
+
+  writeText(
+    path.join(destination, 'README.md'),
+    [
+      `# Erdos Problem ${problemId} Pull Bundle`,
+      '',
+      'This bundle was generated from upstream public metadata.',
+      '',
+      `- Source: https://www.erdosproblems.com/${problemId}`,
+      `- Upstream record included: ${upstreamRecord ? 'yes' : 'no'}`,
+      '',
+      'This problem is not yet seeded locally as a canonical dossier in this package.',
+      '',
+    ].join('\n'),
+  );
+}
+
+async function maybeWriteSiteBundle(problemId, destination, includeSite) {
+  if (!includeSite) {
+    return { attempted: false, included: false, error: null };
+  }
+
+  try {
+    const siteSnapshot = await fetchProblemSiteSnapshot(problemId);
+    writeText(path.join(destination, 'SITE_SNAPSHOT.html'), siteSnapshot.html);
+    writeText(path.join(destination, 'SITE_EXTRACT.txt'), siteSnapshot.text);
+    writeJson(path.join(destination, 'SITE_EXTRACT.json'), {
+      url: siteSnapshot.url,
+      fetchedAt: siteSnapshot.fetchedAt,
+      title: siteSnapshot.title,
+      previewLines: siteSnapshot.previewLines,
+    });
+    writeText(
+      path.join(destination, 'SITE_SUMMARY.md'),
+      [
+        `# Erdős Problem #${problemId} Site Summary`,
+        '',
+        `Source: ${siteSnapshot.url}`,
+        `Fetched at: ${siteSnapshot.fetchedAt}`,
+        `Title: ${siteSnapshot.title}`,
+        '',
+        '## Preview',
+        '',
+        ...siteSnapshot.previewLines.map((line) => `- ${line}`),
+        '',
+      ].join('\n'),
+    );
+    return { attempted: true, included: true, error: null };
+  } catch (error) {
+    writeText(path.join(destination, 'SITE_FETCH_ERROR.txt'), String(error.message ?? error));
+    return { attempted: true, included: false, error: String(error.message ?? error) };
+  }
+}
+
+export async function runPullCommand(args) {
+  if (args.length === 0 || args[0] === 'help' || args[0] === '--help') {
+    console.log('Usage:');
+    console.log('  erdos pull problem <id> [--dest <path>] [--include-site] [--refresh-upstream]');
+    return 0;
+  }
+
+  const parsed = parsePullArgs(args);
+  if (parsed.error) {
+    console.error(parsed.error);
+    return 1;
+  }
+  if (!parsed.problemId) {
+    console.error('Missing problem id.');
+    return 1;
+  }
+
+  if (parsed.refreshUpstream) {
+    await syncUpstream();
+  }
+
+  const localProblem = getProblem(parsed.problemId);
+  const snapshot = loadActiveUpstreamSnapshot();
+  const upstreamRecord = snapshot?.index?.by_number?.[String(parsed.problemId)] ?? null;
+
+  if (!localProblem && !upstreamRecord) {
+    console.error(`Problem ${parsed.problemId} is not present in the local dossier set or upstream snapshot.`);
+    return 1;
+  }
+
+  const destination = parsed.destination
+    ? path.resolve(parsed.destination)
+    : getWorkspaceProblemPullDir(parsed.problemId);
+
+  let scaffoldResult = null;
+  if (localProblem) {
+    scaffoldResult = scaffoldProblem(localProblem, destination);
+  } else {
+    writeUpstreamOnlyBundle(String(parsed.problemId), destination, upstreamRecord, snapshot);
+  }
+
+
+  const siteStatus = await maybeWriteSiteBundle(String(parsed.problemId), destination, parsed.includeSite);
+
+  writeJson(path.join(destination, 'PULL_STATUS.json'), {
+    generatedAt: new Date().toISOString(),
+    problemId: String(parsed.problemId),
+    usedLocalDossier: Boolean(localProblem),
+    includedUpstreamRecord: Boolean(upstreamRecord),
+    upstreamSnapshotKind: snapshot?.kind ?? null,
+    siteSnapshotAttempted: siteStatus.attempted,
+    siteSnapshotIncluded: siteStatus.included,
+    siteSnapshotError: siteStatus.error,
+    scaffoldArtifactsCopied: scaffoldResult?.copiedArtifacts.length ?? 0,
+  });
+
+  console.log(`Pull bundle created: ${destination}`);
+  console.log(`Local canonical dossier included: ${localProblem ? 'yes' : 'no'}`);
+  console.log(`Upstream record included: ${upstreamRecord ? 'yes' : 'no'}`);
+  console.log(`Live site snapshot included: ${siteStatus.included ? 'yes' : 'no'}`);
+  if (siteStatus.error) {
+    console.log(`Live site snapshot note: ${siteStatus.error}`);
+  }
+  return 0;
+}

package/src/commands/scaffold.js

@@ -0,0 +1,60 @@
+import path from 'node:path';
+import { getProblem } from '../atlas/catalog.js';
+import { scaffoldProblem } from '../runtime/problem-artifacts.js';
+import { getWorkspaceProblemScaffoldDir } from '../runtime/paths.js';
+import { readCurrentProblem } from '../runtime/workspace.js';
+
+export function parseScaffoldArgs(args) {
+  const [kind, value, ...rest] = args;
+  if (kind !== 'problem') {
+    return { error: 'Only `erdos scaffold problem <id>` is supported right now.' };
+  }
+  let destination = null;
+  for (let index = 0; index < rest.length; index += 1) {
+    const token = rest[index];
+    if (token === '--dest') {
+      destination = rest[index + 1];
+      if (!destination) {
+        return { error: 'Missing destination path after --dest.' };
+      }
+      index += 1;
+      continue;
+    }
+    return { error: `Unknown scaffold option: ${token}` };
+  }
+  return { problemId: value ?? readCurrentProblem(), destination };
+}
+
+export function runScaffoldCommand(args) {
+  if (args.length === 0 || args[0] === 'help' || args[0] === '--help') {
+    console.log('Usage:');
+    console.log('  erdos scaffold problem <id> [--dest <path>]');
+    return 0;
+  }
+
+  const parsed = parseScaffoldArgs(args);
+  if (parsed.error) {
+    console.error(parsed.error);
+    return 1;
+  }
+  if (!parsed.problemId) {
+    console.error('Missing problem id and no active problem is selected.');
+    return 1;
+  }
+
+  const problem = getProblem(parsed.problemId);
+  if (!problem) {
+    console.error(`Unknown problem: ${parsed.problemId}`);
+    return 1;
+  }
+
+  const destination = parsed.destination
+    ? path.resolve(parsed.destination)
+    : getWorkspaceProblemScaffoldDir(problem.problemId);
+
+  const result = scaffoldProblem(problem, destination);
+  console.log(`Scaffold created: ${result.destination}`);
+  console.log(`Artifacts copied: ${result.copiedArtifacts.length}`);
+  console.log(`Upstream record included: ${result.inventory.upstreamRecordIncluded ? 'yes' : 'no'}`);
+  return 0;
+}

package/src/commands/upstream.js

@@ -0,0 +1,60 @@
+import { buildUpstreamDiff, loadActiveUpstreamSnapshot, syncUpstream, writeDiffArtifacts } from '../upstream/sync.js';
+
+export async function runUpstreamCommand(args) {
+  const [subcommand, ...rest] = args;
+
+  if (!subcommand || subcommand === 'help' || subcommand === '--help') {
+    console.log('Usage:');
+    console.log('  erdos upstream show');
+    console.log('  erdos upstream sync [--write-package-snapshot]');
+    console.log('  erdos upstream diff [--write-package-report]');
+    return 0;
+  }
+
+  if (subcommand === 'show') {
+    const snapshot = loadActiveUpstreamSnapshot();
+    if (!snapshot) {
+      console.log('No upstream snapshot available yet. Run `erdos upstream sync`.');
+      return 0;
+    }
+    console.log(`Snapshot kind: ${snapshot.kind}`);
+    console.log(`Upstream repo: ${snapshot.manifest.upstream_repo}`);
+    console.log(`Upstream commit: ${snapshot.manifest.upstream_commit ?? '(unknown)'}`);
+    console.log(`Fetched at: ${snapshot.manifest.fetched_at}`);
+    console.log(`Entries: ${snapshot.manifest.entry_count}`);
+    return 0;
+  }
+
+  if (subcommand === 'sync') {
+    const writePackageSnapshot = rest.includes('--write-package-snapshot');
+    const result = await syncUpstream({ writePackageSnapshot });
+    console.log(`Fetched upstream commit: ${result.snapshot.manifest.upstream_commit ?? '(unknown)'}`);
+    console.log(`Workspace snapshot: ${result.workspacePaths.manifestPath}`);
+    if (result.bundledPaths) {
+      console.log(`Bundled snapshot: ${result.bundledPaths.manifestPath}`);
+    }
+    const diffArtifacts = writeDiffArtifacts({ writePackageReport: writePackageSnapshot });
+    console.log(`Workspace diff report: ${diffArtifacts.workspaceDiffPath}`);
+    if (diffArtifacts.repoDiffPath) {
+      console.log(`Repo diff report: ${diffArtifacts.repoDiffPath}`);
+    }
+    return 0;
+  }
+
+  if (subcommand === 'diff') {
+    const writePackageReport = rest.includes('--write-package-report');
+    const diffArtifacts = writeDiffArtifacts({ writePackageReport });
+    const diff = buildUpstreamDiff();
+    console.log(`Local seeded problems: ${diff.localProblemCount}`);
+    console.log(`Upstream total problems: ${diff.upstreamProblemCount}`);
+    console.log(`Upstream-only count: ${diff.upstreamOnlyCount}`);
+    console.log(`Workspace diff report: ${diffArtifacts.workspaceDiffPath}`);
+    if (diffArtifacts.repoDiffPath) {
+      console.log(`Repo diff report: ${diffArtifacts.repoDiffPath}`);
+    }
+    return 0;
+  }
+
+  console.error(`Unknown upstream subcommand: ${subcommand}`);
+  return 1;
+}

package/src/commands/workspace.js

@@ -19,6 +19,9 @@ export function runWorkspaceCommand(args) {
   console.log(`State dir: ${summary.stateDir}`);
   console.log(`Initialized: ${summary.hasState ? 'yes' : 'no'}`);
   console.log(`Active problem: ${summary.activeProblem ?? '(none)'}`);
+  console.log(`Workspace upstream dir: ${summary.upstreamDir}`);
+  console.log(`Workspace scaffold dir: ${summary.scaffoldDir}`);
+  console.log(`Workspace pull dir: ${summary.pullDir}`);
   console.log(`Updated at: ${summary.updatedAt ?? '(none)'}`);
   return 0;
 }

package/src/runtime/files.js

@@ -0,0 +1,37 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+export function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+export function writeJson(filePath, payload) {
+  ensureDir(path.dirname(filePath));
+  fs.writeFileSync(filePath, JSON.stringify(payload, null, 2) + '\n');
+}
+
+export function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+export function writeText(filePath, text) {
+  ensureDir(path.dirname(filePath));
+  fs.writeFileSync(filePath, text);
+}
+
+export function readText(filePath) {
+  return fs.readFileSync(filePath, 'utf8');
+}
+
+export function fileExists(filePath) {
+  return fs.existsSync(filePath);
+}
+
+export function copyFileIfPresent(sourcePath, destinationPath) {
+  if (!fileExists(sourcePath)) {
+    return false;
+  }
+  ensureDir(path.dirname(destinationPath));
+  fs.copyFileSync(sourcePath, destinationPath);
+  return true;
+}

package/src/runtime/paths.js

@@ -20,6 +20,78 @@ export function getCurrentProblemPath() {
   return path.join(getWorkspaceDir(), 'current-problem.json');
 }
 
+export function getWorkspaceReportsDir() {
+  return path.join(getWorkspaceDir(), 'reports');
+}
+
+export function getWorkspaceDiffPath() {
+  return path.join(getWorkspaceReportsDir(), 'UPSTREAM_DIFF.md');
+}
+
+export function getWorkspaceUpstreamDir() {
+  return path.join(getWorkspaceDir(), 'upstream', 'erdosproblems');
+}
+
+export function getWorkspaceUpstreamYamlPath() {
+  return path.join(getWorkspaceUpstreamDir(), 'problems.yaml');
+}
+
+export function getWorkspaceUpstreamIndexPath() {
+  return path.join(getWorkspaceUpstreamDir(), 'PROBLEMS_INDEX.json');
+}
+
+export function getWorkspaceUpstreamManifestPath() {
+  return path.join(getWorkspaceUpstreamDir(), 'SYNC_MANIFEST.json');
+}
+
+export function getWorkspaceScaffoldsDir() {
+  return path.join(getWorkspaceDir(), 'scaffolds');
+}
+
+export function getWorkspaceProblemScaffoldDir(problemId) {
+  return path.join(getWorkspaceScaffoldsDir(), String(problemId));
+}
+
+export function getWorkspacePullsDir() {
+  return path.join(getWorkspaceDir(), 'pulls');
+}
+
+export function getWorkspaceProblemPullDir(problemId) {
+  return path.join(getWorkspacePullsDir(), String(problemId));
+}
+
 export function getProblemDir(problemId) {
   return path.join(repoRoot, 'problems', String(problemId));
 }
+
+export function getBundledDataDir() {
+  return path.join(repoRoot, 'data');
+}
+
+export function getBundledUpstreamDir() {
+  return path.join(getBundledDataDir(), 'upstream', 'erdosproblems');
+}
+
+export function getBundledUpstreamYamlPath() {
+  return path.join(getBundledUpstreamDir(), 'problems.yaml');
+}
+
+export function getBundledUpstreamIndexPath() {
+  return path.join(getBundledUpstreamDir(), 'PROBLEMS_INDEX.json');
+}
+
+export function getBundledUpstreamManifestPath() {
+  return path.join(getBundledUpstreamDir(), 'SYNC_MANIFEST.json');
+}
+
+export function getRepoAnalysisDir() {
+  return path.join(repoRoot, 'analysis');
+}
+
+export function getRepoUpstreamDiffPath() {
+  return path.join(getRepoAnalysisDir(), 'UPSTREAM_DIFF.md');
+}
+
+export function getPackDir(packName) {
+  return path.join(repoRoot, 'packs', String(packName));
+}

package/src/runtime/problem-artifacts.js

@@ -0,0 +1,150 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { loadActiveUpstreamSnapshot } from '../upstream/sync.js';
+import { copyFileIfPresent, ensureDir, writeJson, writeText } from './files.js';
+import { getPackDir } from './paths.js';
+
+const DOSSIER_FILES = [
+  ['problem.yaml', 'problemYamlPath', 'problem.yaml'],
+  ['STATEMENT.md', 'statementPath', 'STATEMENT.md'],
+  ['REFERENCES.md', 'referencesPath', 'REFERENCES.md'],
+  ['EVIDENCE.md', 'evidencePath', 'EVIDENCE.md'],
+  ['FORMALIZATION.md', 'formalizationPath', 'FORMALIZATION.md'],
+];
+
+function getPackContextPath(problem) {
+  if (!problem.cluster) {
+    return null;
+  }
+  return path.join(getPackDir(problem.cluster), 'README.md');
+}
+
+export function getProblemArtifactInventory(problem) {
+  const snapshot = loadActiveUpstreamSnapshot();
+  const upstreamRecord = snapshot?.index?.by_number?.[problem.problemId] ?? null;
+  const canonicalArtifacts = DOSSIER_FILES.map(([label, key, destinationName]) => {
+    const filePath = problem[key];
+    return {
+      label,
+      path: filePath,
+      destinationName,
+      exists: fs.existsSync(filePath),
+    };
+  });
+
+  const packContextPath = getPackContextPath(problem);
+  const packContext = packContextPath
+    ? {
+        label: 'PACK_CONTEXT.md',
+        path: packContextPath,
+        destinationName: 'PACK_CONTEXT.md',
+        exists: fs.existsSync(packContextPath),
+      }
+    : null;
+
+  return {
+    generatedAt: new Date().toISOString(),
+    problemId: problem.problemId,
+    displayName: problem.displayName,
+    title: problem.title,
+    sourceUrl: problem.sourceUrl,
+    cluster: problem.cluster,
+    repoStatus: problem.repoStatus,
+    harnessDepth: problem.harnessDepth,
+    problemDir: problem.problemDir,
+    canonicalArtifacts,
+    packContext,
+    upstreamSnapshot: snapshot
+      ? {
+          kind: snapshot.kind,
+          manifestPath: snapshot.manifestPath,
+          indexPath: snapshot.indexPath,
+          yamlPath: snapshot.yamlPath,
+          upstreamCommit: snapshot.manifest.upstream_commit ?? null,
+          fetchedAt: snapshot.manifest.fetched_at,
+        }
+      : null,
+    upstreamRecordIncluded: Boolean(upstreamRecord),
+    upstreamRecord,
+  };
+}
+
+export function scaffoldProblem(problem, destination) {
+  ensureDir(destination);
+  const inventory = getProblemArtifactInventory(problem);
+
+  const copiedArtifacts = [];
+  for (const artifact of inventory.canonicalArtifacts) {
+    const destinationPath = path.join(destination, artifact.destinationName);
+    if (copyFileIfPresent(artifact.path, destinationPath)) {
+      copiedArtifacts.push({
+        label: artifact.label,
+        sourcePath: artifact.path,
+        destinationPath,
+      });
+    }
+  }
+
+  if (inventory.packContext?.exists) {
+    const destinationPath = path.join(destination, inventory.packContext.destinationName);
+    if (copyFileIfPresent(inventory.packContext.path, destinationPath)) {
+      copiedArtifacts.push({
+        label: inventory.packContext.label,
+        sourcePath: inventory.packContext.path,
+        destinationPath,
+      });
+    }
+  }
+
+  if (inventory.upstreamRecord) {
+    writeJson(path.join(destination, 'UPSTREAM_RECORD.json'), inventory.upstreamRecord);
+  }
+
+  const problemRecord = {
+    generatedAt: inventory.generatedAt,
+    problemId: problem.problemId,
+    title: problem.title,
+    cluster: problem.cluster,
+    siteStatus: problem.siteStatus,
+    repoStatus: problem.repoStatus,
+    harnessDepth: problem.harnessDepth,
+    sourceUrl: problem.sourceUrl,
+    activeRoute: problem.researchState?.active_route ?? null,
+  };
+
+  const artifactIndex = {
+    generatedAt: inventory.generatedAt,
+    problemId: problem.problemId,
+    bundledProblemDir: problem.problemDir,
+    copiedArtifacts,
+    packContext: inventory.packContext,
+    canonicalArtifacts: inventory.canonicalArtifacts,
+    upstreamSnapshot: inventory.upstreamSnapshot,
+    includedUpstreamRecord: inventory.upstreamRecordIncluded,
+  };
+
+  writeJson(path.join(destination, 'PROBLEM.json'), problemRecord);
+  writeJson(path.join(destination, 'ARTIFACT_INDEX.json'), artifactIndex);
+  writeText(
+    path.join(destination, 'README.md'),
+    [
+      `# Erdos Problem ${problem.problemId} Scaffold`,
+      '',
+      'This scaffold was generated by the erdos CLI.',
+      '',
+      `- Title: ${problem.title}`,
+      `- Cluster: ${problem.cluster}`,
+      `- Source: ${problem.sourceUrl}`,
+      `- Repo status: ${problem.repoStatus}`,
+      `- Harness depth: ${problem.harnessDepth}`,
+      `- Upstream record included: ${inventory.upstreamRecordIncluded ? 'yes' : 'no'}`,
+      '',
+    ].join('\n'),
+  );
+
+  return {
+    destination,
+    copiedArtifacts,
+    inventory,
+  };
+}

package/src/runtime/workspace.js

@@ -1,16 +1,14 @@
 import fs from 'node:fs';
-import path from 'node:path';
 import {
   getCurrentProblemPath,
   getWorkspaceDir,
+  getWorkspaceProblemPullDir,
+  getWorkspaceProblemScaffoldDir,
   getWorkspaceRoot,
   getWorkspaceStatePath,
+  getWorkspaceUpstreamDir,
 } from './paths.js';
-
-function writeJson(filePath, payload) {
-  fs.mkdirSync(path.dirname(filePath), { recursive: true });
-  fs.writeFileSync(filePath, JSON.stringify(payload, null, 2) + '\n');
-}
+import { writeJson } from './files.js';
 
 export function readWorkspaceState() {
   const filePath = getWorkspaceStatePath();
@@ -61,11 +59,15 @@ export function readCurrentProblem() {
 
 export function getWorkspaceSummary() {
   const state = readWorkspaceState();
+  const activeProblem = readCurrentProblem();
   return {
     workspaceRoot: getWorkspaceRoot(),
     stateDir: getWorkspaceDir(),
     hasState: Boolean(state),
-    activeProblem: readCurrentProblem(),
+    activeProblem,
+    upstreamDir: getWorkspaceUpstreamDir(),
+    scaffoldDir: activeProblem ? getWorkspaceProblemScaffoldDir(activeProblem) : getWorkspaceProblemScaffoldDir('<problem-id>'),
+    pullDir: activeProblem ? getWorkspaceProblemPullDir(activeProblem) : getWorkspaceProblemPullDir('<problem-id>'),
     updatedAt: state?.updatedAt ?? null,
   };
 }