erdos-problems 0.1.2 → 0.1.3

package/README.md

@@ -18,9 +18,10 @@ Official binary:
 
 - atlas layer with canonical local `problems/<id>/problem.yaml` records
 - bundled upstream snapshot from `teorth/erdosproblems`
-- workspace `.erdos/` state for active-problem selection, upstream refreshes, reports, and scaffolds
+- workspace `.erdos/` state for active-problem selection, upstream refreshes, reports, scaffolds, and pull bundles
 - sunflower cluster as the first deep harness pack
 - seeded atlas now includes open and solved problems beyond sunflower
+- unseeded problems can still be pulled into a workspace from the bundled upstream snapshot
 
 Seeded problems:
 - `18`, `20`, `89`, `536`, `542`, `856`, `857`, `1008`
@@ -30,7 +31,7 @@ Seeded problems:
 ```bash
 erdos problem list --cluster sunflower
 erdos bootstrap problem 857
-erdos problem artifacts 857
+erdos problem artifacts 857 --json
 erdos dossier show 857
 ```
 
@@ -40,6 +41,22 @@ What `bootstrap` does:
 - includes the upstream record when a bundled or workspace snapshot is available
 - gives an agent a ready-to-read local artifact bundle immediately after install
 
+## Pull bundles
+
+For any problem number in the upstream snapshot, you can create a workspace bundle even if the problem is not yet seeded locally:
+
+```bash
+erdos pull problem 857
+erdos pull problem 999 --include-site
+erdos pull problem 999 --refresh-upstream
+```
+
+What `pull` does:
+- creates `.erdos/pulls/<id>/`
+- includes the upstream record when available
+- includes the local canonical dossier too when the problem is seeded locally
+- can optionally add a live site snapshot and plain-text extract
+
 ## CLI
 
 ```bash
@@ -62,6 +79,8 @@ erdos upstream diff
 erdos scaffold problem 857
 erdos bootstrap problem 857
 erdos bootstrap problem 857 --sync-upstream
+erdos pull problem 857
+erdos pull problem 857 --include-site
 ```
 
 ## Canonical Sources
@@ -84,8 +103,9 @@ For each seeded problem, the canonical local dossier lives in `problems/<id>/`:
 The CLI can surface these directly:
 - `erdos problem artifacts <id>` shows the canonical inventory
 - `erdos problem artifacts <id> --json` emits machine-readable inventory
-- `erdos scaffold problem <id>` copies the bundle into the active workspace
+- `erdos scaffold problem <id>` copies the seeded dossier into the active workspace
 - `erdos bootstrap problem <id>` selects the problem and creates the scaffold in one step
+- `erdos pull problem <id>` creates a workspace bundle for any problem in the upstream snapshot
 
 ## Notes
 

package/docs/ERDOS_PROBLEMS_PROBLEM_SCHEMA.md

@@ -12,10 +12,11 @@ The goal is:
 - open and solved problems use the same shape
 - local dossier truth and upstream public truth stay explicitly separated
 - packaged CLI installs can scaffold problem workspaces from canonical artifacts immediately
+- unseeded problems can still be pulled into a workspace bundle from upstream truth
 
 ## Canonical Files
 
-Each problem should have:
+Each seeded problem should have:
 
 - `problems/<id>/problem.yaml`
 - `problems/<id>/STATEMENT.md`
@@ -29,6 +30,12 @@ Bundled upstream snapshot artifacts live in:
 - `data/upstream/erdosproblems/PROBLEMS_INDEX.json`
 - `data/upstream/erdosproblems/SYNC_MANIFEST.json`
 
+Workspace-generated artifacts may live in:
+
+- `.erdos/scaffolds/<id>/`
+- `.erdos/pulls/<id>/`
+- `.erdos/upstream/erdosproblems/`
+
 ## Canonical Truth Split
 
 ### External public truth
@@ -178,4 +185,15 @@ The sync commands should produce:
 - upstream record snapshot for that problem when available
 - generated artifact index for agent consumption
 
-This makes a fresh npm-installed CLI immediately useful to an agentic workflow.
+This is the seeded-problem path.
+
+## Pull Contract
+
+`erdos pull problem <id>` should create a broader workspace-ready bundle containing:
+
+- upstream record snapshot for that problem when available
+- generated artifact index for agent consumption
+- seeded local dossier files too when the problem already exists in `problems/<id>/`
+- optional live site snapshot and extracted text when `--include-site` is used
+
+This makes a fresh npm-installed CLI immediately useful to an agentic workflow even for problems that are not yet fully seeded as local dossiers.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "erdos-problems",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "CLI atlas and staged research harness for Paul Erdos problems.",
   "type": "module",
   "bin": {

package/src/cli/index.js

@@ -2,6 +2,7 @@ import { runBootstrapCommand } from '../commands/bootstrap.js';
 import { runClusterCommand } from '../commands/cluster.js';
 import { runDossierCommand } from '../commands/dossier.js';
 import { runProblemCommand } from '../commands/problem.js';
+import { runPullCommand } from '../commands/pull.js';
 import { runScaffoldCommand } from '../commands/scaffold.js';
 import { runUpstreamCommand } from '../commands/upstream.js';
 import { runWorkspaceCommand } from '../commands/workspace.js';
@@ -24,6 +25,7 @@ function printUsage() {
   console.log('  erdos upstream diff [--write-package-report]');
   console.log('  erdos scaffold problem <id> [--dest <path>]');
   console.log('  erdos bootstrap problem <id> [--dest <path>] [--sync-upstream]');
+  console.log('  erdos pull problem <id> [--dest <path>] [--include-site] [--refresh-upstream]');
 }
 
 const args = process.argv.slice(2);
@@ -47,6 +49,8 @@ if (!command || command === 'help' || command === '--help') {
   exitCode = runScaffoldCommand(rest);
 } else if (command === 'bootstrap') {
   exitCode = await runBootstrapCommand(rest);
+} else if (command === 'pull') {
+  exitCode = await runPullCommand(rest);
 } else {
   console.error(`Unknown command: ${command}`);
   printUsage();

package/src/commands/pull.js

@@ -0,0 +1,203 @@
+import path from 'node:path';
+import { getProblem } from '../atlas/catalog.js';
+import { ensureDir, writeJson, writeText } from '../runtime/files.js';
+import { getWorkspaceProblemPullDir } from '../runtime/paths.js';
+import { scaffoldProblem } from '../runtime/problem-artifacts.js';
+import { loadActiveUpstreamSnapshot, syncUpstream } from '../upstream/sync.js';
+import { fetchProblemSiteSnapshot } from '../upstream/site.js';
+
+function parsePullArgs(args) {
+  const [kind, value, ...rest] = args;
+  if (kind !== 'problem') {
+    return { error: 'Only `erdos pull problem <id>` is supported right now.' };
+  }
+
+  let destination = null;
+  let includeSite = false;
+  let refreshUpstream = false;
+
+  for (let index = 0; index < rest.length; index += 1) {
+    const token = rest[index];
+    if (token === '--dest') {
+      destination = rest[index + 1];
+      if (!destination) {
+        return { error: 'Missing destination path after --dest.' };
+      }
+      index += 1;
+      continue;
+    }
+    if (token === '--include-site') {
+      includeSite = true;
+      continue;
+    }
+    if (token === '--refresh-upstream') {
+      refreshUpstream = true;
+      continue;
+    }
+    return { error: `Unknown pull option: ${token}` };
+  }
+
+  return {
+    problemId: value,
+    destination,
+    includeSite,
+    refreshUpstream,
+  };
+}
+
+function writeUpstreamOnlyBundle(problemId, destination, upstreamRecord, snapshot) {
+  ensureDir(destination);
+
+  if (upstreamRecord) {
+    writeJson(path.join(destination, 'UPSTREAM_RECORD.json'), upstreamRecord);
+  }
+
+  const generatedAt = new Date().toISOString();
+  writeJson(path.join(destination, 'PROBLEM.json'), {
+    generatedAt,
+    problemId,
+    title: `Erdos Problem #${problemId}`,
+    cluster: null,
+    siteStatus: upstreamRecord?.status?.state ?? 'unknown',
+    repoStatus: 'upstream-only',
+    harnessDepth: 'unseeded',
+    sourceUrl: `https://www.erdosproblems.com/${problemId}`,
+    activeRoute: null,
+  });
+
+  writeJson(path.join(destination, 'ARTIFACT_INDEX.json'), {
+    generatedAt,
+    problemId,
+    copiedArtifacts: [],
+    canonicalArtifacts: [],
+    upstreamSnapshot: snapshot
+      ? {
+          kind: snapshot.kind,
+          manifestPath: snapshot.manifestPath,
+          indexPath: snapshot.indexPath,
+          yamlPath: snapshot.yamlPath,
+          upstreamCommit: snapshot.manifest.upstream_commit ?? null,
+          fetchedAt: snapshot.manifest.fetched_at,
+        }
+      : null,
+    includedUpstreamRecord: Boolean(upstreamRecord),
+  });
+
+  writeText(
+    path.join(destination, 'README.md'),
+    [
+      `# Erdos Problem ${problemId} Pull Bundle`,
+      '',
+      'This bundle was generated from upstream public metadata.',
+      '',
+      `- Source: https://www.erdosproblems.com/${problemId}`,
+      `- Upstream record included: ${upstreamRecord ? 'yes' : 'no'}`,
+      '',
+      'This problem is not yet seeded locally as a canonical dossier in this package.',
+      '',
+    ].join('\n'),
+  );
+}
+
+async function maybeWriteSiteBundle(problemId, destination, includeSite) {
+  if (!includeSite) {
+    return { attempted: false, included: false, error: null };
+  }
+
+  try {
+    const siteSnapshot = await fetchProblemSiteSnapshot(problemId);
+    writeText(path.join(destination, 'SITE_SNAPSHOT.html'), siteSnapshot.html);
+    writeText(path.join(destination, 'SITE_EXTRACT.txt'), siteSnapshot.text);
+    writeJson(path.join(destination, 'SITE_EXTRACT.json'), {
+      url: siteSnapshot.url,
+      fetchedAt: siteSnapshot.fetchedAt,
+      title: siteSnapshot.title,
+      previewLines: siteSnapshot.previewLines,
+    });
+    writeText(
+      path.join(destination, 'SITE_SUMMARY.md'),
+      [
+        `# Erdős Problem #${problemId} Site Summary`,
+        '',
+        `Source: ${siteSnapshot.url}`,
+        `Fetched at: ${siteSnapshot.fetchedAt}`,
+        `Title: ${siteSnapshot.title}`,
+        '',
+        '## Preview',
+        '',
+        ...siteSnapshot.previewLines.map((line) => `- ${line}`),
+        '',
+      ].join('\n'),
+    );
+    return { attempted: true, included: true, error: null };
+  } catch (error) {
+    writeText(path.join(destination, 'SITE_FETCH_ERROR.txt'), String(error.message ?? error));
+    return { attempted: true, included: false, error: String(error.message ?? error) };
+  }
+}
+
+export async function runPullCommand(args) {
+  if (args.length === 0 || args[0] === 'help' || args[0] === '--help') {
+    console.log('Usage:');
+    console.log('  erdos pull problem <id> [--dest <path>] [--include-site] [--refresh-upstream]');
+    return 0;
+  }
+
+  const parsed = parsePullArgs(args);
+  if (parsed.error) {
+    console.error(parsed.error);
+    return 1;
+  }
+  if (!parsed.problemId) {
+    console.error('Missing problem id.');
+    return 1;
+  }
+
+  if (parsed.refreshUpstream) {
+    await syncUpstream();
+  }
+
+  const localProblem = getProblem(parsed.problemId);
+  const snapshot = loadActiveUpstreamSnapshot();
+  const upstreamRecord = snapshot?.index?.by_number?.[String(parsed.problemId)] ?? null;
+
+  if (!localProblem && !upstreamRecord) {
+    console.error(`Problem ${parsed.problemId} is not present in the local dossier set or upstream snapshot.`);
+    return 1;
+  }
+
+  const destination = parsed.destination
+    ? path.resolve(parsed.destination)
+    : getWorkspaceProblemPullDir(parsed.problemId);
+
+  let scaffoldResult = null;
+  if (localProblem) {
+    scaffoldResult = scaffoldProblem(localProblem, destination);
+  } else {
+    writeUpstreamOnlyBundle(String(parsed.problemId), destination, upstreamRecord, snapshot);
+  }
+
+
+  const siteStatus = await maybeWriteSiteBundle(String(parsed.problemId), destination, parsed.includeSite);
+
+  writeJson(path.join(destination, 'PULL_STATUS.json'), {
+    generatedAt: new Date().toISOString(),
+    problemId: String(parsed.problemId),
+    usedLocalDossier: Boolean(localProblem),
+    includedUpstreamRecord: Boolean(upstreamRecord),
+    upstreamSnapshotKind: snapshot?.kind ?? null,
+    siteSnapshotAttempted: siteStatus.attempted,
+    siteSnapshotIncluded: siteStatus.included,
+    siteSnapshotError: siteStatus.error,
+    scaffoldArtifactsCopied: scaffoldResult?.copiedArtifacts.length ?? 0,
+  });
+
+  console.log(`Pull bundle created: ${destination}`);
+  console.log(`Local canonical dossier included: ${localProblem ? 'yes' : 'no'}`);
+  console.log(`Upstream record included: ${upstreamRecord ? 'yes' : 'no'}`);
+  console.log(`Live site snapshot included: ${siteStatus.included ? 'yes' : 'no'}`);
+  if (siteStatus.error) {
+    console.log(`Live site snapshot note: ${siteStatus.error}`);
+  }
+  return 0;
+}

package/src/commands/workspace.js

@@ -21,6 +21,7 @@ export function runWorkspaceCommand(args) {
   console.log(`Active problem: ${summary.activeProblem ?? '(none)'}`);
   console.log(`Workspace upstream dir: ${summary.upstreamDir}`);
   console.log(`Workspace scaffold dir: ${summary.scaffoldDir}`);
+  console.log(`Workspace pull dir: ${summary.pullDir}`);
   console.log(`Updated at: ${summary.updatedAt ?? '(none)'}`);
   return 0;
 }

package/src/runtime/paths.js

@@ -52,6 +52,14 @@ export function getWorkspaceProblemScaffoldDir(problemId) {
   return path.join(getWorkspaceScaffoldsDir(), String(problemId));
 }
 
+export function getWorkspacePullsDir() {
+  return path.join(getWorkspaceDir(), 'pulls');
+}
+
+export function getWorkspaceProblemPullDir(problemId) {
+  return path.join(getWorkspacePullsDir(), String(problemId));
+}
+
 export function getProblemDir(problemId) {
   return path.join(repoRoot, 'problems', String(problemId));
 }

package/src/runtime/workspace.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import {
   getCurrentProblemPath,
   getWorkspaceDir,
+  getWorkspaceProblemPullDir,
   getWorkspaceProblemScaffoldDir,
   getWorkspaceRoot,
   getWorkspaceStatePath,
@@ -66,6 +67,7 @@ export function getWorkspaceSummary() {
     activeProblem,
     upstreamDir: getWorkspaceUpstreamDir(),
     scaffoldDir: activeProblem ? getWorkspaceProblemScaffoldDir(activeProblem) : getWorkspaceProblemScaffoldDir('<problem-id>'),
+    pullDir: activeProblem ? getWorkspaceProblemPullDir(activeProblem) : getWorkspaceProblemPullDir('<problem-id>'),
     updatedAt: state?.updatedAt ?? null,
   };
 }

package/src/upstream/site.js

@@ -0,0 +1,80 @@
+const SITE_BASE_URL = 'https://www.erdosproblems.com';
+
+function decodeEntities(text) {
+  return text
+    .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
+    .replace(/&#(\d+);/g, (_, decimal) => String.fromCodePoint(Number.parseInt(decimal, 10)))
+    .replace(/ /g, ' ')
+    .replace(/&/g, '&')
+    .replace(/"/g, '"')
+    .replace(/'/g, "'")
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>');
+}
+
+function collapseWhitespace(text) {
+  return text.replace(/[ \t]+/g, ' ').replace(/\s*\n\s*/g, '\n').trim();
+}
+
+function htmlToReadableText(html) {
+  const withoutScripts = html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ');
+  const blockSeparated = withoutScripts
+    .replace(/<(br|\/p|\/div|\/li|\/h1|\/h2|\/h3|\/section|\/article|\/tr)>/gi, '\n')
+    .replace(/<li[^>]*>/gi, '- ')
+    .replace(/<p[^>]*>/gi, '\n')
+    .replace(/<div[^>]*>/gi, '\n')
+    .replace(/<h[1-6][^>]*>/gi, '\n');
+  const stripped = blockSeparated.replace(/<[^>]+>/g, ' ');
+  const decoded = decodeEntities(stripped);
+  const normalizedLines = decoded
+    .split('\n')
+    .map((line) => collapseWhitespace(line))
+    .filter(Boolean);
+  return normalizedLines.join('\n');
+}
+
+function extractTitle(html, problemId) {
+  const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  if (!match) {
+    return `Erdos Problem #${problemId}`;
+  }
+  return collapseWhitespace(decodeEntities(match[1]));
+}
+
+function selectPreviewLines(lines) {
+  const anchorIndex = lines.findIndex((line) => /^(OPEN|SOLVED|PROVED|PARTIAL)\b/i.test(line));
+  if (anchorIndex >= 0) {
+    return lines.slice(anchorIndex, anchorIndex + 24);
+  }
+  return lines.slice(0, 24);
+}
+
+export async function fetchProblemSiteSnapshot(problemId) {
+  const url = `${SITE_BASE_URL}/${problemId}`;
+  const response = await fetch(url, {
+    headers: {
+      'User-Agent': 'erdos-problems-cli',
+      Accept: 'text/html',
+    },
+  });
+
+  if (!response.ok) {
+    throw new Error(`Unable to fetch problem page ${problemId}: ${response.status}`);
+  }
+
+  const html = await response.text();
+  const text = htmlToReadableText(html);
+  const title = extractTitle(html, problemId);
+  const lines = text.split('\n').filter(Boolean);
+
+  return {
+    url,
+    fetchedAt: new Date().toISOString(),
+    html,
+    title,
+    text,
+    previewLines: selectPreviewLines(lines),
+  };
+}