@ktpartners/dgs-platform 2.9.0 → 3.3.0

package/deliver-great-systems/bin/lib/jobs.cjs

@@ -33,6 +33,7 @@ const { execSync } = require('child_process');
 const { output, error, getProjectRoot } = require('./core.cjs');
 const { getPlanningRoot } = require('./paths.cjs');
 const { requireGitIdentity, formatAuthorString } = require('./identity.cjs');
+const { extractFrontmatter, spliceFrontmatter, reconstructFrontmatter } = require('./frontmatter.cjs');
 
 // ─── Constants ──────────────────────────────────────────────────────────────
 
@@ -48,6 +49,7 @@ const KNOWN_COMMANDS = new Set([
   'plan-milestone-gaps',
   'discuss-phase',
   'research-phase',
+  'validate-phase',
   'verify-phase',
 ]);
 
@@ -92,6 +94,8 @@ const STATUS_TO_MARKER = {
   'pending': ' ',
 };
 
+const JOB_STATUSES = ['pending', 'in-progress', 'completed', 'failed', 'rolled_back'];
+
 // ─── Internal Helpers ───────────────────────────────────────────────────────
 
 /**
@@ -339,6 +343,9 @@ function updateJobStep(filePath, stepIndex, status, options) {
 }
 
 /**
+ * @deprecated Use setJobStatus() for state transitions instead of physical file moves.
+ * Kept for backward compatibility with legacy directory-based job layouts.
+ *
  * Move a job file from its current location to a target directory.
  *
  * @param {string} filePath - Absolute path to the job file
@@ -374,13 +381,31 @@ function moveJobFile(filePath, targetDir) {
 }
 
 /**
- * Locate a job file by milestone version, checking directories in priority order.
+ * Read status from a job file: YAML frontmatter first, then bold-key header fallback.
  *
- * Search order: in-progress/ (resume case), pending/ (new job), completed/ (inspection).
+ * @param {string} content - File content
+ * @returns {string|null} Status value or null
+ */
+function _readJobStatus(content) {
+  // Try YAML frontmatter first
+  const fmMatch = content.match(/^---\n([\s\S]+?)\n---/);
+  if (fmMatch) {
+    const statusMatch = fmMatch[1].match(/^status:\s*(.+)$/m);
+    if (statusMatch) return statusMatch[1].trim();
+  }
+  // Fall back to bold-key header: **Status:** value
+  const headerMatch = content.match(/\*\*Status:\*\*\s*(.+)/);
+  if (headerMatch) return headerMatch[1].trim().toLowerCase();
+  return null;
+}
+
+/**
+ * Locate a job file by milestone version.
+ * Scans flat jobs/ directory first (frontmatter status), then falls back to legacy subdirectories.
  *
  * @param {string} cwd - Working directory
  * @param {string} version - Milestone version (e.g., "v6.0" or "6.0")
- * @returns {{ found: boolean, path?: string, directory?: string }}
+ * @returns {{ found: boolean, path?: string, directory?: string, status?: string }}
  */
 function findJobFile(cwd, version) {
   // Normalize version: ensure 'v' prefix
@@ -388,25 +413,56 @@ function findJobFile(cwd, version) {
   const fileName = `milestone-${normalized}.md`;
   const jobsDir = path.join(getPlanningRoot(cwd), 'jobs');
 
-  // Search order: in-progress first (resume case), then pending, then completed (inspection)
+  // Flat-first scan: check jobs/ root directory
+  const flatPath = path.join(jobsDir, fileName);
+  if (fs.existsSync(flatPath)) {
+    const content = fs.readFileSync(flatPath, 'utf-8');
+    const status = _readJobStatus(content);
+    return { found: true, path: flatPath, directory: null, status: status || 'pending' };
+  }
+
+  // Legacy subdirectory scan
   for (const dir of ['in-progress', 'pending', 'completed']) {
     const filePath = path.join(jobsDir, dir, fileName);
     if (fs.existsSync(filePath)) {
-      return { found: true, path: filePath, directory: dir };
+      // Legacy fallback warning
+      process.stderr.write(`[DGS] Warning: job '${fileName}' found in legacy ${dir}/ directory. Run migration to flatten.\n`);
+
+      // Read frontmatter status if available
+      const content = fs.readFileSync(filePath, 'utf-8');
+      const fmStatus = _readJobStatus(content);
+
+      if (fmStatus && fmStatus !== dir && fmStatus !== dir.replace(/-/g, '_')) {
+        process.stderr.write(`[DGS] Warning: job '${fileName}' frontmatter status '${fmStatus}' disagrees with directory '${dir}'. Frontmatter wins.\n`);
+      }
+
+      const effectiveStatus = fmStatus || dir;
+      return { found: true, path: filePath, directory: dir, status: effectiveStatus };
     }
   }
 
-  // Search project subdirectories: projects/*/jobs/
+  // Search project subdirectories: projects/*/jobs/ (flat first, then legacy)
   const projectsDir = path.join(getPlanningRoot(cwd), 'projects');
   if (fs.existsSync(projectsDir)) {
     const projects = fs.readdirSync(projectsDir).filter(d =>
       fs.statSync(path.join(projectsDir, d)).isDirectory()
     );
     for (const project of projects) {
+      // Flat first
+      const projFlatPath = path.join(projectsDir, project, 'jobs', fileName);
+      if (fs.existsSync(projFlatPath)) {
+        const content = fs.readFileSync(projFlatPath, 'utf-8');
+        const status = _readJobStatus(content);
+        return { found: true, path: projFlatPath, directory: null, status: status || 'pending' };
+      }
+      // Legacy subdirectory
       for (const dir of ['in-progress', 'pending', 'completed']) {
         const filePath = path.join(projectsDir, project, 'jobs', dir, fileName);
         if (fs.existsSync(filePath)) {
-          return { found: true, path: filePath, directory: dir };
+          process.stderr.write(`[DGS] Warning: job '${fileName}' found in legacy ${dir}/ directory. Run migration to flatten.\n`);
+          const content = fs.readFileSync(filePath, 'utf-8');
+          const fmStatus = _readJobStatus(content);
+          return { found: true, path: filePath, directory: dir, status: fmStatus || dir };
         }
       }
     }
@@ -628,6 +684,53 @@ function insertGapFixSection(filePath, afterStepIndex, cycleNumber, newPhases, v
  * @param {string} phasesJson - JSON string of phases array [{number, name}]
  * @param {boolean} raw - Raw output mode
  */
+/**
+ * Set a job's status in both YAML frontmatter and bold-key header (internal helper — throws on error).
+ *
+ * Dual-write strategy: YAML frontmatter is added/updated as the source of truth,
+ * and the existing **Status:** markdown header is kept in sync for human readability.
+ *
+ * @param {string} cwd - Working directory
+ * @param {string} version - Milestone version (e.g., "v20.0")
+ * @param {string} status - Target status (must be in JOB_STATUSES)
+ * @returns {{ version: string, previous_status: string, status: string, path: string }}
+ * @throws {Error} If status is invalid or job not found
+ */
+function setJobStatus(cwd, version, status) {
+  if (!JOB_STATUSES.includes(status)) {
+    throw new Error(`Invalid status: ${status}. Allowed: ${JOB_STATUSES.join(', ')}`);
+  }
+
+  const found = findJobFile(cwd, version);
+  if (!found.found) {
+    throw new Error(`Job not found: ${version}`);
+  }
+
+  let content = fs.readFileSync(found.path, 'utf-8');
+  const previousHeader = parseHeader(content);
+  const previousStatus = previousHeader.status;
+
+  // 1. Update bold-key markdown header (existing mechanism)
+  updateJobHeader(found.path, 'Status', status);
+
+  // 2. Update or add YAML frontmatter
+  content = fs.readFileSync(found.path, 'utf-8');
+  const fm = extractFrontmatter(content);
+  fm.status = status;
+
+  if (content.match(/^---\n/)) {
+    // Has frontmatter — splice it
+    content = spliceFrontmatter(content, fm);
+  } else {
+    // No frontmatter — add it at the very top
+    const yamlStr = reconstructFrontmatter({ status });
+    content = `---\n${yamlStr}\n---\n\n${content}`;
+  }
+  fs.writeFileSync(found.path, content, 'utf-8');
+
+  return { version, previous_status: previousStatus, status, path: found.path };
+}
+
 function cmdJobsInsertGapFixSection(cwd, file, afterIndexStr, cycleStr, version, phasesJson, raw) {
   if (!file || afterIndexStr === undefined || cycleStr === undefined || !version || !phasesJson) {
     error('Usage: jobs insert-gap-fix-section <file> <after-index> <cycle> <version> <phases-json>');
@@ -956,9 +1059,11 @@ function generateMilestoneSteps(phases, options) {
       steps.push({ command: 'map-codebase', args: `${num} --auto` });
       steps.push({ command: 'plan-phase', args: `${num} --non-interactive` });
       steps.push({ command: 'execute-phase', args: `${num} --non-interactive` });
+      steps.push({ command: 'validate-phase', args: `${num} --scaffold` });
       steps.push({ command: 'audit-phase', args: `${num}` });
     } else if (NEEDS_EXECUTION.has(phase.disk_status)) {
       steps.push({ command: 'execute-phase', args: `${num} --non-interactive` });
+      steps.push({ command: 'validate-phase', args: `${num} --scaffold` });
       steps.push({ command: 'audit-phase', args: `${num}` });
     }
   }
@@ -966,7 +1071,6 @@ function generateMilestoneSteps(phases, options) {
   // Append audit + complete steps if check is enabled
   if (check) {
     steps.push({ command: 'audit-milestone', args: version });
-    steps.push({ command: 'complete-milestone', args: version });
   }
 
   return steps;
@@ -984,7 +1088,10 @@ function generateMilestoneSteps(phases, options) {
 function buildJobFileContent(version, check, steps, createdBy) {
   const created = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
 
-  let content = `# Milestone Job: ${version}\n\n`;
+  // YAML frontmatter (source of truth for status)
+  let content = `---\nstatus: pending\n---\n\n`;
+
+  content += `# Milestone Job: ${version}\n\n`;
   content += `**Version:** ${version}\n`;
   content += `**Created:** ${created}\n`;
   if (createdBy) {
@@ -1219,11 +1326,11 @@ function cmdJobsCreateMilestone(cwd, version, check, raw) {
   // Build file content
   const content = buildJobFileContent(resolvedVersion, check, steps, createdBy);
 
-  // Write to pending directory
-  const pendingDir = path.join(getPlanningRoot(cwd), 'jobs', 'pending');
-  fs.mkdirSync(pendingDir, { recursive: true });
+  // Write to flat jobs/ directory (status tracked in frontmatter)
+  const jobsDir = path.join(getPlanningRoot(cwd), 'jobs');
+  fs.mkdirSync(jobsDir, { recursive: true });
   const fileName = `milestone-${resolvedVersion}.md`;
-  const filePath = path.join(pendingDir, fileName);
+  const filePath = path.join(jobsDir, fileName);
   fs.writeFileSync(filePath, content, 'utf-8');
 
   // Count phases that contribute at least one step
@@ -1234,7 +1341,7 @@ function cmdJobsCreateMilestone(cwd, version, check, raw) {
   const result = {
     created: true,
     version: resolvedVersion,
-    file: path.join(path.relative(cwd, getPlanningRoot(cwd)) || '.', 'jobs', 'pending', fileName),
+    file: path.join(path.relative(cwd, getPlanningRoot(cwd)) || '.', 'jobs', fileName),
     step_count: steps.length,
     check,
     phase_count: phasesWithSteps.length,
@@ -1357,7 +1464,7 @@ function listJobs(cwd) {
 }
 
 /**
- * Cancel an in-progress job: reset [>] steps to [ ], set Status to pending, move to pending/.
+ * Cancel an in-progress job: reset [>] steps to [ ], set Status to pending.
  *
  * @param {string} cwd - Working directory
  * @param {string} version - Milestone version (e.g., "v6.0")
@@ -1368,7 +1475,7 @@ function cancelJob(cwd, version) {
   if (!found.found) {
     return { cancelled: false, reason: 'not_found' };
   }
-  if (found.directory !== 'in-progress') {
+  if (found.status !== 'in-progress') {
     return { cancelled: false, reason: 'not_in_progress' };
   }
 
@@ -1397,17 +1504,14 @@ function cancelJob(cwd, version) {
     }
   }
 
-  // Update Status header to pending
-  updateJobHeader(filePath, 'Status', 'pending');
-
-  // Move to pending/
-  const pendingDir = path.join(getPlanningRoot(cwd), 'jobs', 'pending');
-  const moveResult = moveJobFile(filePath, pendingDir);
+  // Set job status to pending via frontmatter edit (no file move)
+  // setJobStatus handles both markdown header and frontmatter update
+  setJobStatus(cwd, version, 'pending');
 
   return {
     cancelled: true,
     version: version.startsWith('v') ? version : 'v' + version,
-    path: moveResult.to,
+    path: filePath,
     steps_reset: stepsReset,
   };
 }
@@ -1713,8 +1817,8 @@ function recordStartShas(cwd, jobFilePath) {
         if (name === 'Name' || name.startsWith('-')) continue;
         if (!repoPath || repoPath.startsWith('-')) continue;
 
-        // Resolve absolute path: paths in REPOS.md are relative to planning root's parent
-        const absRepoPath = path.resolve(path.dirname(planningRoot), repoPath);
+        // Resolve absolute path: paths in REPOS.md are relative to the planning root
+        const absRepoPath = path.resolve(planningRoot, repoPath);
         if (!fs.existsSync(absRepoPath)) continue;
 
         try {
@@ -1983,6 +2087,8 @@ module.exports = {
   insertJobSteps,
   buildGapFixSteps,
   buildPhaseGapFixSteps,
+  JOB_STATUSES,
+  setJobStatus,
   insertGapFixSection,
   insertPhaseGapFixSection,
   updatePhaseFixCycleHeader,