@dev-loops/core 0.1.0

package/src/debt/cluster.mjs

@@ -0,0 +1,240 @@
+// ============================================================================
+// Debt signal clustering logic
+//
+// Groups debt_signal objects into debt_finding clusters.
+//
+// Clustering rules:
+//   - file: signals share the same location.filePath
+//   - module: signals share the same parent directory of filePath
+//   - theme: signals share the same signalKind category
+//   - Precedence: file > module > theme
+//   - A signal belongs to exactly one cluster.
+//   - Signals with no file, module, or theme form singleton clusters.
+//
+// Finding ID determinism:
+//   Finding IDs are derived from sorted signalIds + clusterReason using a
+//   stable hash, so re-running clustering on the same input produces the
+//   same debt_finding artifacts.
+//
+// Output: array of debt_finding objects with embedded signal references,
+// signalIds, and aggregated score.
+// ============================================================================
+
+import { createHash } from "node:crypto";
+import { scoreCluster } from "./score.mjs";
+
+/**
+ * Derive a deterministic finding ID from sorted signal IDs and the cluster reason.
+ * Uses SHA-256 truncated to a UUID-v4-format string for stable cross-run identity.
+ *
+ * @param {Array<string>} signalIds — sorted array of signal UUIDs
+ * @param {string} clusterReason — "file", "module", "theme", or "singleton"
+ * @returns {string} UUID-v4-format deterministic ID
+ */
+function deriveFindingId(signalIds, clusterReason) {
+  const sorted = [...signalIds].sort();
+  const input = sorted.join(":") + "|" + clusterReason;
+  const hash = createHash("sha256").update(input).digest("hex");
+  // Format as UUID v4: 8-4-4-4-12 using first 32 hex chars
+  const variantNibble = (parseInt(hash[16], 16) & 0x3 | 0x8).toString(16);
+  return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-4${hash.slice(13, 16)}-${variantNibble}${hash.slice(17, 20)}-${hash.slice(20, 32)}`;
+}
+
+/**
+ * Derive deterministic timestamps from signal timestamps.
+ * Uses the earliest signal timestamp for createdAt and the latest for updatedAt.
+ *
+ * @param {Array<object>} signals
+ * @returns {{ createdAt: string, updatedAt: string }}
+ */
+function deriveTimestamps(signals) {
+  const timestamps = signals.map(s => s.timestamp).filter(Boolean).sort();
+  if (timestamps.length === 0) {
+    const epoch = "1970-01-01T00:00:00.000Z";
+    return { createdAt: epoch, updatedAt: epoch };
+  }
+  return { createdAt: timestamps[0], updatedAt: timestamps[timestamps.length - 1] };
+}
+
+/**
+ * Extract the file key from a signal: location.filePath, or undefined.
+ * @param {object} signal
+ * @returns {string|undefined}
+ */
+function fileKey(signal) {
+  return signal.location?.filePath || undefined;
+}
+
+/**
+ * Extract the module key from a signal:
+ * parent directory of filePath (dirname).
+ * Returns undefined when no filePath exists — module clustering is
+ * strictly directory-based and does not fall back to signalKind.
+ * Theme clustering handles signalKind grouping separately.
+ *
+ * @param {object} signal
+ * @returns {string|undefined}
+ */
+function moduleKey(signal) {
+  const fp = signal.location?.filePath;
+  if (!fp) return undefined;
+  const lastSlash = fp.lastIndexOf("/");
+  if (lastSlash >= 0) return fp.slice(0, lastSlash);
+  return fp; // no directory separator — treat as its own module
+}
+
+/**
+ * Extract the theme key from a signal: signalKind category.
+ * @param {object} signal
+ * @returns {string|undefined}
+ */
+function themeKey(signal) {
+  return signal.signalKind || undefined;
+}
+
+/**
+ * Group signals by a key extraction function.
+ * Returns Map<key, signal[]> where key may be undefined for signals lacking that dimension.
+ *
+ * @param {Array<object>} signals
+ * @param {(signal: object) => string|undefined} keyFn
+ * @returns {Map<string|undefined, Array<object>>}
+ */
+function groupByKey(signals, keyFn) {
+  const map = new Map();
+  for (const signal of signals) {
+    const key = keyFn(signal);
+    if (!map.has(key)) map.set(key, []);
+    map.get(key).push(signal);
+  }
+  return map;
+}
+
+/**
+ * Create a debt_finding from a cluster of signals.
+ *
+ * @param {Array<object>} signals — debt_signal-compatible objects
+ * @param {string} clusterReason — "file", "module", "theme", or "singleton"
+ * @returns {object} enriched debt_finding shape with internal fields
+ */
+function buildFinding(signals, clusterReason) {
+  const score = scoreCluster(signals);
+
+  // Build title from cluster reason + primary category
+  const categories = [...new Set(signals.map(s => s.signalKind).filter(Boolean))].sort();
+  const primaryCategory = categories[0] || "unknown";
+
+  // Build location summary
+  const filePaths = [...new Set(
+    signals.map(s => s.location?.filePath).filter(Boolean)
+  )].sort();
+
+  const title = filePaths.length === 1
+    ? `${clusterReason}: ${primaryCategory} in ${filePaths[0]}`
+    : `${clusterReason}: ${primaryCategory} (${signals.length} signals)`;
+
+  const description = `Clustered by ${clusterReason}. ` +
+    `Categories: ${categories.join(", ")}. ` +
+    `Files: ${filePaths.length > 0 ? filePaths.join(", ") : "none"}.`;
+
+  const sortedSignalIds = signals.map(s => s.id).sort();
+  const id = deriveFindingId(sortedSignalIds, clusterReason);
+  const { createdAt, updatedAt } = deriveTimestamps(signals);
+
+  return {
+    id,
+    signalIds: sortedSignalIds,
+    validationStatus: "pending",
+    score,
+    remediationShape: "watch_only", // placeholder; shaped later
+    title: title.slice(0, 200),
+    description: description.slice(0, 500),
+    locationSummary: filePaths.length > 0
+      ? { filePaths, primaryFilePath: filePaths[0] }
+      : undefined,
+    createdAt,
+    updatedAt,
+    // Internal fields for shaping (not in the DebtFindingSchema output)
+    _clusterReason: clusterReason,
+    _signalCount: signals.length,
+    _categories: categories,
+  };
+}
+
+/**
+ * Cluster debt_signal objects into debt_finding arrays.
+ *
+ * Multi-pass algorithm:
+ *   1. Group by file (location.filePath exact match)
+ *   2. From remaining, group by module (directory of filePath)
+ *   3. From remaining, group by theme (signalKind)
+ *   4. Remaining → singleton clusters
+ *
+ * Each signal appears in exactly one cluster.
+ * Clusters with only one signal when grouped by a dimension
+ * are deferred to the next pass.
+ *
+ * @param {Array<object>} signals — array of debt_signal-compatible objects
+ * @returns {Array<object>} array of clean debt_finding shapes (no internal fields)
+ */
+export function clusterSignals(signals) {
+  const enriched = clusterSignalsEnriched(signals);
+  return enriched.map(({ _clusterReason, _signalCount, _categories, ...clean }) => clean);
+}
+
+/**
+ * Cluster signals and return findings enriched with internal metadata
+ * for shaping (_clusterReason, _signalCount, _categories).
+ *
+ * @param {Array<object>} signals — array of debt_signal-compatible objects
+ * @returns {Array<object>} array of enriched debt_finding shapes
+ */
+export function clusterSignalsEnriched(signals) {
+  if (!Array.isArray(signals)) return [];
+  if (signals.length === 0) return [];
+
+  const findings = [];
+  let remaining = [...signals];
+
+  // Pass 1: file
+  const fileGroups = groupByKey(remaining, fileKey);
+  remaining = [];
+  for (const [key, group] of fileGroups) {
+    if (key === undefined || group.length <= 1) {
+      remaining.push(...group);
+    } else {
+      findings.push(buildFinding(group, "file"));
+    }
+  }
+
+  // Pass 2: module (directory-based only; no signalKind fallback)
+  const moduleGroups = groupByKey(remaining, moduleKey);
+  remaining = [];
+  for (const [key, group] of moduleGroups) {
+    if (key === undefined || group.length <= 1) {
+      remaining.push(...group);
+    } else {
+      findings.push(buildFinding(group, "module"));
+    }
+  }
+
+  // Pass 3: theme
+  const themeGroups = groupByKey(remaining, themeKey);
+  remaining = [];
+  for (const [key, group] of themeGroups) {
+    if (key === undefined || group.length <= 1) {
+      remaining.push(...group);
+    } else {
+      findings.push(buildFinding(group, "theme"));
+    }
+  }
+
+  // Pass 4: singletons for any remaining
+  for (const signal of remaining) {
+    findings.push(buildFinding([signal], "singleton"));
+  }
+
+  // Stable output order: sort findings by id
+  findings.sort((a, b) => a.id.localeCompare(b.id));
+  return findings;
+}

package/src/debt/debt-finding.mjs

@@ -0,0 +1,68 @@
+import { z } from "zod";
+
+export const DebtFindingValidationStatus = z.enum([
+  "pending",
+  "validated",
+  "rejected",
+  "stale",
+]);
+
+export const DebtFindingRemediationShape = z.enum([
+  "item",
+  "epic",
+  "defer",
+  "watch_only",
+  "dismissed",
+]);
+
+export const DebtFindingLocationSummary = z.strictObject({
+  filePaths: z.array(z.string().min(1)).optional(),
+  primaryFilePath: z.string().min(1).optional(),
+});
+
+export const DebtFindingSchema = z.strictObject({
+  id: z.string().uuid(),
+  signalIds: z.array(z.string().uuid()).min(1),
+  validationStatus: DebtFindingValidationStatus,
+  score: z.number().min(0).max(100).optional(),
+  remediationShape: DebtFindingRemediationShape,
+  title: z.string().min(1).max(200),
+  description: z.string().min(1).optional(),
+  locationSummary: DebtFindingLocationSummary.optional(),
+  createdAt: z.string().datetime(),
+  updatedAt: z.string().datetime(),
+});
+
+// ============================================================================
+// Remediation item schema — a bounded, PR-sized fix ready for the execution loop
+// ============================================================================
+export const RemediationItemSchema = z.strictObject({
+  kind: z.literal("remediation_item"),
+  findingId: z.string().uuid(),
+  title: z.string().min(1).max(200),
+  description: z.string().min(1),
+  acceptanceCriteria: z.array(z.string().min(1)).min(1),
+  score: z.number().min(0).max(100),
+  primaryFilePath: z.string().min(1).optional(),
+  filePaths: z.array(z.string().min(1)).min(1),
+  signalIds: z.array(z.string().uuid()).min(1),
+  sourceType: z.string().min(1).optional(),
+  createdAt: z.string().datetime(),
+  updatedAt: z.string().datetime(),
+});
+
+// ============================================================================
+// Debt epic schema — cross-cutting work that needs decomposition before execution
+// ============================================================================
+export const DebtEpicSchema = z.strictObject({
+  kind: z.literal("debt_epic"),
+  findingId: z.string().uuid(),
+  title: z.string().min(1).max(200),
+  description: z.string().min(1),
+  score: z.number().min(0).max(100),
+  filePaths: z.array(z.string().min(1)).min(1),
+  signalIds: z.array(z.string().uuid()).min(1),
+  estimatedItems: z.number().int().positive(),
+  createdAt: z.string().datetime(),
+  updatedAt: z.string().datetime(),
+});

package/src/debt/debt-signal.mjs

@@ -0,0 +1,46 @@
+import { z } from "zod";
+
+export const DebtSignalSourceType = z.enum([
+  "pr_review_deep_persona",
+  "repo_audit",
+  "flaky_test",
+  "ci_failure",
+  "manual_review",
+  "dependency_alert",
+  "review_churn",
+  "incident_followup",
+  "workflow_pain",
+]);
+
+export const DebtSignalSeverity = z.enum([
+  "info",
+  "low",
+  "medium",
+  "high",
+  "critical",
+]);
+
+export const DebtSignalLocation = z.strictObject({
+  filePath: z.string().min(1).optional(),
+  lineStart: z.number().int().positive().optional(),
+  lineEnd: z.number().int().positive().optional(),
+  commitSha: z.string().regex(/^[a-f0-9]{7,40}$/).optional(),
+  url: z.string().url().optional(),
+});
+
+export const DebtSignalRepository = z.strictObject({
+  owner: z.string().min(1),
+  name: z.string().min(1),
+});
+
+export const DebtSignalSchema = z.strictObject({
+  id: z.string().uuid(),
+  sourceType: DebtSignalSourceType,
+  signalKind: z.string().min(1).max(100),
+  location: DebtSignalLocation,
+  severityHint: DebtSignalSeverity,
+  timestamp: z.string().datetime(),
+  rawPayload: z.record(z.string(), z.unknown()).optional(),
+  repository: DebtSignalRepository.optional(),
+  confidence: z.number().min(0).max(1).default(1),
+});

package/src/debt/deep-persona-signals.mjs

@@ -0,0 +1,266 @@
+import { randomUUID } from "node:crypto";
+import { DebtSignalSchema } from "./debt-signal.mjs";
+import { loadDevLoopConfig } from "../config/config.mjs";
+
+// ============================================================================
+// Flag phrase inventory — derived from personas.deep prompt in defaults.yaml
+//
+// Confidence values use the canonical DebtSignalSchema 0..1 range (0.9 = 90%).
+// This matches the schema default: z.number().min(0).max(1).default(1).
+// ============================================================================
+
+/** @type {Array<{ phrase: RegExp, category: string, severity: string, confidence: number }>} */
+const FLAG_PATTERNS = [
+  {
+    phrase: /(?:crossing|crossed|exceeds?)\s+(?:1000|1,000)\+?\s+lines/i,
+    category: "file_size",
+    severity: "high",
+    confidence: 0.9,
+  },
+  {
+    phrase: /conditionals?\s+bolted\s+onto\s+unrelated\s+paths/i,
+    category: "spaghetti_branching",
+    severity: "high",
+    confidence: 0.9,
+  },
+  {
+    phrase: /\bspaghetti\b/i,
+    category: "spaghetti_branching",
+    severity: "high",
+    confidence: 0.9,
+  },
+  {
+    phrase: /thin\s+wrapper/i,
+    category: "thin_wrapper",
+    severity: "medium",
+    confidence: 0.9,
+  },
+  {
+    phrase: /re-export\s*[- ]?only/i,
+    category: "thin_wrapper",
+    severity: "medium",
+    confidence: 0.9,
+  },
+  {
+    phrase: /identity\s+abstraction/i,
+    category: "thin_wrapper",
+    severity: "medium",
+    confidence: 0.9,
+  },
+  {
+    phrase: /feature\s+logic\s+leaking\s+into/i,
+    category: "leaky_feature_logic",
+    severity: "high",
+    confidence: 0.9,
+  },
+  {
+    phrase: /leaking\s+into\s+shared/i,
+    category: "leaky_feature_logic",
+    severity: "high",
+    confidence: 0.9,
+  },
+  {
+    phrase: /cast[- ]?heavy/i,
+    category: "weak_contract",
+    severity: "medium",
+    confidence: 0.9,
+  },
+  {
+    phrase: /optionality[- ]?heavy/i,
+    category: "weak_contract",
+    severity: "medium",
+    confidence: 0.9,
+  },
+  {
+    phrase: /any[- ]?typed\s+contract/i,
+    category: "weak_contract",
+    severity: "medium",
+    confidence: 0.9,
+  },
+  {
+    phrase: /code\s+judo/i,
+    category: "simplification_opportunity",
+    severity: "medium",
+    confidence: 0.9,
+  },
+  {
+    phrase: /prefer\s+deletion\s+over\s+addition/i,
+    category: "simplification_opportunity",
+    severity: "medium",
+    confidence: 0.9,
+  },
+];
+
+const FILE_PATH_RE = /[\w/\-.]+\.(?:m?js|ts|tsx|jsx|mjs)/i;
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/**
+ * Match a comment body against known deep-persona flag phrases.
+ * Returns the match when a specific pattern matches, or null when no
+ * patterns match the body.
+ *
+ * @param {string} body
+ * @returns {{ category: string, severity: string, confidence: number, matchedPhrase: string|null }|null}
+ */
+function matchDeepPersonaFlags(body) {
+  for (const pattern of FLAG_PATTERNS) {
+    if (pattern.phrase.test(body)) {
+      return {
+        category: pattern.category,
+        severity: pattern.severity,
+        confidence: pattern.confidence,
+        matchedPhrase: pattern.phrase.source,
+      };
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Extract a file path from a comment body, or return an empty string if none found.
+ *
+ * @param {string} body
+ * @returns {string}
+ */
+function extractFilePath(body) {
+  const match = body.match(FILE_PATH_RE);
+  return match ? match[0] : "";
+}
+
+/**
+ * Build a debt_signal object from a matched deep-persona comment.
+ *
+ * @param {object} comment - Normalized comment from parseReviewThreads output
+ * @param {string} category - Inferred category
+ * @param {string} severity - Severity hint
+ * @param {number} confidence - Confidence score (0..1)
+ * @param {string|null} matchedPhrase - The regex source that matched
+ * @param {{ prNumber: string|number, prUrl: string }} prMeta
+ * @returns {object}
+ */
+function buildDebtSignal(comment, category, severity, confidence, matchedPhrase, prMeta) {
+  const filePath = extractFilePath(comment.body);
+
+  return {
+    id: randomUUID(),
+    sourceType: "pr_review_deep_persona",
+    signalKind: category,
+    location: filePath ? { filePath } : {},
+    severityHint: severity,
+    timestamp: new Date().toISOString(),
+    confidence,
+    rawPayload: {
+      description: comment.body,
+      metadata: {
+        prNumber: String(prMeta.prNumber),
+        prUrl: prMeta.prUrl,
+        commentId: comment.id,
+        threadId: comment.threadId,
+        isResolved: comment.isResolved ?? false,
+        category,
+        matchedPhrase,
+      },
+    },
+  };
+}
+
+// ============================================================================
+// Public API
+// ============================================================================
+
+/**
+ * Extract deep-persona debt_signal artifacts from normalized review-thread JSON.
+ *
+ * Accepts the output of `parseReviewThreads()` (the `comments` array with
+ * normalized `{ id, threadId, author, body, isActionable }` entries).
+ *
+ * Filters to only bot-authored comments that match known deep-persona flag
+ * phrases. Bots are identified by `author.isBot === true` or `author.type === "Bot"`.
+ *
+ * @param {{ comments: Array<{ id: string, threadId: string, author: { login: string, type: string, isBot: boolean }, body: string, isActionable?: boolean, isResolved?: boolean }>, threads?: Array<{ id: string, isResolved: boolean }> }} parsedOutput - parseReviewThreads() output
+ * @param {{ prNumber: string|number, prUrl: string }} prMeta
+ * @returns {Array<object>} Array of debt_signal objects compatible with DebtSignalSchema
+ */
+export function extractDeepPersonaSignals(parsedOutput, prMeta) {
+  if (!parsedOutput || !Array.isArray(parsedOutput.comments)) {
+    throw new Error("Invalid parsed output: expected { comments: [...] } from parseReviewThreads()");
+  }
+
+  // Build a thread-id → isResolved map for fast lookup
+  const threadResolved = new Map();
+  if (Array.isArray(parsedOutput.threads)) {
+    for (const thread of parsedOutput.threads) {
+      threadResolved.set(thread.id, Boolean(thread.isResolved));
+    }
+  }
+
+  const signals = [];
+
+  for (const comment of parsedOutput.comments) {
+    // Only process bot-authored comments (all Copilot personas emit as bots)
+    if (!comment.author || (!comment.author.isBot && comment.author.type !== "Bot")) {
+      continue;
+    }
+
+    if (!comment.body || comment.body.trim().length === 0) {
+      continue;
+    }
+
+    const match = matchDeepPersonaFlags(comment.body);
+    if (!match) {
+      continue;
+    }
+
+    // Enrich comment with isResolved from thread data
+    const isResolved = threadResolved.has(comment.threadId)
+      ? threadResolved.get(comment.threadId)
+      : (comment.isResolved ?? false);
+
+    const signal = buildDebtSignal(
+      { ...comment, isResolved },
+      match.category,
+      match.severity,
+      match.confidence,
+      match.matchedPhrase,
+      prMeta,
+    );
+
+    // Validate against canonical schema — throw on regression
+    signals.push(DebtSignalSchema.parse(signal));
+  }
+
+  return signals;
+}
+
+/**
+ * Return the known deep-persona flag phrase regex sources for inspection.
+ *
+ * @returns {Array<string>}
+ */
+export function getDeepPersonaFlagPhrases() {
+  return FLAG_PATTERNS.map((p) => p.phrase.source);
+}
+
+/**
+ * Verify that all known deep-persona flag phrase regex patterns match
+ * the loaded deep persona prompt text. Returns an array of regex sources
+ * whose patterns did not find any match in the prompt (empty = all match).
+ *
+ * @returns {Promise<Array<string>>}
+ */
+export async function verifyPromptStability() {
+  const { config, errors } = await loadDevLoopConfig();
+  if (errors.length > 0) {
+    throw new Error("Cannot verify prompt stability: config load errors: " +
+      errors.map(e => e.message).join("; "));
+  }
+  const deepPrompt = config?.personas?.deep?.prompt ?? "";
+
+  return FLAG_PATTERNS
+    .filter((p) => !p.phrase.test(deepPrompt))
+    .map((p) => p.phrase.source);
+}