@contentful/experience-design-system-cli 2.12.0 → 2.12.1-dev-build-589b615.0

package/dist/src/session/db.js

@@ -1,5 +1,5 @@
 import { DatabaseSync } from 'node:sqlite';
-import { mkdirSync } from 'node:fs';
+import { mkdirSync, readFileSync } from 'node:fs';
 import { createHash } from 'node:crypto';
 import { homedir } from 'node:os';
 import { dirname, resolve } from 'node:path';
@@ -46,23 +46,28 @@ CREATE TABLE IF NOT EXISTS raw_components (
   extraction_confidence  INTEGER,
   review_reasons         TEXT NOT NULL DEFAULT '[]',
   needs_review           INTEGER NOT NULL DEFAULT 0,
+  source_path            TEXT,
+  reject_reason          TEXT,
   PRIMARY KEY (session_id, component_id)
 );
 
 CREATE TABLE IF NOT EXISTS raw_props (
-  session_id      TEXT NOT NULL,
-  component_id    TEXT NOT NULL,
-  name            TEXT NOT NULL,
-  type            TEXT NOT NULL,
-  required        INTEGER NOT NULL CHECK (required IN (0, 1)),
-  category        TEXT CHECK (category IN ('content', 'design', 'state')),
-  default_value   TEXT,
-  description     TEXT,
-  token_reference TEXT,
-  position        INTEGER NOT NULL,
-  cdf_type        TEXT,
-  cdf_category    TEXT CHECK (cdf_category IN ('content', 'design', 'state')),
-  cdf_token_kind  TEXT,
+  session_id        TEXT NOT NULL,
+  component_id      TEXT NOT NULL,
+  name              TEXT NOT NULL,
+  type              TEXT NOT NULL,
+  required          INTEGER NOT NULL CHECK (required IN (0, 1)),
+  category          TEXT CHECK (category IN ('content', 'design', 'state')),
+  default_value     TEXT,
+  description       TEXT,
+  token_reference   TEXT,
+  position          INTEGER NOT NULL,
+  cdf_type          TEXT,
+  cdf_category      TEXT CHECK (cdf_category IN ('content', 'design', 'state')),
+  cdf_token_kind    TEXT,
+  rationale         TEXT,
+  source_start_line INTEGER,
+  source_end_line   INTEGER,
   PRIMARY KEY (session_id, component_id, name),
   FOREIGN KEY (session_id, component_id) REFERENCES raw_components(session_id, component_id) ON DELETE CASCADE
 );
@@ -155,6 +160,21 @@ export function openPipelineDb(dbPath) {
     mkdirSync(dirname(path), { recursive: true });
     try {
         const db = new DatabaseSync(path);
+        // Configure connection-level pragmas BEFORE running schema/migrations.
+        // - journal_mode=WAL: persistent on the DB file; readers don't block writers
+        //   and vice versa. (Also set in SCHEMA, but harmless to assert per-open.)
+        // - busy_timeout: per-connection; SQLite waits for a lock instead of
+        //   immediately throwing 'database is locked'. Critical when the wizard
+        //   main process and select-agent subprocesses contend on writes.
+        // - synchronous=NORMAL: safe under WAL, faster than FULL.
+        // busy_timeout MUST come first. PRAGMA journal_mode = WAL needs a
+        // RESERVED lock briefly even when the DB is already WAL; if the
+        // select-agent child is mid-write at that moment, the wizard's open
+        // fails instantly without retry. Setting busy_timeout first makes
+        // SQLite wait up to 5s for the lock.
+        db.exec('PRAGMA busy_timeout = 5000');
+        db.exec('PRAGMA journal_mode = WAL');
+        db.exec('PRAGMA synchronous = NORMAL');
         db.exec(SCHEMA);
         applyDbMigrations(db);
         return db;
@@ -162,8 +182,12 @@ export function openPipelineDb(dbPath) {
     catch (e) {
         const msg = e instanceof Error ? e.message : String(e);
         if (msg.includes('database is locked')) {
-            process.stderr.write('Error: another CLI process is already running. Wait for it to finish and retry.\n');
-            process.exit(1);
+            // Throw a typed-ish error rather than process.exit(1). Calling exit
+            // from inside Ink's render reconciler crashes the TUI mid-frame and
+            // emits a noisy 'setState during render' warning. The CLI entry
+            // point (src/index.ts) catches and exits cleanly for non-TUI
+            // contexts; the wizard catches and transitions to its error step.
+            throw new Error('database is locked: another CLI process may be running. Retry once it exits.');
         }
         throw e;
     }
@@ -186,6 +210,84 @@ function applyDbMigrations(db) {
     if (!rawCompColNames.has('needs_review')) {
         db.exec('ALTER TABLE raw_components ADD COLUMN needs_review INTEGER NOT NULL DEFAULT 0');
     }
+    // Feature 1: per-component source path on raw_components, rationale + per-prop
+    // source location on raw_props. All nullable, no DEFAULT — existing rows survive.
+    if (!rawCompColNames.has('source_path')) {
+        db.exec('ALTER TABLE raw_components ADD COLUMN source_path TEXT');
+    }
+    const rawPropCols = db.prepare('PRAGMA table_info(raw_props)').all();
+    const rawPropColNames = new Set(rawPropCols.map((c) => c.name));
+    if (!rawPropColNames.has('rationale')) {
+        db.exec('ALTER TABLE raw_props ADD COLUMN rationale TEXT');
+    }
+    if (!rawPropColNames.has('source_start_line')) {
+        db.exec('ALTER TABLE raw_props ADD COLUMN source_start_line INTEGER');
+    }
+    if (!rawPropColNames.has('source_end_line')) {
+        db.exec('ALTER TABLE raw_props ADD COLUMN source_end_line INTEGER');
+    }
+    // Feature 3: persist select-agent's reject reason on raw_components. Nullable,
+    // no DEFAULT — accepted components keep NULL, rejected ones store the LLM's
+    // reason (or `validation error: <codes>` for validation auto-rejections).
+    if (!rawCompColNames.has('reject_reason')) {
+        db.exec('ALTER TABLE raw_components ADD COLUMN reject_reason TEXT');
+    }
+    // Component rationale: WHY a component was classified, why these props,
+    // why these slots — surfaced via the `I` panel. All nullable, no DEFAULT
+    // so existing rows survive. Plus per-slot rationale on raw_slots.
+    if (!rawCompColNames.has('component_description_rationale')) {
+        db.exec('ALTER TABLE raw_components ADD COLUMN component_description_rationale TEXT');
+    }
+    if (!rawCompColNames.has('props_rationale')) {
+        db.exec('ALTER TABLE raw_components ADD COLUMN props_rationale TEXT');
+    }
+    if (!rawCompColNames.has('slots_rationale')) {
+        db.exec('ALTER TABLE raw_components ADD COLUMN slots_rationale TEXT');
+    }
+    const rawSlotColsForRationale = db.prepare('PRAGMA table_info(raw_slots)').all();
+    if (!rawSlotColsForRationale.some((c) => c.name === 'rationale')) {
+        db.exec('ALTER TABLE raw_slots ADD COLUMN rationale TEXT');
+    }
+    // Fine-grained cache: extract_cache holds per-file extracted components keyed
+    // on (file_hash, cli_version). cli_version is a content hash of bundled skill
+    // files so cache busts on a prompt change (not on every npm patch bump).
+    const hasExtractCache = db
+        .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='extract_cache'`)
+        .all();
+    if (hasExtractCache.length === 0) {
+        db.exec(`
+      CREATE TABLE extract_cache (
+        file_path        TEXT NOT NULL,
+        file_hash        TEXT NOT NULL,
+        cli_version      TEXT NOT NULL,
+        created_at       TEXT NOT NULL,
+        updated_at       TEXT NOT NULL,
+        components_json  TEXT NOT NULL,
+        PRIMARY KEY (file_hash, cli_version)
+      );
+      CREATE INDEX idx_extract_cache_file ON extract_cache(file_path);
+    `);
+    }
+    // Fine-grained cache: select_cache stores accept/reject decisions per
+    // (component_hash, prompt_hash, cli_version). On hit, the wizard replays the
+    // decision without an LLM call.
+    const hasSelectCache = db
+        .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='select_cache'`)
+        .all();
+    if (hasSelectCache.length === 0) {
+        db.exec(`
+      CREATE TABLE select_cache (
+        component_hash TEXT NOT NULL,
+        prompt_hash    TEXT NOT NULL,
+        cli_version    TEXT NOT NULL,
+        decision       TEXT NOT NULL CHECK (decision IN ('accepted','rejected')),
+        reason         TEXT,
+        created_at     TEXT NOT NULL,
+        updated_at     TEXT NOT NULL,
+        PRIMARY KEY (component_hash, prompt_hash, cli_version)
+      );
+    `);
+    }
     // Add generation_cache table if it doesn't exist yet.
     const tables = db
         .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='generation_cache'`)
@@ -206,6 +308,131 @@ function applyDbMigrations(db) {
       CREATE INDEX idx_generation_cache_session ON generation_cache(source_session_id);
     `);
     }
+    // Fine-grained cache: add prompt_hash to generation_cache so cache busts when
+    // the operator switches prompt files. Existing rows get '' (matches the
+    // historical "default bundled prompt" — note that lookups using a non-empty
+    // hash for the bundled prompt won't match, so first run after upgrade busts
+    // the cache. Documented as acceptable.
+    //
+    // The primary key also needs to expand to include prompt_hash so multiple
+    // prompt variants can coexist per (input_hash, entity_type, entity_id).
+    // SQLite doesn't allow ALTERing a PK, so when promoting the schema we rebuild
+    // the table in-place.
+    const genCacheCols = db.prepare(`PRAGMA table_info(generation_cache)`).all();
+    const hasPromptHash = genCacheCols.some((c) => c.name === 'prompt_hash');
+    const oldPkCols = genCacheCols.filter((c) => c.pk > 0).map((c) => c.name);
+    const promptHashInPk = oldPkCols.includes('prompt_hash');
+    if (!hasPromptHash) {
+        db.exec(`ALTER TABLE generation_cache ADD COLUMN prompt_hash TEXT NOT NULL DEFAULT ''`);
+    }
+    if (!promptHashInPk) {
+        db.exec('BEGIN');
+        try {
+            db.exec(`
+        CREATE TABLE generation_cache__new (
+          input_hash        TEXT NOT NULL,
+          entity_type       TEXT NOT NULL CHECK (entity_type IN ('component', 'token_set')),
+          entity_id         TEXT NOT NULL,
+          source_session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+          human_edited      INTEGER NOT NULL DEFAULT 0 CHECK (human_edited IN (0, 1)),
+          created_at        TEXT NOT NULL,
+          updated_at        TEXT NOT NULL,
+          prompt_hash       TEXT NOT NULL DEFAULT '',
+          PRIMARY KEY (input_hash, prompt_hash, entity_type, entity_id)
+        );
+        INSERT INTO generation_cache__new
+          (input_hash, entity_type, entity_id, source_session_id, human_edited, created_at, updated_at, prompt_hash)
+          SELECT input_hash, entity_type, entity_id, source_session_id, human_edited, created_at, updated_at, prompt_hash
+          FROM generation_cache;
+        DROP TABLE generation_cache;
+        ALTER TABLE generation_cache__new RENAME TO generation_cache;
+        CREATE INDEX IF NOT EXISTS idx_generation_cache_entity  ON generation_cache(entity_type, entity_id);
+        CREATE INDEX IF NOT EXISTS idx_generation_cache_session ON generation_cache(source_session_id);
+      `);
+            db.exec('COMMIT');
+        }
+        catch (e) {
+            db.exec('ROLLBACK');
+            throw e;
+        }
+    }
+}
+export function loadComponentReviewMetadata(db, sessionId, componentName) {
+    const compRow = db
+        .prepare(`SELECT component_id, source, source_path FROM raw_components WHERE session_id = ? AND name = ?`)
+        .get(sessionId, componentName);
+    if (!compRow)
+        return null;
+    const propRows = db
+        .prepare(`SELECT name, rationale, source_start_line, source_end_line FROM raw_props WHERE session_id = ? AND component_id = ?`)
+        .all(sessionId, compRow.component_id);
+    const props = {};
+    for (const r of propRows) {
+        props[r.name] = {
+            rationale: r.rationale,
+            sourceStartLine: r.source_start_line,
+            sourceEndLine: r.source_end_line,
+        };
+    }
+    // raw_components.source historically stores the file path, not the file
+    // text. For the source-view panel to render real lines, prefer reading
+    // the file from disk via source_path. Fall back to whatever's in `source`
+    // (handles in-memory fixtures and tests).
+    let componentSource = compRow.source ?? null;
+    if (compRow.source_path) {
+        try {
+            componentSource = readFileSync(compRow.source_path, 'utf8');
+        }
+        catch {
+            // File no longer exists or unreadable — leave fallback.
+        }
+    }
+    return {
+        sourcePath: compRow.source_path,
+        componentSource,
+        props,
+    };
+}
+/**
+ * Component-rationale loader: data contract for the `I` ComponentRationalePanel.
+ * Returns the component's description + the three component-level rationale
+ * strings (why-description / why-props / why-slots), plus per-prop and per-slot
+ * rationale rows for the operator-facing panel. Returns null when no component
+ * matches.
+ */
+export function loadComponentRationale(db, sessionId, componentName) {
+    const compRow = db
+        .prepare(`SELECT component_id, name, description, component_description_rationale, props_rationale, slots_rationale
+       FROM raw_components WHERE session_id = ? AND name = ?`)
+        .get(sessionId, componentName);
+    if (!compRow)
+        return null;
+    const propRows = db
+        .prepare(`SELECT name, cdf_category, category, description, rationale FROM raw_props
+       WHERE session_id = ? AND component_id = ? ORDER BY position`)
+        .all(sessionId, compRow.component_id);
+    const slotRows = db
+        .prepare(`SELECT name, description, rationale FROM raw_slots
+       WHERE session_id = ? AND component_id = ? ORDER BY position`)
+        .all(sessionId, compRow.component_id);
+    return {
+        name: compRow.name,
+        description: compRow.description,
+        descriptionRationale: compRow.component_description_rationale,
+        propsRationale: compRow.props_rationale,
+        slotsRationale: compRow.slots_rationale,
+        props: propRows.map((p) => ({
+            name: p.name,
+            category: p.cdf_category ?? p.category ?? null,
+            description: p.description,
+            rationale: p.rationale,
+        })),
+        slots: slotRows.map((s) => ({
+            name: s.name,
+            description: s.description,
+            rationale: s.rationale,
+        })),
+    };
 }
 export function applyToolCalls(db, sessionId, componentId, componentName, calls, incomingWarnings) {
     const now = new Date().toISOString();
@@ -213,9 +440,9 @@ export function applyToolCalls(db, sessionId, componentId, componentName, calls,
     let classified = 0;
     let excluded = 0;
     let slots = 0;
-    const updateProp = db.prepare(`UPDATE raw_props SET cdf_type = ?, cdf_category = ?, cdf_token_kind = ?, required = ?, description = ?
+    const updateProp = db.prepare(`UPDATE raw_props SET cdf_type = ?, cdf_category = ?, cdf_token_kind = ?, required = ?, description = ?, rationale = ?
      WHERE session_id = ? AND component_id = ? AND name = ?`);
-    const clearProp = db.prepare(`UPDATE raw_props SET cdf_type = 'excluded', cdf_category = NULL, cdf_token_kind = NULL
+    const clearProp = db.prepare(`UPDATE raw_props SET cdf_type = 'excluded', cdf_category = NULL, cdf_token_kind = NULL, rationale = ?
      WHERE session_id = ? AND component_id = ? AND name = ?`);
     const deleteAllowedValues = db.prepare(`DELETE FROM raw_prop_allowed_values WHERE session_id = ? AND component_id = ? AND prop_name = ?`);
     const insertAllowedValue = db.prepare(`INSERT OR IGNORE INTO raw_prop_allowed_values (session_id, component_id, prop_name, value, position)
@@ -231,9 +458,22 @@ export function applyToolCalls(db, sessionId, componentId, componentName, calls,
                 if (call.description !== undefined) {
                     db.prepare('UPDATE raw_components SET description = ? WHERE session_id = ? AND component_id = ?').run(call.description, sessionId, componentId);
                 }
+                // Component-level rationale: sparse-update each field only when present
+                // so missing keys don't overwrite previously-stored values.
+                if (call.rationale) {
+                    if (call.rationale.description !== undefined) {
+                        db.prepare('UPDATE raw_components SET component_description_rationale = ? WHERE session_id = ? AND component_id = ?').run(call.rationale.description, sessionId, componentId);
+                    }
+                    if (call.rationale.props !== undefined) {
+                        db.prepare('UPDATE raw_components SET props_rationale = ? WHERE session_id = ? AND component_id = ?').run(call.rationale.props, sessionId, componentId);
+                    }
+                    if (call.rationale.slots !== undefined) {
+                        db.prepare('UPDATE raw_components SET slots_rationale = ? WHERE session_id = ? AND component_id = ?').run(call.rationale.slots, sessionId, componentId);
+                    }
+                }
             }
             else if (call.tool === 'classify_prop') {
-                const changes = updateProp.run(call.cdf_type, call.cdf_category, call.token_kind ?? null, call.required !== undefined ? (call.required ? 1 : 0) : 0, call.description ?? null, sessionId, componentId, call.prop);
+                const changes = updateProp.run(call.cdf_type, call.cdf_category, call.token_kind ?? null, call.required !== undefined ? (call.required ? 1 : 0) : 0, call.description ?? null, call.reason ?? null, sessionId, componentId, call.prop);
                 if (changes.changes === 0) {
                     warnings.push(`${componentName}: classify_prop '${call.prop}' — prop not found, skipped`);
                     continue;
@@ -249,7 +489,7 @@ export function applyToolCalls(db, sessionId, componentId, componentName, calls,
                 classified++;
             }
             else if (call.tool === 'exclude_prop') {
-                clearProp.run(sessionId, componentId, call.prop);
+                clearProp.run(call.reason || null, sessionId, componentId, call.prop);
                 excluded++;
             }
             else if (call.tool === 'classify_slot') {
@@ -259,6 +499,11 @@ export function applyToolCalls(db, sessionId, componentId, componentName, calls,
                     warnings.push(`${componentName}: classify_slot '${call.slot}' — slot not found, skipped`);
                     continue;
                 }
+                // Per-slot rationale: sparse-update only when present so omitting it
+                // doesn't blank existing values.
+                if (call.rationale !== undefined) {
+                    db.prepare('UPDATE raw_slots SET rationale = ? WHERE session_id = ? AND component_id = ? AND name = ?').run(call.rationale, sessionId, componentId, call.slot);
+                }
                 if (call.allowed_components !== undefined) {
                     deleteAllowedComponents.run(sessionId, componentId, call.slot);
                     call.allowed_components.forEach((ac, i) => insertAllowedComponent.run(sessionId, componentId, call.slot, ac, i));
@@ -276,15 +521,13 @@ export function applyToolCalls(db, sessionId, componentId, componentName, calls,
     }
     return { classified, excluded, slots, warnings };
 }
-export function getOrCreateSession(db, sessionFlag, sessionName, hints) {
+export function getOrCreateSession(db, sessionFlag, sessionName, 
+// `_hints` is retained for call-site compatibility but no longer drives an
+// implicit auto-match by inputPath/outDir — operators who want to resume must
+// pass an explicit --session <id>.
+_hints) {
     const now = new Date().toISOString();
     if (sessionFlag === 'new' || sessionFlag === undefined) {
-        if (sessionFlag !== 'new') {
-            const match = findMatchingSession(db, hints);
-            if (match) {
-                return { sessionId: match, isNew: false, isResumed: true };
-            }
-        }
         const id = generateSessionId();
         db.prepare('INSERT INTO sessions (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)').run(id, sessionName ?? null, now, now);
         return { sessionId: id, isNew: true, isResumed: false };
@@ -295,48 +538,6 @@ export function getOrCreateSession(db, sessionFlag, sessionName, hints) {
     }
     return { sessionId: sessionFlag, isNew: false, isResumed: false };
 }
-function findMatchingSession(db, hints) {
-    if (!hints.inputPath && !hints.outDir)
-        return null;
-    const rows = db
-        .prepare(`SELECT s.id, st.inputs, st.outputs, st.status
-       FROM sessions s
-       JOIN steps st ON st.session_id = s.id
-       WHERE st.command = ?
-         AND st.status IN ('pending', 'interrupted')
-       ORDER BY st.started_at DESC
-       LIMIT 20`)
-        .all(hints.command);
-    for (const row of rows) {
-        let inputs = {};
-        try {
-            inputs = JSON.parse(row.inputs);
-        }
-        catch {
-            continue;
-        }
-        if (hints.inputPath) {
-            const inputValues = Object.values(inputs);
-            if (inputValues.some((v) => v === hints.inputPath)) {
-                return row.id;
-            }
-        }
-        if (hints.outDir) {
-            let outputs = {};
-            try {
-                outputs = JSON.parse(row.outputs);
-            }
-            catch {
-                // ignore
-            }
-            const allValues = [...Object.values(inputs), ...Object.values(outputs)];
-            if (allValues.some((v) => v === hints.outDir)) {
-                return row.id;
-            }
-        }
-    }
-    return null;
-}
 export function createStep(db, sessionId, command, inputs) {
     const now = new Date().toISOString();
     db.exec('BEGIN');
@@ -374,11 +575,11 @@ export function updateStep(db, stepId, status, outputs, error) {
 }
 export function storeRawComponents(db, sessionId, components, options) {
     const now = new Date().toISOString();
-    const insertComp = db.prepare(`INSERT INTO raw_components (session_id, component_id, name, source, framework, extracted_at, extraction_confidence, review_reasons, needs_review)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
-    const insertProp = db.prepare(`INSERT INTO raw_props
-       (session_id, component_id, name, type, required, category, default_value, description, token_reference, position)
+    const insertComp = db.prepare(`INSERT INTO raw_components (session_id, component_id, name, source, framework, extracted_at, extraction_confidence, review_reasons, needs_review, source_path)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+    const insertProp = db.prepare(`INSERT INTO raw_props
+       (session_id, component_id, name, type, required, category, default_value, description, token_reference, position, source_start_line, source_end_line)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
     const insertAllowedValue = db.prepare(`INSERT INTO raw_prop_allowed_values (session_id, component_id, prop_name, position, value)
      VALUES (?, ?, ?, ?, ?)`);
     const insertSlot = db.prepare(`INSERT INTO raw_slots (session_id, component_id, name, is_default, description, position)
@@ -409,10 +610,10 @@ export function storeRawComponents(db, sessionId, components, options) {
         db.prepare('DELETE FROM raw_components WHERE session_id = ?').run(sessionId);
         for (const comp of components) {
             const componentId = deriveComponentId(comp.name, comp.source);
-            insertComp.run(sessionId, componentId, comp.name, comp.source, comp.framework, now, comp.extractionConfidence ?? null, JSON.stringify(comp.reviewReasons ?? []), comp.needsReview ? 1 : 0);
+            insertComp.run(sessionId, componentId, comp.name, comp.source, comp.framework, now, comp.extractionConfidence ?? null, JSON.stringify(comp.reviewReasons ?? []), comp.needsReview ? 1 : 0, comp.sourcePath ?? null);
             for (let i = 0; i < comp.props.length; i++) {
                 const prop = comp.props[i];
-                insertProp.run(sessionId, componentId, prop.name, prop.type, prop.required ? 1 : 0, prop.category ?? null, prop.defaultValue ?? null, prop.description ?? null, prop.tokenReference ?? null, i);
+                insertProp.run(sessionId, componentId, prop.name, prop.type, prop.required ? 1 : 0, prop.category ?? null, prop.defaultValue ?? null, prop.description ?? null, prop.tokenReference ?? null, i, prop.sourceStartLine ?? null, prop.sourceEndLine ?? null);
                 if (prop.allowedValues) {
                     prop.allowedValues.forEach((v, j) => insertAllowedValue.run(sessionId, componentId, prop.name, j, v));
                 }
@@ -489,13 +690,14 @@ export function storeRawComponents(db, sessionId, components, options) {
 }
 export function loadRawComponents(db, sessionId, allowedNames) {
     const all = db
-        .prepare('SELECT component_id, name, source, framework, extraction_confidence, review_reasons, needs_review FROM raw_components WHERE session_id = ? ORDER BY rowid')
+        .prepare('SELECT component_id, name, source, framework, extraction_confidence, review_reasons, needs_review, source_path FROM raw_components WHERE session_id = ? ORDER BY rowid')
         .all(sessionId);
     const components = allowedNames ? all.filter((c) => allowedNames.has(c.name)) : all;
     if (components.length === 0)
         return [];
     const props = db
-        .prepare(`SELECT component_id, name, type, required, category, default_value, description, token_reference, position
+        .prepare(`SELECT component_id, name, type, required, category, default_value, description, token_reference, position,
+              rationale, source_start_line, source_end_line
        FROM raw_props WHERE session_id = ? ORDER BY component_id, position`)
         .all(sessionId);
     const allowedValues = db
@@ -530,6 +732,7 @@ export function loadRawComponents(db, sessionId, allowedNames) {
             }
         })(),
         needsReview: Boolean(c.needs_review),
+        sourcePath: c.source_path ?? undefined,
         props: (propsByComponent.get(c.component_id) ?? []).map((p) => {
             const av = allowedValuesByProp.get(`${c.component_id}::${p.name}`);
             const prop = {
@@ -547,6 +750,10 @@ export function loadRawComponents(db, sessionId, allowedNames) {
                 prop.tokenReference = p.token_reference;
             if (av && av.length > 0)
                 prop.allowedValues = av.map((v) => v.value);
+            if (p.source_start_line !== null)
+                prop.sourceStartLine = p.source_start_line;
+            if (p.source_end_line !== null)
+                prop.sourceEndLine = p.source_end_line;
             return prop;
         }),
         slots: (slotsByComponent.get(c.component_id) ?? []).map((s) => {
@@ -706,13 +913,16 @@ export function loadCDFComponents(db, sessionId) {
     const allowedValuesByProp = groupBy(allowedValues, (av) => `${av.component_id}::${av.prop_name}`);
     const slotsByComponent = groupBy(slots, (s) => s.component_id);
     const allowedComponentsBySlot = groupBy(allowedComponents, (ac) => `${ac.component_id}::${ac.slot_name}`);
-    return components
-        .map(({ component_id, name, description }) => {
+    return components.map(({ component_id, name, description }) => {
         const compProps = propsByComponent.get(component_id) ?? [];
-        // Skip components with no CDF-classified props — sending them with empty
-        // $properties causes the server to flag all stored props as "removed" (breaking).
-        if (compProps.length === 0)
-            return null;
+        // NOTE: Components with zero CDF-classified props are intentionally
+        // returned with an empty `$properties` object. Filtering them out here
+        // silently dropped them from the wizard's final-review (INTEG-4257),
+        // so the user couldn't see that the LLM had failed to classify anything
+        // and couldn't reject the empty component. Surface them instead — the
+        // wizard tags them with a ⚠ badge and an in-panel banner so the user
+        // can manually add props in FieldEditor or reject the component.
+        // Downstream consumers (buildManifest, push) tolerate empty $properties.
         const $properties = {};
         for (const p of compProps) {
             // Hallucination insurance: drop any prop whose name didn't survive trim.
@@ -768,8 +978,42 @@ export function loadCDFComponents(db, sessionId) {
         if (Object.keys($slots).length > 0)
             entry.$slots = $slots;
         return { key: name, entry };
-    })
-        .filter((c) => c !== null);
+    });
+}
+export function loadScopeComponents(db, sessionId) {
+    // Loads extract-stage components: those still pending operator review (`extracted`)
+    // plus those the Feature 3 auto-filter has already classified (`accepted` /
+    // `rejected`). Excludes `generated` (already past scope-gate) so re-runs of the
+    // wizard don't show already-confirmed components.
+    const rows = db
+        .prepare(`SELECT name, component_id, status, reject_reason FROM raw_components
+       WHERE session_id = ? AND status IN ('extracted', 'accepted', 'rejected')
+       ORDER BY name`)
+        .all(sessionId);
+    return rows.map((r) => ({
+        name: r.name,
+        componentId: r.component_id,
+        aiDecision: r.status === 'accepted' ? 'accepted' : r.status === 'rejected' ? 'rejected' : null,
+        aiReason: r.reject_reason,
+    }));
+}
+export function applyScopeDecisions(db, sessionId, decisions) {
+    const now = new Date().toISOString();
+    const acceptedSet = new Set(decisions.accepted);
+    const accepted = [...acceptedSet];
+    // Conflict precedence: accepted wins over rejected.
+    const rejected = [...new Set(decisions.rejected)].filter((n) => !acceptedSet.has(n));
+    // Rejected first so accepted writes can flip back to 'generated' afterwards
+    // if a name appears in both lists.
+    if (rejected.length > 0) {
+        const placeholders = rejected.map(() => '?').join(',');
+        db.prepare(`UPDATE raw_components SET status = 'rejected' WHERE session_id = ? AND name IN (${placeholders})`).run(sessionId, ...rejected);
+    }
+    if (accepted.length > 0) {
+        const placeholders = accepted.map(() => '?').join(',');
+        db.prepare(`UPDATE raw_components SET status = 'generated' WHERE session_id = ? AND name IN (${placeholders})`).run(sessionId, ...accepted);
+    }
+    db.prepare('UPDATE sessions SET updated_at = ? WHERE id = ?').run(now, sessionId);
 }
 export function storeDTCGTokens(db, sessionId, groups, tokens) {
     const now = new Date().toISOString();
@@ -1053,37 +1297,36 @@ function mapServerTypeToCDFType(serverType) {
     }
 }
 export function computeComponentInputHash(component) {
+    // Narrow the hashed payload to extractor-only fields. Any field that
+    // `applyToolCalls` may overwrite (prop description/required/default/allowed
+    // values, slot description/allowed components) is excluded so that the cache
+    // key does not depend on its own output — otherwise re-reading raw props
+    // after a generation pass would drift the hash even when the underlying
+    // source has not changed.
     const payload = {
         framework: component.framework,
         name: component.name,
+        source: component.source,
         props: component.props.map((p) => ({
-            allowedValues: p.allowedValues ?? [],
-            defaultValue: p.defaultValue ?? null,
-            description: p.description ?? null,
             name: p.name,
-            required: p.required,
-            tokenReference: p.tokenReference ?? null,
             type: p.type,
         })),
         slots: component.slots.map((s) => ({
-            allowedComponents: s.allowedComponents ?? [],
-            description: s.description ?? null,
-            isDefault: s.isDefault,
             name: s.name,
+            isDefault: s.isDefault,
         })),
-        source: component.source,
     };
     return createHash('sha256').update(JSON.stringify(payload)).digest('hex');
 }
 export function computeTokenInputHash(rawTokenContent) {
     return createHash('sha256').update(rawTokenContent.trim()).digest('hex');
 }
-export function lookupCache(db, inputHash, entityType, entityId) {
+export function lookupCache(db, inputHash, entityType, entityId, promptHash = '') {
     const row = db
-        .prepare(`SELECT input_hash, entity_type, entity_id, source_session_id, human_edited, created_at, updated_at
+        .prepare(`SELECT input_hash, entity_type, entity_id, source_session_id, human_edited, created_at, updated_at, prompt_hash
        FROM generation_cache
-       WHERE input_hash = ? AND entity_type = ? AND entity_id = ?`)
-        .get(inputHash, entityType, entityId);
+       WHERE input_hash = ? AND entity_type = ? AND entity_id = ? AND prompt_hash = ?`)
+        .get(inputHash, entityType, entityId, promptHash);
     if (!row)
         return null;
     return {
@@ -1092,13 +1335,14 @@ export function lookupCache(db, inputHash, entityType, entityId) {
         entityId: row.entity_id,
         sourceSessionId: row.source_session_id,
         humanEdited: row.human_edited === 1,
+        promptHash: row.prompt_hash,
         createdAt: row.created_at,
         updatedAt: row.updated_at,
     };
 }
 export function lookupCacheByEntity(db, entityType, entityId) {
     const row = db
-        .prepare(`SELECT input_hash, entity_type, entity_id, source_session_id, human_edited, created_at, updated_at
+        .prepare(`SELECT input_hash, entity_type, entity_id, source_session_id, human_edited, created_at, updated_at, prompt_hash
        FROM generation_cache
        WHERE entity_type = ? AND entity_id = ?
        ORDER BY updated_at DESC LIMIT 1`)
@@ -1111,18 +1355,19 @@ export function lookupCacheByEntity(db, entityType, entityId) {
         entityId: row.entity_id,
         sourceSessionId: row.source_session_id,
         humanEdited: row.human_edited === 1,
+        promptHash: row.prompt_hash,
         createdAt: row.created_at,
         updatedAt: row.updated_at,
     };
 }
-export function storeCache(db, inputHash, entityType, entityId, sourceSessionId, humanEdited) {
+export function storeCache(db, inputHash, entityType, entityId, sourceSessionId, humanEdited, promptHash = '') {
     const now = new Date().toISOString();
-    db.prepare(`INSERT INTO generation_cache (input_hash, entity_type, entity_id, source_session_id, human_edited, created_at, updated_at)
-     VALUES (?, ?, ?, ?, ?, ?, ?)
-     ON CONFLICT(input_hash, entity_type, entity_id) DO UPDATE SET
+    db.prepare(`INSERT INTO generation_cache (input_hash, entity_type, entity_id, source_session_id, human_edited, created_at, updated_at, prompt_hash)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(input_hash, prompt_hash, entity_type, entity_id) DO UPDATE SET
        source_session_id = excluded.source_session_id,
        human_edited = CASE WHEN generation_cache.human_edited = 1 THEN 1 ELSE excluded.human_edited END,
-       updated_at = excluded.updated_at`).run(inputHash, entityType, entityId, sourceSessionId, humanEdited ? 1 : 0, now, now);
+       updated_at = excluded.updated_at`).run(inputHash, entityType, entityId, sourceSessionId, humanEdited ? 1 : 0, now, now, promptHash);
 }
 export function storeScannedFiles(db, sessionId, filePaths) {
     db.exec('BEGIN');
@@ -1213,3 +1458,101 @@ export function copyTokensFromCache(db, sourceSessionId, targetSessionId) {
         throw e;
     }
 }
+// ---------------------------------------------------------------------------
+// Fine-grained cache: extract_cache helpers
+// ---------------------------------------------------------------------------
+let _cliCacheVersionCache = null;
+/**
+ * Returns the CLI cache version used to namespace cache rows. To avoid busting
+ * the cache on every npm patch bump, this hashes the content of the bundled
+ * skill prompts. When the prompts change, cache invalidates; when only library
+ * code or test fixtures change, the cache stays warm.
+ *
+ * Memoized per-process. Falls back to package version on any I/O error.
+ */
+export async function getCliCacheVersion() {
+    if (_cliCacheVersionCache)
+        return _cliCacheVersionCache;
+    try {
+        const { hashContent } = await import('./cache-keys.js');
+        const { resolveSkillPath } = await import('../generate/prompt-builder.js');
+        const skills = ['components', 'tokens', 'select'];
+        const parts = [];
+        for (const s of skills) {
+            try {
+                const p = resolveSkillPath(s);
+                parts.push(readFileSync(p, 'utf8'));
+            }
+            catch {
+                parts.push(`<missing:${s}>`);
+            }
+        }
+        _cliCacheVersionCache = hashContent(parts.join('\n---\n'));
+        return _cliCacheVersionCache;
+    }
+    catch {
+        _cliCacheVersionCache = 'fallback';
+        return _cliCacheVersionCache;
+    }
+}
+export function storeExtractCache(db, filePath, fileHash, cliVersion, components) {
+    const now = new Date().toISOString();
+    const componentsJson = JSON.stringify(components);
+    db.prepare(`INSERT INTO extract_cache (file_path, file_hash, cli_version, created_at, updated_at, components_json)
+     VALUES (?, ?, ?, ?, ?, ?)
+     ON CONFLICT(file_hash, cli_version) DO UPDATE SET
+       file_path = excluded.file_path,
+       updated_at = excluded.updated_at,
+       components_json = excluded.components_json`).run(filePath, fileHash, cliVersion, now, now, componentsJson);
+}
+export function lookupExtractCache(db, fileHash, cliVersion) {
+    const row = db
+        .prepare(`SELECT file_path, file_hash, cli_version, created_at, updated_at, components_json
+       FROM extract_cache
+       WHERE file_hash = ? AND cli_version = ?`)
+        .get(fileHash, cliVersion);
+    if (!row)
+        return null;
+    let components = [];
+    try {
+        components = JSON.parse(row.components_json);
+    }
+    catch {
+        return null;
+    }
+    return {
+        filePath: row.file_path,
+        fileHash: row.file_hash,
+        cliVersion: row.cli_version,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+        components,
+    };
+}
+export function storeSelectCache(db, componentHash, promptHash, cliVersion, decision, reason) {
+    const now = new Date().toISOString();
+    db.prepare(`INSERT INTO select_cache (component_hash, prompt_hash, cli_version, decision, reason, created_at, updated_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(component_hash, prompt_hash, cli_version) DO UPDATE SET
+       decision = excluded.decision,
+       reason = excluded.reason,
+       updated_at = excluded.updated_at`).run(componentHash, promptHash, cliVersion, decision, reason, now, now);
+}
+export function lookupSelectCache(db, componentHash, promptHash, cliVersion) {
+    const row = db
+        .prepare(`SELECT component_hash, prompt_hash, cli_version, decision, reason, created_at, updated_at
+       FROM select_cache
+       WHERE component_hash = ? AND prompt_hash = ? AND cli_version = ?`)
+        .get(componentHash, promptHash, cliVersion);
+    if (!row)
+        return null;
+    return {
+        componentHash: row.component_hash,
+        promptHash: row.prompt_hash,
+        cliVersion: row.cli_version,
+        decision: row.decision,
+        reason: row.reason,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+    };
+}