mustflow 2.115.3 → 2.115.4

package/dist/cli/lib/local-index/constants.js

@@ -1,4 +1,4 @@
-export const LOCAL_INDEX_SCHEMA_VERSION = '20';
+export const LOCAL_INDEX_SCHEMA_VERSION = '21';
 export const LOCAL_INDEX_PARSER_VERSION = '1';
 export const DEFAULT_DATABASE_RELATIVE_PATH = '.mustflow/cache/mustflow.sqlite';
 export const LATEST_RUN_STATE_RELATIVE_PATH = '.mustflow/state/runs/latest.json';

package/dist/cli/lib/local-index/freshness.js

@@ -1,6 +1,6 @@
-import { LOCAL_INDEX_SCHEMA_VERSION } from './constants.js';
+import { DEFAULT_DATABASE_RELATIVE_PATH, LOCAL_INDEX_SCHEMA_VERSION } from './constants.js';
 import { hasTable, queryRows, readMetadataValue, toSearchString } from './database-read.js';
-import { normalizeIndexedFileSourceScope, readIndexedFileRecord } from './source-index.js';
+import { collectSourceAnchorCandidatePaths, getSourceScopeHash, normalizeIndexedFileSourceScope, readIndexedFileRecord, readLocalIndexSourceConfig, } from './source-index.js';
 import { collectDocuments } from './workflow-documents.js';
 export function readStoredSchemaVersion(database) {
     return readMetadataValue(database, 'schema_version');
@@ -11,46 +11,76 @@ export function getStalePaths(projectRoot, database, options = {}) {
     if (schemaVersion !== LOCAL_INDEX_SCHEMA_VERSION) {
         return ['.mustflow/cache/mustflow.sqlite'];
     }
-    if (hasTable(database, 'indexed_files')) {
-        const stalePaths = new Set();
-        const indexedRows = queryRows(database, 'SELECT path, source_scope, content_hash FROM indexed_files');
-        const indexedPaths = new Set(indexedRows.map((row) => toSearchString(row.path)));
-        for (const row of indexedRows) {
-            const indexedPath = toSearchString(row.path);
-            const sourceScope = normalizeIndexedFileSourceScope(toSearchString(row.source_scope));
-            if (!includeState && sourceScope === 'state') {
-                continue;
-            }
-            try {
-                const current = readIndexedFileRecord(projectRoot, indexedPath, sourceScope);
-                if (current.contentHash !== toSearchString(row.content_hash)) {
-                    stalePaths.add(indexedPath);
-                }
-            }
-            catch {
+    if (!hasTable(database, 'indexed_files') ||
+        !hasTable(database, 'indexed_source_candidates') ||
+        !hasTable(database, 'source_anchors') ||
+        !hasTable(database, 'source_anchor_status')) {
+        return [DEFAULT_DATABASE_RELATIVE_PATH];
+    }
+    const stalePaths = new Set();
+    const indexedRows = queryRows(database, 'SELECT path, source_scope, content_hash FROM indexed_files');
+    const indexedPaths = new Set(indexedRows.map((row) => toSearchString(row.path)));
+    const indexedSourcePaths = new Set(queryRows(database, 'SELECT path FROM indexed_source_candidates').map((row) => toSearchString(row.path)));
+    for (const sourcePath of indexedSourcePaths) {
+        if (!indexedPaths.has(sourcePath)) {
+            stalePaths.add(sourcePath);
+        }
+    }
+    for (const row of indexedRows) {
+        const indexedPath = toSearchString(row.path);
+        const sourceScope = normalizeIndexedFileSourceScope(toSearchString(row.source_scope));
+        if (!includeState && sourceScope === 'state') {
+            continue;
+        }
+        try {
+            const current = readIndexedFileRecord(projectRoot, indexedPath, sourceScope);
+            if (current.contentHash !== toSearchString(row.content_hash)) {
                 stalePaths.add(indexedPath);
             }
         }
-        for (const document of collectDocuments(projectRoot)) {
-            if (!indexedPaths.has(document.path)) {
-                stalePaths.add(document.path);
-            }
+        catch {
+            stalePaths.add(indexedPath);
         }
-        return Array.from(stalePaths).sort((left, right) => left.localeCompare(right));
     }
-    const indexedRows = queryRows(database, 'SELECT path, content_hash FROM documents');
-    const indexedHashes = new Map(indexedRows.map((row) => [toSearchString(row.path), toSearchString(row.content_hash)]));
-    const currentDocuments = collectDocuments(projectRoot);
-    const currentHashes = new Map(currentDocuments.map((document) => [document.path, document.contentHash]));
-    const stalePaths = new Set();
-    for (const [indexedPath, indexedHash] of indexedHashes) {
-        if (currentHashes.get(indexedPath) !== indexedHash) {
-            stalePaths.add(indexedPath);
+    for (const document of collectDocuments(projectRoot)) {
+        if (!indexedPaths.has(document.path)) {
+            stalePaths.add(document.path);
         }
     }
-    for (const currentPath of currentHashes.keys()) {
-        if (!indexedHashes.has(currentPath)) {
-            stalePaths.add(currentPath);
+    const sourceIndexEnabledValue = readMetadataValue(database, 'source_index_enabled');
+    if (sourceIndexEnabledValue !== 'true' && sourceIndexEnabledValue !== 'false') {
+        stalePaths.add(DEFAULT_DATABASE_RELATIVE_PATH);
+    }
+    else {
+        const sourceIndexEnabled = sourceIndexEnabledValue === 'true';
+        const hasIndexedSourceScope = indexedRows.some((row) => normalizeIndexedFileSourceScope(toSearchString(row.source_scope)) === 'source_anchor');
+        const hasSourceAnchorRows = hasTable(database, 'source_anchors') && queryRows(database, 'SELECT id FROM source_anchors LIMIT 1').length > 0;
+        if (!sourceIndexEnabled && (indexedSourcePaths.size > 0 || hasIndexedSourceScope || hasSourceAnchorRows)) {
+            stalePaths.add(DEFAULT_DATABASE_RELATIVE_PATH);
+        }
+        try {
+            const sourceConfig = readLocalIndexSourceConfig(projectRoot);
+            const storedSourceScopeHash = readMetadataValue(database, 'source_scope_hash');
+            const currentSourceScopeHash = getSourceScopeHash(sourceIndexEnabled, sourceConfig);
+            if (storedSourceScopeHash !== currentSourceScopeHash) {
+                stalePaths.add(DEFAULT_DATABASE_RELATIVE_PATH);
+            }
+            if (sourceIndexEnabled) {
+                const currentSourcePaths = new Set(collectSourceAnchorCandidatePaths(projectRoot, sourceConfig));
+                for (const sourcePath of currentSourcePaths) {
+                    if (!indexedSourcePaths.has(sourcePath)) {
+                        stalePaths.add(sourcePath);
+                    }
+                }
+                for (const sourcePath of indexedSourcePaths) {
+                    if (!currentSourcePaths.has(sourcePath)) {
+                        stalePaths.add(sourcePath);
+                    }
+                }
+            }
+        }
+        catch {
+            stalePaths.add(DEFAULT_DATABASE_RELATIVE_PATH);
         }
     }
     return Array.from(stalePaths).sort((left, right) => left.localeCompare(right));

package/dist/cli/lib/local-index/index.js

@@ -158,6 +158,14 @@ function indexedFilesMatch(database, currentFiles) {
     }
     return true;
 }
+function indexedSourceCandidatesMatch(database, currentPaths) {
+    if (!hasTable(database, 'indexed_source_candidates')) {
+        return false;
+    }
+    const storedPaths = new Set(queryRows(database, 'SELECT path FROM indexed_source_candidates').map((row) => toSearchString(row.path)));
+    const currentPathSet = new Set(currentPaths);
+    return storedPaths.size === currentPathSet.size && [...storedPaths].every((storedPath) => currentPathSet.has(storedPath));
+}
 const INDEXED_FILE_MTIME_TOLERANCE_MS = 1;
 function indexedFileMtimeMsEqual(storedMtimeMs, currentMtimeMs) {
     return storedMtimeMs !== null && Math.abs(storedMtimeMs - currentMtimeMs) <= INDEXED_FILE_MTIME_TOLERANCE_MS;
@@ -183,8 +191,8 @@ function indexedFileMetadataMatch(database, currentFiles) {
     }
     return true;
 }
-async function readIncrementalPreflightReuse(SQL, databasePath, projectRoot, currentFiles, sourceScopeHash, dryRun, indexMode) {
-    if (!currentFiles) {
+async function readIncrementalPreflightReuse(SQL, databasePath, projectRoot, currentFiles, currentSourceCandidatePaths, sourceScopeHash, dryRun, indexMode) {
+    if (!currentFiles || !currentSourceCandidatePaths) {
         return { result: null, rebuildReason: null };
     }
     if (!existsSync(databasePath)) {
@@ -205,6 +213,9 @@ async function readIncrementalPreflightReuse(SQL, databasePath, projectRoot, cur
         if (!hasTable(database, 'indexed_files')) {
             return { result: null, rebuildReason: 'indexed_files_missing' };
         }
+        if (!indexedSourceCandidatesMatch(database, currentSourceCandidatePaths)) {
+            return { result: null, rebuildReason: 'source_scope_mismatch' };
+        }
         if (!indexedFileMetadataMatch(database, currentFiles)) {
             return { result: null, rebuildReason: 'file_fingerprint_mismatch' };
         }
@@ -225,7 +236,7 @@ async function readIncrementalPreflightReuse(SQL, databasePath, projectRoot, cur
         database?.close();
     }
 }
-async function readIncrementalReuseDecision(SQL, databasePath, currentFiles, sourceScopeHash) {
+async function readIncrementalReuseDecision(SQL, databasePath, currentFiles, currentSourceCandidatePaths, sourceScopeHash) {
     if (!existsSync(databasePath)) {
         return { reusable: false, rebuildReason: 'missing_index', capabilities: null };
     }
@@ -244,6 +255,9 @@ async function readIncrementalReuseDecision(SQL, databasePath, currentFiles, sou
         if (!hasTable(database, 'indexed_files')) {
             return { reusable: false, rebuildReason: 'indexed_files_missing', capabilities: null };
         }
+        if (!indexedSourceCandidatesMatch(database, currentSourceCandidatePaths)) {
+            return { reusable: false, rebuildReason: 'source_scope_mismatch', capabilities: null };
+        }
         if (!indexedFilesMatch(database, currentFiles)) {
             return { reusable: false, rebuildReason: 'file_fingerprint_mismatch', capabilities: null };
         }
@@ -284,7 +298,7 @@ export async function createLocalIndex(projectRoot, options = {}) {
     capabilityDatabase.close();
     if (incremental) {
         const preflightFiles = collectFastPreflightIndexedFileMetadataRecords(projectRoot, includeSource, sourceConfig);
-        const preflightReuse = await readIncrementalPreflightReuse(SQL, databasePath, projectRoot, preflightFiles, sourceScopeHash, dryRun, indexMode);
+        const preflightReuse = await readIncrementalPreflightReuse(SQL, databasePath, projectRoot, preflightFiles?.records ?? null, preflightFiles?.sourceCandidatePaths ?? null, sourceScopeHash, dryRun, indexMode);
         if (preflightReuse.result) {
             return preflightReuse.result;
         }
@@ -307,16 +321,16 @@ export async function createLocalIndex(projectRoot, options = {}) {
     const verificationEvidence = createVerificationEvidenceIndex(projectRoot);
     const indexedFiles = collectIndexedFileRecords(projectRoot, documents, sourceAnchors, sourceAnchorCandidatePaths);
     if (incremental) {
-        const reuseDecision = await readIncrementalReuseDecision(SQL, databasePath, indexedFiles, sourceScopeHash);
+        const reuseDecision = await readIncrementalReuseDecision(SQL, databasePath, indexedFiles, sourceAnchorCandidatePaths, sourceScopeHash);
         reusedExisting = reuseDecision.reusable;
-        rebuildReason = reuseDecision.rebuildReason ?? rebuildReason;
+        rebuildReason = reuseDecision.reusable ? null : (reuseDecision.rebuildReason ?? rebuildReason);
         capabilities = reuseDecision.capabilities ?? capabilities;
     }
     if (!dryRun && !reusedExisting) {
         const database = new SQL.Database();
         try {
             createSchema(database, capabilities);
-            populateDatabase(database, capabilities, documents, skills, skillRoutes, commandIntents, sourceAnchors, indexedFiles, verificationEvidence, indexMode, sourceScopeHash, includeSource, new Date().toISOString());
+            populateDatabase(database, capabilities, documents, skills, skillRoutes, commandIntents, sourceAnchors, indexedFiles, sourceAnchorCandidatePaths, verificationEvidence, indexMode, sourceScopeHash, includeSource, new Date().toISOString());
             writeFileInsideWithoutSymlinks(projectRoot, databasePath, database.export());
         }
         finally {
@@ -702,10 +716,13 @@ export async function readLocalSourceAnchorCheckWarnings(projectRoot) {
         if (readMetadataValue(database, 'source_index_enabled') !== 'true') {
             return [];
         }
-        if (!hasTable(database, 'indexed_files') || !hasTable(database, 'source_anchors') || !hasTable(database, 'source_anchor_status')) {
+        if (!hasTable(database, 'indexed_files') ||
+            !hasTable(database, 'indexed_source_candidates') ||
+            !hasTable(database, 'source_anchors') ||
+            !hasTable(database, 'source_anchor_status')) {
             return ['source anchor local index is missing source-anchor tables; refresh with mf index --source'];
         }
-        const indexedSourcePaths = new Set(queryRows(database, 'SELECT path FROM indexed_files WHERE source_scope = "source_anchor"')
+        const indexedSourcePaths = new Set(queryRows(database, 'SELECT path FROM indexed_source_candidates')
             .map((row) => toSearchString(row.path))
             .filter(Boolean));
         const staleSourcePaths = getStalePaths(projectRoot, database, { includeState: false }).filter((stalePath) => indexedSourcePaths.has(stalePath));

package/dist/cli/lib/local-index/populate.js

@@ -172,7 +172,7 @@ function rollbackTransaction(database) {
         // Keep the original indexing failure as the actionable error.
     }
 }
-export function populateDatabase(database, capabilities, documents, skills, skillRoutes, commandIntents, sourceAnchors, indexedFiles, verificationEvidence, indexMode, sourceScopeHash, sourceIndexEnabled, indexedAt) {
+export function populateDatabase(database, capabilities, documents, skills, skillRoutes, commandIntents, sourceAnchors, indexedFiles, indexedSourceCandidates, verificationEvidence, indexMode, sourceScopeHash, sourceIndexEnabled, indexedAt) {
     const sourceAnchorRiskSignals = createSourceAnchorRiskSignals(sourceAnchors);
     database.run('BEGIN TRANSACTION');
     try {
@@ -223,6 +223,9 @@ export function populateDatabase(database, capabilities, documents, skills, skil
                 LOCAL_INDEX_PARSER_VERSION,
             ]);
         }
+        for (const sourceCandidatePath of indexedSourceCandidates) {
+            database.run('INSERT INTO indexed_source_candidates (path) VALUES (?)', [sourceCandidatePath]);
+        }
         for (const document of documents) {
             database.run('INSERT INTO documents (path, type, title, locale, revision, content_hash, content_snippet) VALUES (?, ?, ?, ?, ?, ?, ?)', [
                 document.path,

package/dist/cli/lib/local-index/schema.js

@@ -1,5 +1,10 @@
 import { SEARCH_BACKEND_FTS5 } from './constants.js';
 export function createSchema(database, capabilities) {
+    database.run('PRAGMA foreign_keys = ON');
+    const [foreignKeyResult] = database.exec('PRAGMA foreign_keys');
+    if (foreignKeyResult?.values[0]?.[0] !== 1) {
+        throw new Error('SQLite foreign-key enforcement is unavailable for the local index.');
+    }
     database.run(`
 CREATE TABLE metadata (
   key TEXT PRIMARY KEY,
@@ -17,6 +22,11 @@ CREATE TABLE indexed_files (
   parser_version TEXT NOT NULL
 );
 
+CREATE TABLE indexed_source_candidates (
+  path TEXT PRIMARY KEY,
+  FOREIGN KEY (path) REFERENCES indexed_files(path) ON UPDATE RESTRICT ON DELETE RESTRICT
+);
+
 CREATE TABLE documents (
   path TEXT PRIMARY KEY,
   type TEXT NOT NULL,

package/dist/cli/lib/local-index/source-index.js

@@ -1,11 +1,34 @@
-import { existsSync, readFileSync, statSync } from 'node:fs';
+import { existsSync, statSync } from 'node:fs';
 import path from 'node:path';
 import { isRecord, readStringArray } from '../command-contract.js';
+import { MUSTFLOW_TEXT_MAX_BYTES, mustflowProjectPath } from '../mustflow-read.js';
 import { readMustflowTomlFile } from '../toml.js';
+import { ensureInsideWithoutSymlinks, readFileInsideWithoutSymlinks } from '../../../core/safe-filesystem.js';
 import { listSourceAnchorFiles } from '../../../core/source-anchors.js';
 import { INDEX_CONFIG_RELATIVE_PATH, LATEST_RUN_STATE_RELATIVE_PATH, MUSTFLOW_RELATIVE_PATH, SOURCE_INDEX_MAX_FILE_BYTES, } from './constants.js';
 import { sha256Bytes, sha256Text } from './hashing.js';
 import { getExistingIndexablePaths } from './workflow-documents.js';
+function indexedProjectPath(projectRoot, relativePath) {
+    if (relativePath.length === 0 ||
+        relativePath.includes('\0') ||
+        relativePath.includes('\\') ||
+        path.posix.isAbsolute(relativePath) ||
+        path.win32.isAbsolute(relativePath)) {
+        throw new Error(`Indexed file path must be a canonical project-relative path: ${relativePath}`);
+    }
+    const segments = relativePath.split('/');
+    if (segments.some((segment) => segment.length === 0 || segment === '.' || segment === '..')) {
+        throw new Error(`Indexed file path must be a canonical project-relative path: ${relativePath}`);
+    }
+    const fullPath = mustflowProjectPath(projectRoot, relativePath);
+    ensureInsideWithoutSymlinks(projectRoot, fullPath);
+    return fullPath;
+}
+function readIndexedFileBytes(projectRoot, relativePath) {
+    return readFileInsideWithoutSymlinks(projectRoot, indexedProjectPath(projectRoot, relativePath), {
+        maxBytes: MUSTFLOW_TEXT_MAX_BYTES,
+    });
+}
 function readIndexToml(projectRoot) {
     const indexConfigPath = path.join(projectRoot, ...INDEX_CONFIG_RELATIVE_PATH.split('/'));
     if (!existsSync(indexConfigPath)) {
@@ -69,13 +92,13 @@ export function readIndexedFileRecord(projectRoot, relativePath, sourceScope, co
     const metadata = readIndexedFileMetadataRecord(projectRoot, relativePath, sourceScope);
     return {
         ...metadata,
-        contentHash: contentHash ?? sha256Bytes(readFileSync(path.join(projectRoot, ...relativePath.split('/')))),
+        contentHash: contentHash ?? sha256Bytes(readIndexedFileBytes(projectRoot, relativePath)),
     };
 }
 function hashIndexedFileMetadataRecord(projectRoot, metadata) {
     return {
         ...metadata,
-        contentHash: sha256Bytes(readFileSync(path.join(projectRoot, ...metadata.path.split('/')))),
+        contentHash: sha256Bytes(readIndexedFileBytes(projectRoot, metadata.path)),
     };
 }
 function tryHashIndexedFileMetadataRecord(projectRoot, metadata) {
@@ -92,8 +115,11 @@ export function hashIndexedFileMetadataRecords(projectRoot, metadataRecords) {
         .filter((record) => Boolean(record));
 }
 export function readIndexedFileMetadataRecord(projectRoot, relativePath, sourceScope) {
-    const fullPath = path.join(projectRoot, ...relativePath.split('/'));
+    const fullPath = indexedProjectPath(projectRoot, relativePath);
     const stats = statSync(fullPath);
+    if (!stats.isFile()) {
+        throw new Error(`Indexed path must be a regular file: ${relativePath}`);
+    }
     return {
         path: relativePath,
         sourceScope,
@@ -117,10 +143,7 @@ export function collectIndexedFileRecords(projectRoot, documents, sourceAnchors,
     const sourcePaths = new Set([...sourceAnchorCandidatePaths, ...sourceAnchors.map((anchor) => anchor.path)]);
     for (const anchorPath of [...sourcePaths].sort((left, right) => left.localeCompare(right))) {
         if (!records.has(anchorPath)) {
-            const record = tryReadIndexedFileRecord(projectRoot, anchorPath, 'source_anchor');
-            if (record) {
-                records.set(anchorPath, record);
-            }
+            records.set(anchorPath, readIndexedFileRecord(projectRoot, anchorPath, 'source_anchor'));
         }
     }
     if (existsSync(path.join(projectRoot, ...LATEST_RUN_STATE_RELATIVE_PATH.split('/')))) {
@@ -132,19 +155,25 @@ export function collectIndexedFileRecords(projectRoot, documents, sourceAnchors,
     return [...records.values()].sort((left, right) => left.path.localeCompare(right.path));
 }
 export function collectSourceAnchorCandidatePaths(projectRoot, sourceConfig) {
-    return listSourceAnchorFiles(projectRoot, {
+    const paths = listSourceAnchorFiles(projectRoot, {
         ...sourceConfig,
         excludeGeneratedOrVendor: true,
     });
+    for (const relativePath of paths) {
+        indexedProjectPath(projectRoot, relativePath);
+    }
+    return paths;
 }
 export function collectFastPreflightIndexedFileMetadataRecords(projectRoot, includeSource, sourceConfig) {
     const records = new Map();
+    let sourceCandidatePaths = [];
     for (const relativePath of getExistingIndexablePaths(projectRoot)) {
         records.set(relativePath, readIndexedFileMetadataRecord(projectRoot, relativePath, 'workflow'));
     }
     if (includeSource) {
         try {
-            for (const sourcePath of collectSourceAnchorCandidatePaths(projectRoot, sourceConfig)) {
+            sourceCandidatePaths = collectSourceAnchorCandidatePaths(projectRoot, sourceConfig);
+            for (const sourcePath of sourceCandidatePaths) {
                 if (!records.has(sourcePath)) {
                     records.set(sourcePath, readIndexedFileMetadataRecord(projectRoot, sourcePath, 'source_anchor'));
                 }
@@ -157,5 +186,8 @@ export function collectFastPreflightIndexedFileMetadataRecords(projectRoot, incl
     if (existsSync(path.join(projectRoot, ...LATEST_RUN_STATE_RELATIVE_PATH.split('/')))) {
         records.set(LATEST_RUN_STATE_RELATIVE_PATH, readIndexedFileMetadataRecord(projectRoot, LATEST_RUN_STATE_RELATIVE_PATH, 'state'));
     }
-    return [...records.values()].sort((left, right) => left.path.localeCompare(right.path));
+    return {
+        records: [...records.values()].sort((left, right) => left.path.localeCompare(right.path)),
+        sourceCandidatePaths,
+    };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mustflow",
-  "version": "2.115.3",
+  "version": "2.115.4",
   "description": "Agent workflow documents and CLI for mustflow repository roots.",
   "type": "module",
   "license": "MIT-0",

package/templates/default/manifest.toml

@@ -1,6 +1,6 @@
 id = "default"
 name = "default"
-version = "2.115.3"
+version = "2.115.4"
 description = "Minimal workflow for LLM agents to read, edit, and verify their work in a repository."
 common_root = "common"
 locales_root = "locales"