mustflow 1.18.0 → 1.18.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +16 -6
  2. package/dist/cli/commands/context.js +2 -2
  3. package/dist/cli/commands/dashboard.js +61 -7
  4. package/dist/cli/commands/explain.js +47 -7
  5. package/dist/cli/commands/index.js +9 -2
  6. package/dist/cli/commands/run.js +7 -15
  7. package/dist/cli/commands/verify.js +44 -9
  8. package/dist/cli/i18n/en.js +3 -0
  9. package/dist/cli/i18n/es.js +3 -0
  10. package/dist/cli/i18n/fr.js +3 -0
  11. package/dist/cli/i18n/hi.js +3 -0
  12. package/dist/cli/i18n/ko.js +3 -0
  13. package/dist/cli/i18n/zh.js +3 -0
  14. package/dist/cli/lib/agent-context.js +19 -4
  15. package/dist/cli/lib/dashboard-html.js +41 -0
  16. package/dist/cli/lib/dashboard-locale.js +2 -0
  17. package/dist/cli/lib/local-index.js +910 -32
  18. package/dist/core/change-classification.js +33 -60
  19. package/dist/core/command-classification.js +0 -2
  20. package/dist/core/source-anchor-status.js +4 -4
  21. package/dist/core/source-anchor-validation.js +2 -6
  22. package/dist/core/source-anchors.js +81 -3
  23. package/package.json +1 -1
  24. package/schemas/change-verification-report.schema.json +194 -0
  25. package/schemas/context-report.schema.json +30 -2
  26. package/schemas/explain-report.schema.json +191 -0
  27. package/templates/default/i18n.toml +16 -6
  28. package/templates/default/locales/en/.mustflow/skills/INDEX.md +2 -1
  29. package/templates/default/locales/en/.mustflow/skills/database-change-safety/SKILL.md +155 -0
  30. package/templates/default/locales/en/AGENTS.md +5 -5
  31. package/templates/default/locales/es/.mustflow/skills/INDEX.md +2 -1
  32. package/templates/default/locales/es/.mustflow/skills/database-change-safety/SKILL.md +155 -0
  33. package/templates/default/locales/fr/.mustflow/skills/INDEX.md +2 -1
  34. package/templates/default/locales/fr/.mustflow/skills/database-change-safety/SKILL.md +155 -0
  35. package/templates/default/locales/hi/.mustflow/skills/INDEX.md +2 -1
  36. package/templates/default/locales/hi/.mustflow/skills/database-change-safety/SKILL.md +155 -0
  37. package/templates/default/locales/ko/.mustflow/skills/INDEX.md +2 -1
  38. package/templates/default/locales/ko/.mustflow/skills/database-change-safety/SKILL.md +155 -0
  39. package/templates/default/locales/zh/.mustflow/skills/INDEX.md +2 -1
  40. package/templates/default/locales/zh/.mustflow/skills/database-change-safety/SKILL.md +155 -0
  41. package/templates/default/manifest.toml +7 -1
@@ -1,4 +1,4 @@
1
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
1
+ import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
2
2
  import { createRequire } from 'node:module';
3
3
  import { createHash } from 'node:crypto';
4
4
  import path from 'node:path';
@@ -7,12 +7,20 @@ import { listFilesRecursive, toPosixPath } from './filesystem.js';
7
7
  import { readTomlFile } from './toml.js';
8
8
  import { collectSourceAnchorIndexRecords, } from '../../core/source-anchor-status.js';
9
9
  import { normalizeCommandEffects } from '../../core/command-effects.js';
10
- const LOCAL_INDEX_SCHEMA_VERSION = '7';
10
+ import { listChangeClassificationRuleDescriptors } from '../../core/change-classification.js';
11
+ const LOCAL_INDEX_SCHEMA_VERSION = '12';
12
+ const LOCAL_INDEX_PARSER_VERSION = '1';
11
13
  const DEFAULT_DATABASE_RELATIVE_PATH = '.mustflow/cache/mustflow.sqlite';
12
14
  const LOCAL_INDEX_CONTENT_MODE = 'metadata_and_snippets';
13
15
  const LOCAL_INDEX_STORE_FULL_CONTENT = false;
14
16
  const MAX_SNIPPET_BYTES_PER_DOCUMENT = 2048;
17
+ const SEARCH_NGRAM_MIN_LENGTH = 2;
18
+ const SEARCH_NGRAM_MAX_LENGTH = 3;
19
+ const SEARCH_BACKEND_FTS5 = 'fts5';
20
+ const SEARCH_BACKEND_TABLE_SCAN = 'table_scan';
21
+ const TEST_DISABLE_FTS5_ENV = 'MUSTFLOW_TEST_DISABLE_FTS5';
15
22
  const MUSTFLOW_RELATIVE_PATH = '.mustflow/config/mustflow.toml';
23
+ const INDEX_CONFIG_RELATIVE_PATH = '.mustflow/config/index.toml';
16
24
  const DEFAULT_PROMPT_CACHE_STABLE_READ = [
17
25
  'AGENTS.md',
18
26
  '.mustflow/docs/agent-workflow.md',
@@ -76,6 +84,14 @@ function readMustflowToml(projectRoot) {
76
84
  const parsed = readTomlFile(mustflowPath);
77
85
  return isRecord(parsed) ? parsed : undefined;
78
86
  }
87
+ function readIndexToml(projectRoot) {
88
+ const indexConfigPath = path.join(projectRoot, ...INDEX_CONFIG_RELATIVE_PATH.split('/'));
89
+ if (!existsSync(indexConfigPath)) {
90
+ return undefined;
91
+ }
92
+ const parsed = readTomlFile(indexConfigPath);
93
+ return isRecord(parsed) ? parsed : undefined;
94
+ }
79
95
  function readNestedTable(table, key) {
80
96
  if (!table || !isRecord(table[key])) {
81
97
  return undefined;
@@ -85,9 +101,39 @@ function readNestedTable(table, key) {
85
101
  function readOptionalStringArray(table, key) {
86
102
  return table ? readStringArray(table, key) ?? null : null;
87
103
  }
104
+ function readBoolean(table, key) {
105
+ const value = table?.[key];
106
+ return typeof value === 'boolean' ? value : undefined;
107
+ }
108
+ function readPositiveInteger(table, key) {
109
+ const value = table?.[key];
110
+ if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) {
111
+ return null;
112
+ }
113
+ return value;
114
+ }
115
+ function readLocalIndexSourceConfig(projectRoot) {
116
+ const sourceIndexTable = readNestedTable(readIndexToml(projectRoot), 'source_index');
117
+ return {
118
+ enabledByDefault: readBoolean(sourceIndexTable, 'enabled_by_default') === true,
119
+ include: readOptionalStringArray(sourceIndexTable, 'include') ?? [],
120
+ exclude: readOptionalStringArray(sourceIndexTable, 'exclude') ?? [],
121
+ maxFileBytes: readPositiveInteger(sourceIndexTable, 'max_file_bytes'),
122
+ allowedExtensions: readOptionalStringArray(sourceIndexTable, 'allowed_extensions') ?? [],
123
+ };
124
+ }
88
125
  function sha256Text(content) {
89
126
  return `sha256:${createHash('sha256').update(content).digest('hex')}`;
90
127
  }
128
+ function sha256Bytes(content) {
129
+ return `sha256:${createHash('sha256').update(content).digest('hex')}`;
130
+ }
131
+ function getSourceScopeHash(includeSource, sourceConfig) {
132
+ return sha256Text(JSON.stringify({
133
+ includeSource,
134
+ sourceConfig,
135
+ }));
136
+ }
91
137
  function getDocumentType(relativePath) {
92
138
  if (relativePath === 'AGENTS.md') {
93
139
  return 'agent_rules';
@@ -268,6 +314,29 @@ function collectCommandIntents(projectRoot) {
268
314
  }
269
315
  return intents;
270
316
  }
317
+ function readIndexedFileRecord(projectRoot, relativePath, sourceScope, contentHash = null) {
318
+ const fullPath = path.join(projectRoot, ...relativePath.split('/'));
319
+ const stats = statSync(fullPath);
320
+ return {
321
+ path: relativePath,
322
+ sourceScope,
323
+ sizeBytes: stats.size,
324
+ mtimeMs: Math.round(stats.mtimeMs),
325
+ contentHash: contentHash ?? sha256Bytes(readFileSync(fullPath)),
326
+ };
327
+ }
328
+ function collectIndexedFileRecords(projectRoot, documents, sourceAnchors) {
329
+ const records = new Map();
330
+ for (const document of documents) {
331
+ records.set(document.path, readIndexedFileRecord(projectRoot, document.path, 'workflow', document.contentHash));
332
+ }
333
+ for (const anchorPath of [...new Set(sourceAnchors.map((anchor) => anchor.path))].sort((left, right) => left.localeCompare(right))) {
334
+ if (!records.has(anchorPath)) {
335
+ records.set(anchorPath, readIndexedFileRecord(projectRoot, anchorPath, 'source_anchor'));
336
+ }
337
+ }
338
+ return [...records.values()].sort((left, right) => left.path.localeCompare(right.path));
339
+ }
271
340
  async function loadSqlJs() {
272
341
  const require = createRequire(import.meta.url);
273
342
  const wasmPath = require.resolve('sql.js/dist/sql-wasm.wasm');
@@ -285,6 +354,28 @@ async function loadSqlJs() {
285
354
  function normalizeSearchText(value) {
286
355
  return value.trim().replace(/\s+/g, ' ');
287
356
  }
357
+ function normalizeSearchTokenText(value) {
358
+ return normalizeSearchText(value).normalize('NFKC').toLowerCase();
359
+ }
360
+ function extractSearchTokens(value) {
361
+ return [...normalizeSearchTokenText(value).matchAll(/[\p{L}\p{N}]+/gu)]
362
+ .map((match) => match[0])
363
+ .filter((token) => Boolean(token));
364
+ }
365
+ function buildSearchNgrams(values) {
366
+ const grams = new Set();
367
+ for (const value of values) {
368
+ for (const token of extractSearchTokens(value)) {
369
+ const maxLength = Math.min(SEARCH_NGRAM_MAX_LENGTH, token.length);
370
+ for (let length = SEARCH_NGRAM_MIN_LENGTH; length <= maxLength; length += 1) {
371
+ for (let index = 0; index <= token.length - length; index += 1) {
372
+ grams.add(token.slice(index, index + length));
373
+ }
374
+ }
375
+ }
376
+ }
377
+ return [...grams].sort((left, right) => left.localeCompare(right));
378
+ }
288
379
  function toSearchString(value) {
289
380
  if (value === null || value === undefined) {
290
381
  return '';
@@ -307,12 +398,64 @@ function queryRows(database, sql, params = []) {
307
398
  return row;
308
399
  });
309
400
  }
401
+ function searchCapabilities(fts5Available) {
402
+ return {
403
+ backend: fts5Available ? SEARCH_BACKEND_FTS5 : SEARCH_BACKEND_TABLE_SCAN,
404
+ fts5Available,
405
+ };
406
+ }
407
+ function detectLocalSearchCapabilities(database) {
408
+ if (process.env[TEST_DISABLE_FTS5_ENV] === '1') {
409
+ return searchCapabilities(false);
410
+ }
411
+ try {
412
+ database.run('CREATE VIRTUAL TABLE temp.mustflow_fts5_probe USING fts5(value)');
413
+ database.run('DROP TABLE temp.mustflow_fts5_probe');
414
+ return searchCapabilities(true);
415
+ }
416
+ catch {
417
+ return searchCapabilities(false);
418
+ }
419
+ }
420
+ function readMetadataValue(database, key) {
421
+ return toSearchString(queryRows(database, 'SELECT value FROM metadata WHERE key = ?', [key])[0]?.value) || undefined;
422
+ }
423
+ function hasTable(database, tableName) {
424
+ return queryRows(database, 'SELECT name FROM sqlite_master WHERE type = "table" AND name = ?', [tableName]).length > 0;
425
+ }
426
+ function readStoredSearchCapabilities(database) {
427
+ const fts5Available = readMetadataValue(database, 'search_fts5_available') === 'true';
428
+ const backend = readMetadataValue(database, 'search_backend');
429
+ if (backend === SEARCH_BACKEND_FTS5 && hasTable(database, 'search_documents_fts')) {
430
+ return { backend: SEARCH_BACKEND_FTS5, fts5Available };
431
+ }
432
+ return { backend: SEARCH_BACKEND_TABLE_SCAN, fts5Available };
433
+ }
310
434
  function toNullableNumber(value) {
311
435
  if (typeof value !== 'number') {
312
436
  return null;
313
437
  }
314
438
  return Number.isFinite(value) ? value : null;
315
439
  }
440
+ function splitIndexedList(value) {
441
+ return toSearchString(value)
442
+ .split(',')
443
+ .map((item) => item.trim())
444
+ .filter(Boolean)
445
+ .sort((left, right) => left.localeCompare(right));
446
+ }
447
+ function createCommandEffectGraphStatus(databasePath, status, stalePaths = []) {
448
+ return {
449
+ source: 'local_index',
450
+ status,
451
+ databasePath,
452
+ indexFresh: status === 'fresh',
453
+ stalePaths,
454
+ writeLocks: [],
455
+ lockConflicts: [],
456
+ refreshHint: status === 'fresh' ? null : 'Run `mf index` to refresh command-effect graph explanations.',
457
+ };
458
+ }
316
459
  async function readPreviousSourceAnchorSnapshots(databasePath) {
317
460
  if (!existsSync(databasePath)) {
318
461
  return [];
@@ -480,12 +623,18 @@ function sourceAnchorAuthority() {
480
623
  function getMatchSnippet(fields, query) {
481
624
  const normalized = normalizeSearchText(fields.join(' '));
482
625
  const lower = normalized.toLowerCase();
483
- const start = lower.indexOf(query.toLowerCase());
626
+ let start = lower.indexOf(query.toLowerCase());
627
+ let matchLength = query.length;
484
628
  if (start === -1) {
485
- return normalized.slice(0, 160);
629
+ const [firstGram] = buildSearchNgrams([query]).filter((gram) => lower.includes(gram));
630
+ if (!firstGram) {
631
+ return normalized.slice(0, 160);
632
+ }
633
+ start = lower.indexOf(firstGram);
634
+ matchLength = firstGram.length;
486
635
  }
487
636
  const from = Math.max(0, start - 48);
488
- const to = Math.min(normalized.length, start + query.length + 96);
637
+ const to = Math.min(normalized.length, start + matchLength + 96);
489
638
  const prefix = from > 0 ? '...' : '';
490
639
  const suffix = to < normalized.length ? '...' : '';
491
640
  return `${prefix}${normalized.slice(from, to)}${suffix}`;
@@ -507,13 +656,24 @@ function isMatched(fields, query) {
507
656
  const lowerQuery = query.toLowerCase();
508
657
  return fields.some((field) => field.toLowerCase().includes(lowerQuery));
509
658
  }
510
- function createSchema(database) {
659
+ function createSchema(database, capabilities) {
511
660
  database.run(`
512
661
  CREATE TABLE metadata (
513
662
  key TEXT PRIMARY KEY,
514
663
  value TEXT NOT NULL
515
664
  );
516
665
 
666
+ CREATE TABLE indexed_files (
667
+ path TEXT PRIMARY KEY,
668
+ source_scope TEXT NOT NULL,
669
+ size_bytes INTEGER NOT NULL,
670
+ mtime_ms INTEGER NOT NULL,
671
+ content_hash TEXT NOT NULL,
672
+ indexed_at TEXT NOT NULL,
673
+ index_mode TEXT NOT NULL,
674
+ parser_version TEXT NOT NULL
675
+ );
676
+
517
677
  CREATE TABLE documents (
518
678
  path TEXT PRIMARY KEY,
519
679
  type TEXT NOT NULL,
@@ -538,6 +698,16 @@ CREATE TABLE document_terms (
538
698
  PRIMARY KEY (document_path, term, source)
539
699
  );
540
700
 
701
+ CREATE TABLE search_ngrams (
702
+ target_kind TEXT NOT NULL,
703
+ target_key TEXT NOT NULL,
704
+ gram TEXT NOT NULL,
705
+ source TEXT NOT NULL,
706
+ PRIMARY KEY (target_kind, target_key, gram, source)
707
+ );
708
+
709
+ CREATE INDEX search_ngrams_lookup ON search_ngrams(target_kind, gram, target_key);
710
+
541
711
  CREATE TABLE skills (
542
712
  name TEXT PRIMARY KEY,
543
713
  path TEXT NOT NULL,
@@ -575,6 +745,62 @@ CREATE TABLE command_effects (
575
745
  PRIMARY KEY (intent, source, access, mode, path, lock, concurrency)
576
746
  );
577
747
 
748
+ CREATE VIEW command_write_locks AS
749
+ SELECT
750
+ intent,
751
+ lock,
752
+ group_concat(DISTINCT path) AS paths,
753
+ group_concat(DISTINCT mode) AS modes,
754
+ group_concat(DISTINCT source) AS sources,
755
+ group_concat(DISTINCT concurrency) AS concurrencies,
756
+ count(*) AS effect_count
757
+ FROM command_effects
758
+ WHERE access = 'write'
759
+ GROUP BY intent, lock;
760
+
761
+ CREATE VIEW command_lock_conflicts AS
762
+ SELECT
763
+ a.intent AS left_intent,
764
+ b.intent AS right_intent,
765
+ a.lock AS lock,
766
+ group_concat(DISTINCT a.path) AS left_paths,
767
+ group_concat(DISTINCT b.path) AS right_paths,
768
+ group_concat(DISTINCT a.mode) AS left_modes,
769
+ group_concat(DISTINCT b.mode) AS right_modes,
770
+ group_concat(DISTINCT a.concurrency) AS left_concurrencies,
771
+ group_concat(DISTINCT b.concurrency) AS right_concurrencies
772
+ FROM command_effects a
773
+ JOIN command_effects b
774
+ ON a.lock = b.lock
775
+ AND a.intent < b.intent
776
+ WHERE
777
+ a.access = 'write'
778
+ OR b.access = 'write'
779
+ OR a.concurrency = 'exclusive'
780
+ OR b.concurrency = 'exclusive'
781
+ OR a.mode = 'delete_recreate'
782
+ OR b.mode = 'delete_recreate'
783
+ GROUP BY a.intent, b.intent, a.lock;
784
+
785
+ CREATE TABLE path_surfaces (
786
+ rule_id TEXT PRIMARY KEY,
787
+ pattern_kind TEXT NOT NULL,
788
+ pattern TEXT NOT NULL,
789
+ pattern_flags TEXT NOT NULL,
790
+ surface_kind TEXT NOT NULL,
791
+ category TEXT NOT NULL,
792
+ is_public_surface INTEGER NOT NULL,
793
+ update_policy TEXT NOT NULL
794
+ );
795
+
796
+ CREATE TABLE path_surface_reasons (
797
+ rule_id TEXT NOT NULL,
798
+ reason_kind TEXT NOT NULL,
799
+ reason TEXT NOT NULL,
800
+ ordinal INTEGER NOT NULL,
801
+ PRIMARY KEY (rule_id, reason_kind, reason)
802
+ );
803
+
578
804
  CREATE TABLE source_anchors (
579
805
  id TEXT PRIMARY KEY,
580
806
  path TEXT NOT NULL,
@@ -621,6 +847,54 @@ CREATE TABLE source_anchor_status (
621
847
  can_instruct_agent INTEGER NOT NULL
622
848
  );
623
849
  `);
850
+ if (capabilities.backend === SEARCH_BACKEND_FTS5) {
851
+ database.run(`
852
+ CREATE VIRTUAL TABLE search_documents_fts USING fts5(
853
+ path UNINDEXED,
854
+ type UNINDEXED,
855
+ title,
856
+ sections,
857
+ terms,
858
+ snippet
859
+ );
860
+
861
+ CREATE VIRTUAL TABLE search_skills_fts USING fts5(
862
+ name UNINDEXED,
863
+ path UNINDEXED,
864
+ title
865
+ );
866
+
867
+ CREATE VIRTUAL TABLE search_skill_routes_fts USING fts5(
868
+ route_key UNINDEXED,
869
+ skill_name UNINDEXED,
870
+ skill_path UNINDEXED,
871
+ trigger,
872
+ required_input,
873
+ edit_scope,
874
+ risk,
875
+ verification_intents,
876
+ expected_output
877
+ );
878
+
879
+ CREATE VIRTUAL TABLE search_command_intents_fts USING fts5(
880
+ name UNINDEXED,
881
+ status UNINDEXED,
882
+ lifecycle UNINDEXED,
883
+ run_policy UNINDEXED,
884
+ description,
885
+ effects
886
+ );
887
+
888
+ CREATE VIRTUAL TABLE search_source_anchors_fts USING fts5(
889
+ id UNINDEXED,
890
+ path UNINDEXED,
891
+ purpose,
892
+ search_terms,
893
+ invariant,
894
+ risk
895
+ );
896
+ `);
897
+ }
624
898
  }
625
899
  function insertDocumentTerm(database, documentPath, term, source) {
626
900
  const normalized = normalizeSearchText(term ?? '');
@@ -633,8 +907,131 @@ function insertDocumentTerm(database, documentPath, term, source) {
633
907
  source,
634
908
  ]);
635
909
  }
636
- function populateDatabase(database, documents, skills, skillRoutes, commandIntents, sourceAnchors) {
910
+ function insertSearchNgrams(database, targetKind, targetKey, values, source) {
911
+ for (const gram of buildSearchNgrams(values)) {
912
+ database.run('INSERT OR IGNORE INTO search_ngrams (target_kind, target_key, gram, source) VALUES (?, ?, ?, ?)', [targetKind, targetKey, gram, source]);
913
+ }
914
+ }
915
+ function insertPathSurfaceReasons(database, ruleId, reasonKind, values) {
916
+ values.forEach((value, index) => {
917
+ database.run('INSERT INTO path_surface_reasons (rule_id, reason_kind, reason, ordinal) VALUES (?, ?, ?, ?)', [ruleId, reasonKind, value, index + 1]);
918
+ });
919
+ }
920
+ function populatePathSurfaceReadModel(database) {
921
+ for (const rule of listChangeClassificationRuleDescriptors()) {
922
+ database.run('INSERT INTO path_surfaces (rule_id, pattern_kind, pattern, pattern_flags, surface_kind, category, is_public_surface, update_policy) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', [
923
+ rule.id,
924
+ rule.patternKind,
925
+ rule.pattern,
926
+ rule.patternFlags,
927
+ rule.surface.kind,
928
+ rule.surface.category,
929
+ rule.surface.isPublicSurface ? 1 : 0,
930
+ rule.surface.updatePolicy,
931
+ ]);
932
+ insertPathSurfaceReasons(database, rule.id, 'change_kind', rule.changeKinds);
933
+ insertPathSurfaceReasons(database, rule.id, 'validation_reason', rule.surface.validationReasons);
934
+ insertPathSurfaceReasons(database, rule.id, 'affected_contract', rule.surface.affectedContracts);
935
+ insertPathSurfaceReasons(database, rule.id, 'drift_check', rule.surface.driftChecks);
936
+ }
937
+ }
938
+ function skillRouteKey(route) {
939
+ return `${route.skillName}\u0000${route.trigger}`;
940
+ }
941
+ function populateSearchTables(database, capabilities, documents, skills, skillRoutes, commandIntents, sourceAnchors) {
942
+ for (const document of documents) {
943
+ const documentTerms = queryRows(database, 'SELECT term FROM document_terms WHERE document_path = ? ORDER BY term', [
944
+ document.path,
945
+ ]).map((row) => toSearchString(row.term));
946
+ insertSearchNgrams(database, 'document', document.path, [
947
+ document.path,
948
+ document.type,
949
+ document.title,
950
+ document.sections.join(' '),
951
+ documentTerms.join(' '),
952
+ document.contentSnippet,
953
+ ], 'workflow_document');
954
+ if (capabilities.backend === SEARCH_BACKEND_FTS5) {
955
+ database.run('INSERT INTO search_documents_fts (path, type, title, sections, terms, snippet) VALUES (?, ?, ?, ?, ?, ?)', [
956
+ document.path,
957
+ document.type,
958
+ document.title,
959
+ document.sections.join(' '),
960
+ documentTerms.join(' '),
961
+ document.contentSnippet,
962
+ ]);
963
+ }
964
+ }
965
+ for (const skill of skills) {
966
+ insertSearchNgrams(database, 'skill', skill.name, [skill.name, skill.path, skill.title], 'skill');
967
+ if (capabilities.backend === SEARCH_BACKEND_FTS5) {
968
+ database.run('INSERT INTO search_skills_fts (name, path, title) VALUES (?, ?, ?)', [
969
+ skill.name,
970
+ skill.path,
971
+ skill.title,
972
+ ]);
973
+ }
974
+ }
975
+ for (const route of skillRoutes) {
976
+ const verificationIntents = route.verificationIntents.join(' ');
977
+ insertSearchNgrams(database, 'skill_route', skillRouteKey(route), [
978
+ skillRouteKey(route),
979
+ route.skillName,
980
+ route.skillPath,
981
+ route.trigger,
982
+ route.requiredInput,
983
+ route.editScope,
984
+ route.risk,
985
+ verificationIntents,
986
+ route.expectedOutput,
987
+ ], 'skill_route');
988
+ if (capabilities.backend === SEARCH_BACKEND_FTS5) {
989
+ database.run('INSERT INTO search_skill_routes_fts (route_key, skill_name, skill_path, trigger, required_input, edit_scope, risk, verification_intents, expected_output) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', [
990
+ skillRouteKey(route),
991
+ route.skillName,
992
+ route.skillPath,
993
+ route.trigger,
994
+ route.requiredInput,
995
+ route.editScope,
996
+ route.risk,
997
+ verificationIntents,
998
+ route.expectedOutput,
999
+ ]);
1000
+ }
1001
+ }
1002
+ for (const intent of commandIntents) {
1003
+ const effects = intent.effects
1004
+ .flatMap((effect) => [effect.lock, effect.path, effect.mode, effect.access, effect.concurrency])
1005
+ .join(' ');
1006
+ insertSearchNgrams(database, 'command_intent', intent.name, [intent.name, intent.status, intent.lifecycle ?? '', intent.runPolicy ?? '', intent.description ?? '', effects], 'command_intent');
1007
+ if (capabilities.backend === SEARCH_BACKEND_FTS5) {
1008
+ database.run('INSERT INTO search_command_intents_fts (name, status, lifecycle, run_policy, description, effects) VALUES (?, ?, ?, ?, ?, ?)', [intent.name, intent.status, intent.lifecycle, intent.runPolicy, intent.description, effects]);
1009
+ }
1010
+ }
1011
+ for (const anchor of sourceAnchors) {
1012
+ insertSearchNgrams(database, 'source_anchor', anchor.id, [
1013
+ anchor.id,
1014
+ anchor.path,
1015
+ anchor.purpose ?? '',
1016
+ anchor.search.join(' '),
1017
+ anchor.invariant ?? '',
1018
+ anchor.risk.join(' '),
1019
+ ], 'source_anchor');
1020
+ if (capabilities.backend === SEARCH_BACKEND_FTS5) {
1021
+ database.run('INSERT INTO search_source_anchors_fts (id, path, purpose, search_terms, invariant, risk) VALUES (?, ?, ?, ?, ?, ?)', [
1022
+ anchor.id,
1023
+ anchor.path,
1024
+ anchor.purpose,
1025
+ anchor.search.join(' '),
1026
+ anchor.invariant,
1027
+ anchor.risk.join(' '),
1028
+ ]);
1029
+ }
1030
+ }
1031
+ }
1032
+ function populateDatabase(database, capabilities, documents, skills, skillRoutes, commandIntents, sourceAnchors, indexedFiles, indexMode, sourceScopeHash, sourceIndexEnabled, indexedAt) {
637
1033
  database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', ['schema_version', LOCAL_INDEX_SCHEMA_VERSION]);
1034
+ database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', ['parser_version', LOCAL_INDEX_PARSER_VERSION]);
638
1035
  database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', ['content_mode', LOCAL_INDEX_CONTENT_MODE]);
639
1036
  database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', [
640
1037
  'store_full_content',
@@ -644,6 +1041,26 @@ function populateDatabase(database, documents, skills, skillRoutes, commandInten
644
1041
  'max_snippet_bytes_per_document',
645
1042
  String(MAX_SNIPPET_BYTES_PER_DOCUMENT),
646
1043
  ]);
1044
+ database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', ['search_backend', capabilities.backend]);
1045
+ database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', [
1046
+ 'search_fts5_available',
1047
+ String(capabilities.fts5Available),
1048
+ ]);
1049
+ database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', ['source_scope_hash', sourceScopeHash]);
1050
+ database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', ['source_index_enabled', String(sourceIndexEnabled)]);
1051
+ database.run('INSERT INTO metadata (key, value) VALUES (?, ?)', ['index_mode', indexMode]);
1052
+ for (const indexedFile of indexedFiles) {
1053
+ database.run('INSERT INTO indexed_files (path, source_scope, size_bytes, mtime_ms, content_hash, indexed_at, index_mode, parser_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', [
1054
+ indexedFile.path,
1055
+ indexedFile.sourceScope,
1056
+ indexedFile.sizeBytes,
1057
+ indexedFile.mtimeMs,
1058
+ indexedFile.contentHash,
1059
+ indexedAt,
1060
+ indexMode,
1061
+ LOCAL_INDEX_PARSER_VERSION,
1062
+ ]);
1063
+ }
647
1064
  for (const document of documents) {
648
1065
  database.run('INSERT INTO documents (path, type, title, locale, revision, content_hash, content_snippet) VALUES (?, ?, ?, ?, ?, ?, ?)', [
649
1066
  document.path,
@@ -743,29 +1160,110 @@ function populateDatabase(database, documents, skills, skillRoutes, commandInten
743
1160
  anchor.canInstructAgent ? 1 : 0,
744
1161
  ]);
745
1162
  }
1163
+ populatePathSurfaceReadModel(database);
1164
+ populateSearchTables(database, capabilities, documents, skills, skillRoutes, commandIntents, sourceAnchors);
1165
+ }
1166
+ function indexedFilesMatch(database, currentFiles) {
1167
+ const rows = queryRows(database, 'SELECT path, source_scope, content_hash, parser_version FROM indexed_files ORDER BY path');
1168
+ if (rows.length !== currentFiles.length) {
1169
+ return false;
1170
+ }
1171
+ const currentByPath = new Map(currentFiles.map((file) => [file.path, file]));
1172
+ for (const row of rows) {
1173
+ const storedPath = toSearchString(row.path);
1174
+ const current = currentByPath.get(storedPath);
1175
+ if (!current) {
1176
+ return false;
1177
+ }
1178
+ if (toSearchString(row.source_scope) !== current.sourceScope ||
1179
+ toSearchString(row.content_hash) !== current.contentHash ||
1180
+ toSearchString(row.parser_version) !== LOCAL_INDEX_PARSER_VERSION) {
1181
+ return false;
1182
+ }
1183
+ }
1184
+ return true;
1185
+ }
1186
+ async function readIncrementalReuseDecision(SQL, databasePath, currentFiles, sourceScopeHash) {
1187
+ if (!existsSync(databasePath)) {
1188
+ return { reusable: false, rebuildReason: 'missing_index', capabilities: null };
1189
+ }
1190
+ let database;
1191
+ try {
1192
+ database = new SQL.Database(readFileSync(databasePath));
1193
+ if (readStoredSchemaVersion(database) !== LOCAL_INDEX_SCHEMA_VERSION) {
1194
+ return { reusable: false, rebuildReason: 'schema_version_mismatch', capabilities: null };
1195
+ }
1196
+ if (readMetadataValue(database, 'parser_version') !== LOCAL_INDEX_PARSER_VERSION) {
1197
+ return { reusable: false, rebuildReason: 'parser_version_mismatch', capabilities: null };
1198
+ }
1199
+ if (readMetadataValue(database, 'source_scope_hash') !== sourceScopeHash) {
1200
+ return { reusable: false, rebuildReason: 'source_scope_mismatch', capabilities: null };
1201
+ }
1202
+ if (!hasTable(database, 'indexed_files')) {
1203
+ return { reusable: false, rebuildReason: 'indexed_files_missing', capabilities: null };
1204
+ }
1205
+ if (!indexedFilesMatch(database, currentFiles)) {
1206
+ return { reusable: false, rebuildReason: 'file_fingerprint_mismatch', capabilities: null };
1207
+ }
1208
+ return {
1209
+ reusable: true,
1210
+ rebuildReason: null,
1211
+ capabilities: readStoredSearchCapabilities(database),
1212
+ };
1213
+ }
1214
+ catch {
1215
+ return { reusable: false, rebuildReason: 'unreadable_index', capabilities: null };
1216
+ }
1217
+ finally {
1218
+ database?.close();
1219
+ }
746
1220
  }
747
1221
  /**
748
1222
  * mf:anchor cli.index.create
749
1223
  * purpose: Build the local SQLite index for workflow documents and optional source anchors.
750
1224
  * search: mf index, local index, sqlite, source anchors, workflow documents
751
- * invariant: Source anchors are indexed only when source indexing is explicitly requested.
1225
+ * invariant: Source anchors are indexed only when requested by CLI flag or local index configuration.
752
1226
  * risk: cache, config
753
1227
  */
754
1228
  export async function createLocalIndex(projectRoot, options = {}) {
755
1229
  const databasePath = getLocalIndexDatabasePath(projectRoot);
1230
+ const dryRun = options.dryRun === true;
1231
+ const incremental = options.incremental === true;
1232
+ const indexMode = incremental ? 'incremental' : 'full';
756
1233
  const documents = collectDocuments(projectRoot);
757
1234
  const skills = collectSkills(documents);
758
1235
  const skillRoutes = collectSkillRoutes(projectRoot);
759
1236
  const commandIntents = collectCommandIntents(projectRoot);
760
- const includeSource = options.includeSource === true;
761
- const previousSourceAnchors = includeSource ? await readPreviousSourceAnchorSnapshots(databasePath) : [];
762
- const sourceAnchors = includeSource ? collectSourceAnchorIndexRecords(projectRoot, previousSourceAnchors) : [];
763
- const dryRun = options.dryRun === true;
764
- if (!dryRun) {
765
- const SQL = await loadSqlJs();
1237
+ const sourceConfig = readLocalIndexSourceConfig(projectRoot);
1238
+ const includeSource = options.includeSource === true || sourceConfig.enabledByDefault;
1239
+ const sourceScopeHash = getSourceScopeHash(includeSource, sourceConfig);
1240
+ const previousSourceAnchors = includeSource
1241
+ ? await readPreviousSourceAnchorSnapshots(databasePath).catch(() => [])
1242
+ : [];
1243
+ const sourceAnchors = includeSource
1244
+ ? collectSourceAnchorIndexRecords(projectRoot, previousSourceAnchors, {
1245
+ ...sourceConfig,
1246
+ excludeGeneratedOrVendor: true,
1247
+ })
1248
+ : [];
1249
+ const indexedFiles = collectIndexedFileRecords(projectRoot, documents, sourceAnchors);
1250
+ let capabilities = searchCapabilities(false);
1251
+ let reusedExisting = false;
1252
+ let rebuildReason = null;
1253
+ const SQL = await loadSqlJs();
1254
+ const capabilityDatabase = new SQL.Database();
1255
+ capabilities = detectLocalSearchCapabilities(capabilityDatabase);
1256
+ capabilityDatabase.close();
1257
+ if (incremental) {
1258
+ const reuseDecision = await readIncrementalReuseDecision(SQL, databasePath, indexedFiles, sourceScopeHash);
1259
+ reusedExisting = reuseDecision.reusable;
1260
+ rebuildReason = reuseDecision.rebuildReason;
1261
+ capabilities = reuseDecision.capabilities ?? capabilities;
1262
+ }
1263
+ if (!dryRun && !reusedExisting) {
766
1264
  const database = new SQL.Database();
767
- createSchema(database);
768
- populateDatabase(database, documents, skills, skillRoutes, commandIntents, sourceAnchors);
1265
+ createSchema(database, capabilities);
1266
+ populateDatabase(database, capabilities, documents, skills, skillRoutes, commandIntents, sourceAnchors, indexedFiles, indexMode, sourceScopeHash, includeSource, new Date().toISOString());
769
1267
  mkdirSync(path.dirname(databasePath), { recursive: true });
770
1268
  writeFileSync(databasePath, database.export());
771
1269
  database.close();
@@ -777,7 +1275,10 @@ export async function createLocalIndex(projectRoot, options = {}) {
777
1275
  mustflow_root: path.resolve(projectRoot),
778
1276
  database_path: databasePath,
779
1277
  dry_run: dryRun,
780
- wrote_files: !dryRun,
1278
+ wrote_files: !dryRun && !reusedExisting,
1279
+ index_mode: indexMode,
1280
+ reused_existing: reusedExisting,
1281
+ rebuild_reason: rebuildReason,
781
1282
  document_count: documents.length,
782
1283
  skill_count: skills.length,
783
1284
  skill_route_count: skillRoutes.length,
@@ -785,20 +1286,47 @@ export async function createLocalIndex(projectRoot, options = {}) {
785
1286
  command_effect_count: commandIntents.reduce((count, intent) => count + intent.effects.length, 0),
786
1287
  source_index_enabled: includeSource,
787
1288
  source_anchor_count: sourceAnchors.length,
1289
+ search_backend: capabilities.backend,
1290
+ search_fts5_available: capabilities.fts5Available,
788
1291
  content_mode: LOCAL_INDEX_CONTENT_MODE,
789
1292
  store_full_content: LOCAL_INDEX_STORE_FULL_CONTENT,
790
1293
  max_snippet_bytes_per_document: MAX_SNIPPET_BYTES_PER_DOCUMENT,
1294
+ indexed_file_count: indexedFiles.length,
791
1295
  indexed_paths: documents.map((document) => document.path),
792
1296
  };
793
1297
  }
794
1298
  function readStoredSchemaVersion(database) {
795
- return toSearchString(queryRows(database, 'SELECT value FROM metadata WHERE key = "schema_version"')[0]?.value) || undefined;
1299
+ return readMetadataValue(database, 'schema_version');
796
1300
  }
797
1301
  function getStalePaths(projectRoot, database) {
798
1302
  const schemaVersion = readStoredSchemaVersion(database);
799
1303
  if (schemaVersion !== LOCAL_INDEX_SCHEMA_VERSION) {
800
1304
  return ['.mustflow/cache/mustflow.sqlite'];
801
1305
  }
1306
+ if (hasTable(database, 'indexed_files')) {
1307
+ const stalePaths = new Set();
1308
+ const indexedRows = queryRows(database, 'SELECT path, source_scope, content_hash FROM indexed_files');
1309
+ const indexedPaths = new Set(indexedRows.map((row) => toSearchString(row.path)));
1310
+ for (const row of indexedRows) {
1311
+ const indexedPath = toSearchString(row.path);
1312
+ const sourceScope = toSearchString(row.source_scope) === 'source_anchor' ? 'source_anchor' : 'workflow';
1313
+ try {
1314
+ const current = readIndexedFileRecord(projectRoot, indexedPath, sourceScope);
1315
+ if (current.contentHash !== toSearchString(row.content_hash)) {
1316
+ stalePaths.add(indexedPath);
1317
+ }
1318
+ }
1319
+ catch {
1320
+ stalePaths.add(indexedPath);
1321
+ }
1322
+ }
1323
+ for (const document of collectDocuments(projectRoot)) {
1324
+ if (!indexedPaths.has(document.path)) {
1325
+ stalePaths.add(document.path);
1326
+ }
1327
+ }
1328
+ return Array.from(stalePaths).sort((left, right) => left.localeCompare(right));
1329
+ }
802
1330
  const indexedRows = queryRows(database, 'SELECT path, content_hash FROM documents');
803
1331
  const indexedHashes = new Map(indexedRows.map((row) => [toSearchString(row.path), toSearchString(row.content_hash)]));
804
1332
  const currentDocuments = collectDocuments(projectRoot);
@@ -816,6 +1344,258 @@ function getStalePaths(projectRoot, database) {
816
1344
  }
817
1345
  return Array.from(stalePaths).sort((left, right) => left.localeCompare(right));
818
1346
  }
1347
+ function mapCommandLockConflict(row, intent) {
1348
+ const targetIsLeft = toSearchString(row.left_intent) === intent;
1349
+ const targetPrefix = targetIsLeft ? 'left' : 'right';
1350
+ const otherPrefix = targetIsLeft ? 'right' : 'left';
1351
+ return {
1352
+ intent: toSearchString(row[`${otherPrefix}_intent`]),
1353
+ lock: toSearchString(row.lock),
1354
+ paths: splitIndexedList(row[`${targetPrefix}_paths`]),
1355
+ modes: splitIndexedList(row[`${targetPrefix}_modes`]),
1356
+ concurrencies: splitIndexedList(row[`${targetPrefix}_concurrencies`]),
1357
+ conflictingPaths: splitIndexedList(row[`${otherPrefix}_paths`]),
1358
+ conflictingModes: splitIndexedList(row[`${otherPrefix}_modes`]),
1359
+ conflictingConcurrencies: splitIndexedList(row[`${otherPrefix}_concurrencies`]),
1360
+ };
1361
+ }
1362
+ /**
1363
+ * mf:anchor cli.index.command-effect-graph
1364
+ * purpose: Read command-effect lock and conflict explanations from the local SQLite index.
1365
+ * search: mf explain command, command locks, local index, sqlite graph
1366
+ * invariant: Indexed command-effect rows explain current commands.toml only when the index is fresh and never grant command authority.
1367
+ * risk: cache, config
1368
+ */
1369
+ function queryLocalCommandEffectGraph(databasePath, database, intent) {
1370
+ const writeLocks = queryRows(database, `
1371
+ SELECT lock, paths, modes, sources, concurrencies, effect_count
1372
+ FROM command_write_locks
1373
+ WHERE intent = ?
1374
+ ORDER BY lock
1375
+ `, [intent]).map((row) => ({
1376
+ lock: toSearchString(row.lock),
1377
+ paths: splitIndexedList(row.paths),
1378
+ modes: splitIndexedList(row.modes),
1379
+ sources: splitIndexedList(row.sources),
1380
+ concurrencies: splitIndexedList(row.concurrencies),
1381
+ effectCount: typeof row.effect_count === 'number' && Number.isFinite(row.effect_count) ? row.effect_count : 0,
1382
+ }));
1383
+ const lockConflicts = queryRows(database, `
1384
+ SELECT
1385
+ left_intent,
1386
+ right_intent,
1387
+ lock,
1388
+ left_paths,
1389
+ right_paths,
1390
+ left_modes,
1391
+ right_modes,
1392
+ left_concurrencies,
1393
+ right_concurrencies
1394
+ FROM command_lock_conflicts
1395
+ WHERE left_intent = ? OR right_intent = ?
1396
+ ORDER BY lock, left_intent, right_intent
1397
+ `, [intent, intent]).map((row) => mapCommandLockConflict(row, intent));
1398
+ return {
1399
+ source: 'local_index',
1400
+ status: 'fresh',
1401
+ databasePath,
1402
+ indexFresh: true,
1403
+ stalePaths: [],
1404
+ writeLocks,
1405
+ lockConflicts,
1406
+ refreshHint: null,
1407
+ };
1408
+ }
1409
+ export async function readLocalCommandEffectGraph(projectRoot, intent) {
1410
+ const graphs = await readLocalCommandEffectGraphs(projectRoot, [intent]);
1411
+ return graphs.get(intent) ?? createCommandEffectGraphStatus(getLocalIndexDatabasePath(projectRoot), 'unreadable');
1412
+ }
1413
+ export async function readLocalCommandEffectGraphs(projectRoot, intents) {
1414
+ const databasePath = getLocalIndexDatabasePath(projectRoot);
1415
+ const intentNames = [...new Set(intents)];
1416
+ const statusMap = (status, stalePaths = []) => new Map(intentNames.map((intent) => [intent, createCommandEffectGraphStatus(databasePath, status, stalePaths)]));
1417
+ if (!existsSync(databasePath)) {
1418
+ return statusMap('missing');
1419
+ }
1420
+ const SQL = await loadSqlJs();
1421
+ const database = new SQL.Database(readFileSync(databasePath));
1422
+ try {
1423
+ const stalePaths = getStalePaths(projectRoot, database);
1424
+ if (stalePaths.length > 0) {
1425
+ return statusMap('stale', stalePaths);
1426
+ }
1427
+ return new Map(intentNames.map((intent) => [intent, queryLocalCommandEffectGraph(databasePath, database, intent)]));
1428
+ }
1429
+ catch {
1430
+ return statusMap('unreadable');
1431
+ }
1432
+ finally {
1433
+ database.close();
1434
+ }
1435
+ }
1436
+ function createPathSurfaceReadModelStatus(databasePath, status, inputPath, stalePaths = []) {
1437
+ return {
1438
+ source: 'local_index',
1439
+ status,
1440
+ databasePath,
1441
+ indexFresh: status === 'fresh',
1442
+ stalePaths,
1443
+ inputPath,
1444
+ match: null,
1445
+ refreshHint: status === 'fresh' ? null : 'Run `mf index` to refresh path-surface explanations.',
1446
+ };
1447
+ }
1448
+ function createLocalIndexPromptContextStatus(databasePath, status, stalePaths = [], capabilities = null) {
1449
+ return {
1450
+ source: 'local_index',
1451
+ status,
1452
+ databasePath,
1453
+ indexFresh: status === 'fresh',
1454
+ stalePaths,
1455
+ searchBackend: capabilities?.backend ?? null,
1456
+ searchFts5Available: capabilities?.fts5Available ?? null,
1457
+ refreshHint: status === 'fresh' ? null : 'Run `mf index` to refresh prompt-cache task context local-index metadata.',
1458
+ };
1459
+ }
1460
+ export async function readLocalIndexPromptContext(projectRoot) {
1461
+ const databasePath = getLocalIndexDatabasePath(projectRoot);
1462
+ if (!existsSync(databasePath)) {
1463
+ return createLocalIndexPromptContextStatus(databasePath, 'missing');
1464
+ }
1465
+ let database;
1466
+ try {
1467
+ const SQL = await loadSqlJs();
1468
+ database = new SQL.Database(readFileSync(databasePath));
1469
+ const capabilities = readStoredSearchCapabilities(database);
1470
+ const stalePaths = getStalePaths(projectRoot, database);
1471
+ if (stalePaths.length > 0) {
1472
+ return createLocalIndexPromptContextStatus(databasePath, 'stale', stalePaths, capabilities);
1473
+ }
1474
+ return createLocalIndexPromptContextStatus(databasePath, 'fresh', [], capabilities);
1475
+ }
1476
+ catch {
1477
+ return createLocalIndexPromptContextStatus(databasePath, 'unreadable');
1478
+ }
1479
+ finally {
1480
+ database?.close();
1481
+ }
1482
+ }
1483
+ function pathSurfaceReadModelWithMatch(base, match) {
1484
+ return {
1485
+ ...base,
1486
+ match,
1487
+ };
1488
+ }
1489
+ function readPathSurfaceReasonMap(database) {
1490
+ const byRule = new Map();
1491
+ for (const row of queryRows(database, 'SELECT rule_id, reason_kind, reason FROM path_surface_reasons ORDER BY rule_id, reason_kind, ordinal')) {
1492
+ const ruleId = toSearchString(row.rule_id);
1493
+ const reasonKind = toSearchString(row.reason_kind);
1494
+ const reason = toSearchString(row.reason);
1495
+ let reasonsByKind = byRule.get(ruleId);
1496
+ if (!reasonsByKind) {
1497
+ reasonsByKind = new Map();
1498
+ byRule.set(ruleId, reasonsByKind);
1499
+ }
1500
+ const reasons = reasonsByKind.get(reasonKind) ?? [];
1501
+ reasons.push(reason);
1502
+ reasonsByKind.set(reasonKind, reasons);
1503
+ }
1504
+ return byRule;
1505
+ }
1506
+ function readPathSurfaceRuleMatches(database) {
1507
+ const reasons = readPathSurfaceReasonMap(database);
1508
+ return queryRows(database, 'SELECT rule_id, pattern_kind, pattern, pattern_flags, surface_kind, category, is_public_surface, update_policy FROM path_surfaces ORDER BY rowid').map((row) => {
1509
+ const ruleId = toSearchString(row.rule_id);
1510
+ const reasonsByKind = reasons.get(ruleId);
1511
+ const reasonList = (kind) => reasonsByKind?.get(kind) ?? [];
1512
+ return {
1513
+ ruleId,
1514
+ patternKind: toSearchString(row.pattern_kind),
1515
+ pattern: toSearchString(row.pattern),
1516
+ patternFlags: toSearchString(row.pattern_flags),
1517
+ changeKinds: reasonList('change_kind'),
1518
+ surface: {
1519
+ kind: toSearchString(row.surface_kind),
1520
+ category: toSearchString(row.category),
1521
+ isPublicSurface: Number(row.is_public_surface) === 1,
1522
+ validationReasons: reasonList('validation_reason'),
1523
+ affectedContracts: reasonList('affected_contract'),
1524
+ updatePolicy: toSearchString(row.update_policy),
1525
+ driftChecks: reasonList('drift_check'),
1526
+ },
1527
+ };
1528
+ });
1529
+ }
1530
+ function matchPathSurfaceRule(relativePath, rules) {
1531
+ if (!relativePath) {
1532
+ return null;
1533
+ }
1534
+ for (const rule of rules) {
1535
+ try {
1536
+ if (new RegExp(rule.pattern, rule.patternFlags).test(relativePath)) {
1537
+ return rule;
1538
+ }
1539
+ }
1540
+ catch {
1541
+ continue;
1542
+ }
1543
+ }
1544
+ return null;
1545
+ }
1546
+ export async function readLocalPathSurfaces(projectRoot, relativePaths) {
1547
+ const databasePath = getLocalIndexDatabasePath(projectRoot);
1548
+ const normalizedPaths = [...new Set(relativePaths.map((relativePath) => toPosixPath(relativePath)).filter(Boolean))];
1549
+ const statusMap = (status, stalePaths = []) => new Map(normalizedPaths.map((relativePath) => [
1550
+ relativePath,
1551
+ createPathSurfaceReadModelStatus(databasePath, status, relativePath, stalePaths),
1552
+ ]));
1553
+ if (!existsSync(databasePath)) {
1554
+ return statusMap('missing');
1555
+ }
1556
+ const SQL = await loadSqlJs();
1557
+ const database = new SQL.Database(readFileSync(databasePath));
1558
+ try {
1559
+ const stalePaths = getStalePaths(projectRoot, database);
1560
+ if (stalePaths.length > 0) {
1561
+ return statusMap('stale', stalePaths);
1562
+ }
1563
+ const rules = readPathSurfaceRuleMatches(database);
1564
+ return new Map(normalizedPaths.map((relativePath) => {
1565
+ const base = createPathSurfaceReadModelStatus(databasePath, 'fresh', relativePath);
1566
+ return [relativePath, pathSurfaceReadModelWithMatch(base, matchPathSurfaceRule(relativePath, rules))];
1567
+ }));
1568
+ }
1569
+ catch {
1570
+ return statusMap('unreadable');
1571
+ }
1572
+ finally {
1573
+ database.close();
1574
+ }
1575
+ }
1576
+ export async function readLocalPathSurface(projectRoot, relativePath) {
1577
+ const databasePath = getLocalIndexDatabasePath(projectRoot);
1578
+ const inputPath = relativePath ? toPosixPath(relativePath) : null;
1579
+ if (!inputPath) {
1580
+ if (!existsSync(databasePath)) {
1581
+ return createPathSurfaceReadModelStatus(databasePath, 'missing', null);
1582
+ }
1583
+ const SQL = await loadSqlJs();
1584
+ const database = new SQL.Database(readFileSync(databasePath));
1585
+ try {
1586
+ const stalePaths = getStalePaths(projectRoot, database);
1587
+ return createPathSurfaceReadModelStatus(databasePath, stalePaths.length > 0 ? 'stale' : 'fresh', null, stalePaths);
1588
+ }
1589
+ catch {
1590
+ return createPathSurfaceReadModelStatus(databasePath, 'unreadable', null);
1591
+ }
1592
+ finally {
1593
+ database.close();
1594
+ }
1595
+ }
1596
+ const surfaces = await readLocalPathSurfaces(projectRoot, [inputPath]);
1597
+ return surfaces.get(inputPath) ?? createPathSurfaceReadModelStatus(databasePath, 'unreadable', inputPath);
1598
+ }
819
1599
  function getSectionHeadings(database, documentPath) {
820
1600
  return queryRows(database, 'SELECT heading FROM sections WHERE document_path = ? ORDER BY ordinal', [documentPath]).map((row) => toSearchString(row.heading));
821
1601
  }
@@ -833,6 +1613,94 @@ function getCommandEffects(database, intent) {
833
1613
  concurrency: toSearchString(row.concurrency),
834
1614
  }));
835
1615
  }
1616
+ const EMPTY_INDEXED_SEARCH_MATCHES = {
1617
+ active: false,
1618
+ documents: new Set(),
1619
+ skills: new Set(),
1620
+ skillRoutes: new Set(),
1621
+ commandIntents: new Set(),
1622
+ sourceAnchors: new Set(),
1623
+ };
1624
+ function buildFtsQuery(query) {
1625
+ const tokens = extractSearchTokens(query);
1626
+ if (tokens.length === 0) {
1627
+ return null;
1628
+ }
1629
+ return [...new Set(tokens)].map((token) => `"${token.replaceAll('"', '""')}"`).join(' AND ');
1630
+ }
1631
+ function queryFtsSet(database, sql, ftsQuery, column) {
1632
+ return new Set(queryRows(database, sql, [ftsQuery]).map((row) => toSearchString(row[column])));
1633
+ }
1634
+ function mergeSearchSets(left, right) {
1635
+ return new Set([...left, ...right]);
1636
+ }
1637
+ function mergeIndexedSearchMatches(left, right) {
1638
+ return {
1639
+ active: left.active || right.active,
1640
+ documents: mergeSearchSets(left.documents, right.documents),
1641
+ skills: mergeSearchSets(left.skills, right.skills),
1642
+ skillRoutes: mergeSearchSets(left.skillRoutes, right.skillRoutes),
1643
+ commandIntents: mergeSearchSets(left.commandIntents, right.commandIntents),
1644
+ sourceAnchors: mergeSearchSets(left.sourceAnchors, right.sourceAnchors),
1645
+ };
1646
+ }
1647
+ function queryNgramSet(database, targetKind, grams) {
1648
+ const placeholders = grams.map(() => '?').join(', ');
1649
+ if (!placeholders) {
1650
+ return new Set();
1651
+ }
1652
+ return new Set(queryRows(database, `SELECT target_key
1653
+ FROM search_ngrams
1654
+ WHERE target_kind = ? AND gram IN (${placeholders})
1655
+ GROUP BY target_key
1656
+ HAVING COUNT(DISTINCT gram) = ?`, [targetKind, ...grams, grams.length]).map((row) => toSearchString(row.target_key)));
1657
+ }
1658
+ function getNgramSearchMatches(database, query) {
1659
+ if (!hasTable(database, 'search_ngrams')) {
1660
+ return EMPTY_INDEXED_SEARCH_MATCHES;
1661
+ }
1662
+ const grams = buildSearchNgrams([query]);
1663
+ if (grams.length === 0) {
1664
+ return EMPTY_INDEXED_SEARCH_MATCHES;
1665
+ }
1666
+ return {
1667
+ active: true,
1668
+ documents: queryNgramSet(database, 'document', grams),
1669
+ skills: queryNgramSet(database, 'skill', grams),
1670
+ skillRoutes: queryNgramSet(database, 'skill_route', grams),
1671
+ commandIntents: queryNgramSet(database, 'command_intent', grams),
1672
+ sourceAnchors: queryNgramSet(database, 'source_anchor', grams),
1673
+ };
1674
+ }
1675
+ function getIndexedSearchMatches(database, query) {
1676
+ const capabilities = readStoredSearchCapabilities(database);
1677
+ const ftsQuery = capabilities.backend === SEARCH_BACKEND_FTS5 ? buildFtsQuery(query) : null;
1678
+ const ngramMatches = getNgramSearchMatches(database, query);
1679
+ if (!ftsQuery) {
1680
+ return ngramMatches;
1681
+ }
1682
+ try {
1683
+ const ftsMatches = {
1684
+ active: true,
1685
+ documents: queryFtsSet(database, 'SELECT path FROM search_documents_fts WHERE search_documents_fts MATCH ?', ftsQuery, 'path'),
1686
+ skills: queryFtsSet(database, 'SELECT name FROM search_skills_fts WHERE search_skills_fts MATCH ?', ftsQuery, 'name'),
1687
+ skillRoutes: queryFtsSet(database, 'SELECT route_key FROM search_skill_routes_fts WHERE search_skill_routes_fts MATCH ?', ftsQuery, 'route_key'),
1688
+ commandIntents: queryFtsSet(database, 'SELECT name FROM search_command_intents_fts WHERE search_command_intents_fts MATCH ?', ftsQuery, 'name'),
1689
+ sourceAnchors: queryFtsSet(database, 'SELECT id FROM search_source_anchors_fts WHERE search_source_anchors_fts MATCH ?', ftsQuery, 'id'),
1690
+ };
1691
+ return mergeIndexedSearchMatches(ftsMatches, ngramMatches);
1692
+ }
1693
+ catch {
1694
+ return ngramMatches;
1695
+ }
1696
+ }
1697
+ function matchesIndexedOrTableScan(fields, query, indexedMatches, matchSet, key) {
1698
+ return (indexedMatches.active && matchSet.has(key)) || isMatched(fields, query);
1699
+ }
1700
+ function scoreIndexedOrTableScan(primaryFields, secondaryFields, query, indexedMatches, matchSet, key) {
1701
+ const tableScore = scoreMatch(primaryFields, secondaryFields, query);
1702
+ return indexedMatches.active && matchSet.has(key) ? Math.max(tableScore, 20) : tableScore;
1703
+ }
836
1704
  /**
837
1705
  * mf:anchor cli.search.local-index
838
1706
  * purpose: Search the local index while preserving workflow authority above source navigation hints.
@@ -854,9 +1722,12 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
854
1722
  const SQL = await loadSqlJs();
855
1723
  const database = new SQL.Database(readFileSync(databasePath));
856
1724
  const cacheLayers = readCacheLayerSets(projectRoot);
1725
+ let capabilities = searchCapabilities(false);
857
1726
  const results = [];
858
1727
  try {
859
1728
  const stalePaths = getStalePaths(projectRoot, database);
1729
+ capabilities = readStoredSearchCapabilities(database);
1730
+ const indexedMatches = getIndexedSearchMatches(database, normalizedQuery);
860
1731
  if (stalePaths.length > 0) {
861
1732
  throw new Error(`Local mustflow index is stale: ${stalePaths.join(', ')}. Run \`mf index\` before searching. Refresh command: mf index`);
862
1733
  }
@@ -870,7 +1741,8 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
870
1741
  const documentTerms = getDocumentTerms(database, pathValue);
871
1742
  const primaryFields = [pathValue, title];
872
1743
  const secondaryFields = [typeValue, contentSnippet, ...sectionHeadings, ...documentTerms];
873
- if (!isMatched([...primaryFields, ...secondaryFields], normalizedQuery)) {
1744
+ const fields = [...primaryFields, ...secondaryFields];
1745
+ if (!matchesIndexedOrTableScan(fields, normalizedQuery, indexedMatches, indexedMatches.documents, pathValue)) {
874
1746
  continue;
875
1747
  }
876
1748
  results.push(withCacheHint({
@@ -879,8 +1751,8 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
879
1751
  title,
880
1752
  document_type: typeValue,
881
1753
  ...workflowAuthorityForDocument(typeValue),
882
- match: getMatchSnippet([...primaryFields, ...secondaryFields], normalizedQuery),
883
- score: scoreMatch(primaryFields, secondaryFields, normalizedQuery),
1754
+ match: getMatchSnippet(fields, normalizedQuery),
1755
+ score: scoreIndexedOrTableScan(primaryFields, secondaryFields, normalizedQuery, indexedMatches, indexedMatches.documents, pathValue),
884
1756
  }, cacheLayers));
885
1757
  }
886
1758
  for (const row of queryRows(database, 'SELECT name, path, title FROM skills')) {
@@ -888,7 +1760,7 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
888
1760
  const pathValue = toSearchString(row.path);
889
1761
  const title = toSearchString(row.title);
890
1762
  const fields = [name, pathValue, title];
891
- if (!isMatched(fields, normalizedQuery)) {
1763
+ if (!matchesIndexedOrTableScan(fields, normalizedQuery, indexedMatches, indexedMatches.skills, name)) {
892
1764
  continue;
893
1765
  }
894
1766
  results.push(withCacheHint({
@@ -898,7 +1770,7 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
898
1770
  title,
899
1771
  ...skillAuthority(),
900
1772
  match: getMatchSnippet(fields, normalizedQuery),
901
- score: scoreMatch([name, pathValue, title], [], normalizedQuery),
1773
+ score: scoreIndexedOrTableScan([name, pathValue, title], [], normalizedQuery, indexedMatches, indexedMatches.skills, name),
902
1774
  }, cacheLayers));
903
1775
  }
904
1776
  for (const row of queryRows(database, 'SELECT skill_name, skill_path, trigger, required_input, edit_scope, risk, verification_intents, expected_output FROM skill_routes')) {
@@ -912,7 +1784,9 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
912
1784
  const expectedOutput = toSearchString(row.expected_output);
913
1785
  const primaryFields = [name, trigger];
914
1786
  const secondaryFields = [pathValue, requiredInput, editScope, risk, expectedOutput];
915
- if (!isMatched([...primaryFields, ...secondaryFields], normalizedQuery)) {
1787
+ const fields = [...primaryFields, ...secondaryFields];
1788
+ const routeKey = skillRouteKey({ skillName: name, trigger });
1789
+ if (!matchesIndexedOrTableScan(fields, normalizedQuery, indexedMatches, indexedMatches.skillRoutes, routeKey)) {
916
1790
  continue;
917
1791
  }
918
1792
  results.push(withCacheHint({
@@ -924,8 +1798,8 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
924
1798
  route_risk: risk,
925
1799
  verification_intents: verificationIntents,
926
1800
  ...skillAuthority(),
927
- match: getMatchSnippet([...primaryFields, ...secondaryFields], normalizedQuery),
928
- score: scoreMatch(primaryFields, secondaryFields, normalizedQuery),
1801
+ match: getMatchSnippet(fields, normalizedQuery),
1802
+ score: scoreIndexedOrTableScan(primaryFields, secondaryFields, normalizedQuery, indexedMatches, indexedMatches.skillRoutes, routeKey),
929
1803
  }, cacheLayers));
930
1804
  }
931
1805
  for (const row of queryRows(database, 'SELECT name, status, lifecycle, run_policy, description FROM command_intents')) {
@@ -940,7 +1814,8 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
940
1814
  const effectModes = [...new Set(effects.map((effect) => effect.mode))].sort((left, right) => left.localeCompare(right));
941
1815
  const primaryFields = [name];
942
1816
  const secondaryFields = [status, lifecycle, runPolicy, description, ...effectLocks, ...effectPaths, ...effectModes];
943
- if (!isMatched([...primaryFields, ...secondaryFields], normalizedQuery)) {
1817
+ const fields = [...primaryFields, ...secondaryFields];
1818
+ if (!matchesIndexedOrTableScan(fields, normalizedQuery, indexedMatches, indexedMatches.commandIntents, name)) {
944
1819
  continue;
945
1820
  }
946
1821
  results.push(withCacheHint({
@@ -951,8 +1826,8 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
951
1826
  effect_paths: effectPaths,
952
1827
  effect_modes: effectModes,
953
1828
  ...commandIntentAuthority(),
954
- match: getMatchSnippet([...primaryFields, ...secondaryFields], normalizedQuery),
955
- score: scoreMatch(primaryFields, secondaryFields, normalizedQuery),
1829
+ match: getMatchSnippet(fields, normalizedQuery),
1830
+ score: scoreIndexedOrTableScan(primaryFields, secondaryFields, normalizedQuery, indexedMatches, indexedMatches.commandIntents, name),
956
1831
  }, cacheLayers));
957
1832
  }
958
1833
  }
@@ -966,7 +1841,8 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
966
1841
  const risk = toSearchString(row.risk);
967
1842
  const primaryFields = [id, pathValue];
968
1843
  const secondaryFields = [purpose, searchTerms, invariant, risk];
969
- if (!isMatched([...primaryFields, ...secondaryFields], normalizedQuery)) {
1844
+ const fields = [...primaryFields, ...secondaryFields];
1845
+ if (!matchesIndexedOrTableScan(fields, normalizedQuery, indexedMatches, indexedMatches.sourceAnchors, id)) {
970
1846
  continue;
971
1847
  }
972
1848
  results.push(withCacheHint({
@@ -980,8 +1856,8 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
980
1856
  ...sourceAnchorAuthority(),
981
1857
  stale_status: toSearchString(row.status),
982
1858
  stale_confidence: Number(row.confidence),
983
- match: getMatchSnippet([...primaryFields, ...secondaryFields], normalizedQuery),
984
- score: scoreMatch(primaryFields, secondaryFields, normalizedQuery),
1859
+ match: getMatchSnippet(fields, normalizedQuery),
1860
+ score: scoreIndexedOrTableScan(primaryFields, secondaryFields, normalizedQuery, indexedMatches, indexedMatches.sourceAnchors, id),
985
1861
  }, cacheLayers));
986
1862
  }
987
1863
  }
@@ -1008,6 +1884,8 @@ export async function searchLocalIndex(projectRoot, query, options = {}) {
1008
1884
  scope,
1009
1885
  index_fresh: true,
1010
1886
  stale_paths: [],
1887
+ search_backend: capabilities.backend,
1888
+ search_fts5_available: capabilities.fts5Available,
1011
1889
  result_count: sortedResults.length,
1012
1890
  results: sortedResults,
1013
1891
  };