gitnexus 1.6.12 → 1.6.13-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +28 -31
  2. package/dist/cli/analyze.js +7 -24
  3. package/dist/cli/doctor.js +14 -9
  4. package/dist/core/lbug/extension-load-error.d.ts +8 -2
  5. package/dist/core/lbug/extension-load-error.js +23 -16
  6. package/dist/core/lbug/extension-loader.d.ts +21 -6
  7. package/dist/core/lbug/extension-loader.js +72 -17
  8. package/dist/core/lbug/lbug-adapter.d.ts +6 -25
  9. package/dist/core/lbug/lbug-adapter.js +36 -47
  10. package/dist/core/lbug/native-check.d.ts +15 -1
  11. package/dist/core/lbug/native-check.js +45 -22
  12. package/dist/core/lbug/pool-adapter.js +8 -1
  13. package/dist/core/lbug/sidecar-recovery.d.ts +32 -1
  14. package/dist/core/lbug/sidecar-recovery.js +63 -1
  15. package/dist/core/lbug/vendored-extension-path.d.ts +26 -0
  16. package/dist/core/lbug/vendored-extension-path.js +98 -0
  17. package/dist/core/lbug/wal-checkpoint-driver.d.ts +5 -2
  18. package/dist/core/lbug/wal-checkpoint-driver.js +7 -3
  19. package/dist/core/run-analyze.d.ts +4 -2
  20. package/dist/core/run-analyze.js +205 -68
  21. package/dist/core/search/fts-crash-marker.d.ts +59 -0
  22. package/dist/core/search/fts-crash-marker.js +55 -0
  23. package/dist/core/search/fts-indexes.js +2 -2
  24. package/dist/core/search/fts-policy.d.ts +11 -1
  25. package/dist/core/search/fts-policy.js +47 -0
  26. package/dist/core/tree-sitter/vendored-grammars.d.ts +2 -10
  27. package/dist/core/tree-sitter/vendored-grammars.js +2 -11
  28. package/dist/core/vendor-root.d.ts +8 -0
  29. package/dist/core/vendor-root.js +10 -0
  30. package/dist/mcp/server.js +10 -3
  31. package/dist/server/api.js +21 -14
  32. package/dist/storage/repo-meta.d.ts +31 -2
  33. package/package.json +2 -2
  34. package/scripts/assert-publish-fts-coverage.cjs +246 -0
  35. package/scripts/cross-platform-shard.ts +7 -2
  36. package/scripts/cross-platform-tests.ts +15 -0
  37. package/scripts/ensure-fts.ts +13 -11
  38. package/vendor/lbug-fts/LICENSE +27 -0
  39. package/vendor/lbug-fts/manifest.json +21 -0
  40. package/vendor/lbug-fts/prebuilds/SHA256SUMS +5 -0
  41. package/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension +0 -0
  42. package/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension +0 -0
  43. package/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension +0 -0
  44. package/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension +0 -0
  45. package/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension +0 -0
  46. package/web/assets/{agent-DeT_Hy8W.js → agent-DiLVv9Xg.js} +8 -8
  47. package/web/assets/{index-K4KOdfVv.js → index-coee0RPm.js} +3 -3
  48. package/web/index.html +1 -1
@@ -10,6 +10,7 @@
10
10
  */
11
11
  import { detectGraphWriteCollapse } from './index-freshness.js';
12
12
  import { resolveFtsDisableReason, getFtsDisabledReason, withExplicitFtsDisablement, FTS_DISABLED_MESSAGE, DEFAULT_GRAPH_CAPABILITY, DEFAULT_VECTOR_SEARCH_CAPABILITY, } from './search/fts-policy.js';
13
+ import { allowsFtsCrashWalPark, buildFtsDirtyStamp, inferNativeAbortSkip, isBoundaryCheckpointFatal, isFtsStagingDirty, resolveFtsWritePlan, shouldRefuseFtsCrashWal, shouldRefuseRepairFtsWhileDirty, shouldStampFtsDirtyPhase, } from './search/fts-crash-marker.js';
13
14
  import { PDG_EDGE_TYPES } from './lbug/pdg-emit-sink.js';
14
15
  import path from 'path';
15
16
  import fs from 'fs/promises';
@@ -33,9 +34,10 @@ import { getFtsIndexes } from './search/fts-schema.js';
33
34
  import { applyContentRetention, contentRetentionFromEnvironment, contentRetentionMismatch, ftsProfileForContentRetention, } from './content-retention.js';
34
35
  import { cjkSegmentationModeMismatch, getSearchFTSCjkSegmentation, initialiseSearchFTSCjkSegmentation, } from './search/cjk-segmentation.js';
35
36
  import { getExtensionCapability, getExtensionCapabilities, getFtsCapability, resolveAnalyzeInstallPolicy, } from './lbug/extension-loader.js';
36
- import { diagnoseExtensionLoad } from './lbug/extension-load-error.js';
37
+ import { diagnoseExtensionLoad, extractExtensionPath, usesClassifiedLoadRemedy, } from './lbug/extension-load-error.js';
38
+ import { resolveFtsVersionPair } from './lbug/vendored-extension-path.js';
37
39
  import { startWalCheckpointDriver, checkpointOnce, } from './lbug/wal-checkpoint-driver.js';
38
- import { quarantineSidecarsForDirtyRecovery, inspectLbugSidecars, } from './lbug/sidecar-recovery.js';
40
+ import { ftsCrashParkFailureMessage, quarantineSidecarsForDirtyRecovery, inspectLbugSidecars, } from './lbug/sidecar-recovery.js';
39
41
  import { getStoragePaths, resolveBranchPlacement, saveMeta, loadMeta, ensureGitNexusIgnored, registerRepo, adoptFlatBranchLabel, isReadOnlyFilesystemError, isRepoRegistered, cleanupOldKuzuFiles, reconcileMetadataFiles, ensureStoragePathWritable, isMissingFilesystemError, INDEX_METADATA_FILE, CONTENT_RETENTION_SCHEMA_VERSION, } from '../storage/repo-manager.js';
40
42
  import { ANALYZE_FORCE_STORAGE_REQUIREMENTS, ANALYZE_STORAGE_REQUIREMENTS, requireStoragePath, } from '../storage/storage-resolver.js';
41
43
  import { DEFAULT_PDG_MAX_FUNCTION_LINES } from './ingestion/cfg/collect.js';
@@ -713,6 +715,10 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
713
715
  const existingMeta = loadedMeta
714
716
  ? withExplicitFtsDisablement(loadedMeta, ftsDisabledReason)
715
717
  : undefined;
718
+ // KTD6: the dying process writes nothing. Infer skip from the FTS-phase
719
+ // dirty flag (or a persisted native-abort skipReason) BEFORE later
720
+ // saveMeta calls overwrite the on-disk phase.
721
+ const priorFtsNativeAbort = inferNativeAbortSkip(existingMeta?.incrementalInProgress, existingMeta?.capabilities?.fts?.skipReason);
716
722
  // Claim a fresh, ownership-validated slot before the pipeline writes caches.
717
723
  // A later registry-name collision or pipeline failure can otherwise leave
718
724
  // cache files without metadata, which must be treated as unowned on the next
@@ -727,6 +733,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
727
733
  });
728
734
  }
729
735
  // ── FTS-only repair path ────────────────────────────────────────────
736
+ const requestedRepairFts = Boolean(options.repairFts);
730
737
  if (options.repairFts &&
731
738
  existingMeta &&
732
739
  contentRetentionMismatch(existingMeta, contentRetention)) {
@@ -738,13 +745,17 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
738
745
  throw new Error('Cannot repair FTS indexes because this repository has not been analyzed yet. ' +
739
746
  'Run `gitnexus analyze` first to create the initial index, then retry `--repair-fts`.');
740
747
  }
741
- if (existingMeta.incrementalInProgress) {
742
- // #2409 / tri-review 4669518496 (R6): a dirty flag means the previous
743
- // run died mid-writeback — the graph may be half-written and its WAL
744
- // possibly poisoned. This branch returns early, so the dirty-recovery
745
- // sidecar quarantine below would never run: repairing FTS now would
746
- // open the DB and replay that WAL pre-quarantine, and even a
747
- // survivable open would certify FTS over a half-written graph.
748
+ if (shouldRefuseRepairFtsWhileDirty(existingMeta.incrementalInProgress)) {
749
+ // #2409 / tri-review 4669518496 (R6): a non-FTS dirty flag means the
750
+ // previous run died mid-writeback — the graph may be half-written and
751
+ // its WAL possibly poisoned. This branch returns early, so the
752
+ // dirty-recovery sidecar quarantine below would never run: repairing
753
+ // FTS now would open the DB and replay that WAL pre-quarantine, and
754
+ // even a survivable open would certify FTS over a half-written graph.
755
+ // An FTS-phase flag with a successful checkpoint is different (KTD4):
756
+ // the graph-boundary checkpoint already ran, so `--repair-fts` must
757
+ // stay usable (R8). Missing/failed checkpoint is treated like a
758
+ // half-written graph.
748
759
  throw new Error('Cannot repair FTS indexes: the index is mid-incremental-recovery ' +
749
760
  '(a previous analyze run did not complete cleanly). ' +
750
761
  'Run `gitnexus analyze` first — it recovers the index automatically — ' +
@@ -776,6 +787,18 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
776
787
  'Run `gitnexus analyze` (full) to rebuild from scratch.');
777
788
  }
778
789
  await ensureWritableStorage();
790
+ // P1 R8: park a poisoned live WAL before opening. Staging never parks —
791
+ // the live index next to an unpublished staging file must replay its WAL.
792
+ const repairDirty = existingMeta.incrementalInProgress;
793
+ if (shouldRefuseFtsCrashWal(repairDirty, existingMeta.capabilities?.fts)) {
794
+ const { moved: repairParked, removed: repairRemoved, failed: repairParkFailed, } = await quarantineSidecarsForDirtyRecovery(lbugPath, (message) => log(` ${message}`));
795
+ if (repairParkFailed.length > 0) {
796
+ throw new Error(ftsCrashParkFailureMessage(repairParkFailed[0]));
797
+ }
798
+ if (repairParked.length + repairRemoved.length > 0) {
799
+ log('Parked leftover WAL/shadow from the previous in-place FTS abort before --repair-fts.');
800
+ }
801
+ }
779
802
  try {
780
803
  await initAnalysisLbug(lbugPath);
781
804
  // Gate on FTS availability BEFORE touching any index. createSearchFTSIndexes
@@ -803,8 +826,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
803
826
  // by re-installing — the file is already present. Route that class to the
804
827
  // classified remedy (install VC++ redist / OpenSSL) instead of the old
805
828
  // "retry the network install" text that trapped the user in a loop.
806
- const { kind, remedy } = diagnoseExtensionLoad(rawFtsReason);
807
- const remedyTail = kind === 'missing_dependency'
829
+ const inspectPath = extractExtensionPath(rawFtsReason);
830
+ const { kind, remedy } = diagnoseExtensionLoad(rawFtsReason, 'FTS', inspectPath, resolveFtsVersionPair(inspectPath));
831
+ const remedyTail = usesClassifiedLoadRemedy(kind)
808
832
  ? ` ${remedy}`
809
833
  : '. Retry with network access and GITNEXUS_LBUG_EXTENSION_INSTALL=auto to install it, ' +
810
834
  'or pre-install the extension file; run `gitnexus doctor` for live FTS status.';
@@ -813,6 +837,22 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
813
837
  remedyTail);
814
838
  }
815
839
  progress('fts', 85, 'Repairing search indexes...');
840
+ // Restamp before CREATE so a second native abort still has in-place
841
+ // FTS dirty evidence after persist cleared the first stamp.
842
+ try {
843
+ const latestBeforeCreate = (await loadMeta(metaDir)) ?? existingMeta;
844
+ await saveMeta(metaDir, {
845
+ ...latestBeforeCreate,
846
+ incrementalInProgress: buildFtsDirtyStamp({
847
+ prior: latestBeforeCreate.incrementalInProgress,
848
+ writePlan: 'in-place',
849
+ checkpointSucceeded: true,
850
+ }),
851
+ });
852
+ }
853
+ catch (err) {
854
+ log(`FTS dirty restamp write failed (non-critical, continuing with repair${err instanceof Error ? `: ${err.message}` : ''}).`);
855
+ }
816
856
  const repairFailures = await createSearchFTSIndexes({
817
857
  indexes: ftsIndexes,
818
858
  onIndexStart: options.verbose
@@ -859,6 +899,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
859
899
  const latestMeta = (await loadMeta(metaDir)) ?? existingMeta;
860
900
  await saveMeta(metaDir, {
861
901
  ...latestMeta,
902
+ incrementalInProgress: undefined,
862
903
  capabilities: {
863
904
  graph: latestMeta.capabilities?.graph ?? DEFAULT_GRAPH_CAPABILITY,
864
905
  fts: { provider: 'ladybugdb-fts', status: 'available' },
@@ -939,10 +980,12 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
939
980
  resumedEmbeddingCheckpoint = decision.resumedFrom;
940
981
  }
941
982
  }
942
- // ── Crash recovery: dirty flag forces full rebuild ────────────────
943
- // If the previous incremental run set incrementalInProgress and didn't
944
- // clear it, the on-disk index may be in a half-state. Cheapest path
945
- // back to a known-good index is to wipe + rebuild from scratch.
983
+ // ── Crash recovery ────────────────────────────────────────────────
984
+ // A non-FTS dirty flag (or an FTS abort that never checkpointed) still
985
+ // forces a wipe + rebuild. An in-place FTS abort AFTER a successful
986
+ // graph-boundary checkpoint (Windows full rebuild, POSIX incremental)
987
+ // parks the live WAL and keeps the graph. A staging FTS abort never
988
+ // parks the live WAL — that index must replay its own delta.
946
989
  if (existingMeta?.incrementalInProgress) {
947
990
  const dirty = existingMeta.incrementalInProgress;
948
991
  const dirtyDetails = typeof dirty === 'object'
@@ -967,48 +1010,82 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
967
1010
  .filter(Boolean)
968
1011
  .join(', ')
969
1012
  : 'legacy dirty flag';
970
- log(
971
- // "analyze run", not "incremental run" — since #2099 F1 the flag is a
972
- // generic dirty marker written by BOTH writeback branches.
973
- 'Previous analyze run did not complete cleanly (incrementalInProgress flag set); ' +
974
- `last dirty state: ${dirtyDetails}; ` +
975
- 'forcing full rebuild to restore a known-good index.');
976
- options = { ...options, force: true };
977
- // Reload meta after clearing the flag in-memory; we still want fileHashes
978
- // for the post-rebuild meta carry-over, but force=true ensures the
979
- // rebuild path executes.
980
- //
981
- // #2409 defect 2: the crashed writeback's WAL can be poisoned — replaying
982
- // it kills the process natively, and the first DB open of this recovery
983
- // run (the embedding-cache preservation open below) happens BEFORE the
984
- // rebuild wipe that would discard it. Park the WAL/shadow sidecars aside
985
- // now, while nothing is open, so every open in this run is replay-free.
986
- // The rebuild wipes the DB regardless, so no committed data is at stake.
987
- await ensureWritableStorage();
988
- const { removed, failed } = await quarantineSidecarsForDirtyRecovery(lbugPath, log);
989
- if (removed.length > 0) {
990
- log(`Dirty-state recovery discarded ${removed.map((p) => path.basename(p)).join(', ')} ` +
991
- 'from the interrupted run (the file could not be moved aside, so its bytes were ' +
992
- 'removed — post-mortem forensics lost). Recovery proceeds with full embedding ' +
993
- 'preservation.');
994
- }
995
- if (failed.length > 0) {
996
- // FIX 1 (this shipping review, replacing the tri-review 4669518496
997
- // P2-3 drop-shape design): under a persistent lock the old drop-shape
998
- // run derived its embedding mode as "drop", ran the WHOLE pipeline,
999
- // and then died at the rebuild wipe on the very same handle — wasting
1000
- // minutes and zeroing embeddings on the way. A possibly-poisoned
1001
- // sidecar still sits next to the DB (any pre-wipe open would replay it
1002
- // and die), so failing here, in seconds, with the same actionable
1003
- // typed error the wipe would eventually throw is strictly better —
1004
- // and the CLI's LbugWipeError handler already renders it
1005
- // (recoveryHint 'lbug-wipe-failed'). The message is self-contained
1006
- // (headline + paths + lock guidance) because serve forwards only
1007
- // err.message over worker IPC.
1008
- throw new LbugWipeError(failed, {
1009
- headline: "Cannot start dirty-state recovery — the interrupted run's LadybugDB sidecars " +
1010
- 'could neither be moved aside nor removed:',
1011
- });
1013
+ const persistFtsNativeAbortRecovery = async () => {
1014
+ existingMeta.incrementalInProgress = undefined;
1015
+ existingMeta.capabilities = {
1016
+ ...existingMeta.capabilities,
1017
+ graph: existingMeta.capabilities?.graph ?? DEFAULT_GRAPH_CAPABILITY,
1018
+ fts: {
1019
+ provider: existingMeta.capabilities?.fts?.provider ?? 'ladybugdb-fts',
1020
+ status: 'unavailable',
1021
+ skipReason: 'native-abort',
1022
+ ...(dirty.writePlan ? { writePlan: dirty.writePlan } : {}),
1023
+ },
1024
+ vectorSearch: existingMeta.capabilities?.vectorSearch ?? DEFAULT_VECTOR_SEARCH_CAPABILITY,
1025
+ };
1026
+ await saveMeta(metaDir, existingMeta);
1027
+ };
1028
+ if (allowsFtsCrashWalPark(dirty)) {
1029
+ log('Previous analyze run aborted during in-place FTS after a successful graph checkpoint ' +
1030
+ `(${dirtyDetails}); parking leftover WAL/shadow so the existing graph can reopen. ` +
1031
+ 'Search indexes stay skipped until `gitnexus analyze --repair-fts`.');
1032
+ await ensureWritableStorage();
1033
+ const { failed } = await quarantineSidecarsForDirtyRecovery(lbugPath, (message) => log(` ${message}`));
1034
+ if (failed.length > 0) {
1035
+ throw new Error(ftsCrashParkFailureMessage(failed[0]));
1036
+ }
1037
+ await persistFtsNativeAbortRecovery();
1038
+ }
1039
+ else if (isFtsStagingDirty(dirty)) {
1040
+ log('Previous analyze run aborted during FTS after a staging writeback ' +
1041
+ `(${dirtyDetails}); leaving the live index WAL in place so the unpublished ` +
1042
+ 'staging file can still replay. Not forcing a rebuild.');
1043
+ await persistFtsNativeAbortRecovery();
1044
+ }
1045
+ else {
1046
+ log(
1047
+ // "analyze run", not "incremental run" — since #2099 F1 the flag is a
1048
+ // generic dirty marker written by BOTH writeback branches.
1049
+ 'Previous analyze run did not complete cleanly (incrementalInProgress flag set); ' +
1050
+ `last dirty state: ${dirtyDetails}; ` +
1051
+ 'forcing full rebuild to restore a known-good index.');
1052
+ options = { ...options, force: true };
1053
+ // Reload meta after clearing the flag in-memory; we still want fileHashes
1054
+ // for the post-rebuild meta carry-over, but force=true ensures the
1055
+ // rebuild path executes.
1056
+ //
1057
+ // #2409 defect 2: the crashed writeback's WAL can be poisoned — replaying
1058
+ // it kills the process natively, and the first DB open of this recovery
1059
+ // run (the embedding-cache preservation open below) happens BEFORE the
1060
+ // rebuild wipe that would discard it. Park the WAL/shadow sidecars aside
1061
+ // now, while nothing is open, so every open in this run is replay-free.
1062
+ // The rebuild wipes the DB regardless, so no committed data is at stake.
1063
+ await ensureWritableStorage();
1064
+ const { removed, failed } = await quarantineSidecarsForDirtyRecovery(lbugPath, log);
1065
+ if (removed.length > 0) {
1066
+ log(`Dirty-state recovery discarded ${removed.map((p) => path.basename(p)).join(', ')} ` +
1067
+ 'from the interrupted run (the file could not be moved aside, so its bytes were ' +
1068
+ 'removed — post-mortem forensics lost). Recovery proceeds with full embedding ' +
1069
+ 'preservation.');
1070
+ }
1071
+ if (failed.length > 0) {
1072
+ // FIX 1 (this shipping review, replacing the tri-review 4669518496
1073
+ // P2-3 drop-shape design): under a persistent lock the old drop-shape
1074
+ // run derived its embedding mode as "drop", ran the WHOLE pipeline,
1075
+ // and then died at the rebuild wipe on the very same handle — wasting
1076
+ // minutes and zeroing embeddings on the way. A possibly-poisoned
1077
+ // sidecar still sits next to the DB (any pre-wipe open would replay it
1078
+ // and die), so failing here, in seconds, with the same actionable
1079
+ // typed error the wipe would eventually throw is strictly better —
1080
+ // and the CLI's LbugWipeError handler already renders it
1081
+ // (recoveryHint 'lbug-wipe-failed'). The message is self-contained
1082
+ // (headline + paths + lock guidance) because serve forwards only
1083
+ // err.message over worker IPC.
1084
+ throw new LbugWipeError(failed, {
1085
+ headline: "Cannot start dirty-state recovery — the interrupted run's LadybugDB sidecars " +
1086
+ 'could neither be moved aside nor removed:',
1087
+ });
1088
+ }
1012
1089
  }
1013
1090
  }
1014
1091
  // ── pdg-mode flip forces full writeback (#2099 F1) ─────────────────
@@ -1387,6 +1464,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
1387
1464
  stats: existingMeta.stats ?? {},
1388
1465
  alreadyUpToDate: true,
1389
1466
  ...(ftsDisabledReason ? { ftsSkipped: true, ftsSkipReason: ftsDisabledReason } : {}),
1467
+ ...(!ftsDisabledReason && priorFtsNativeAbort
1468
+ ? { ftsSkipped: true, ftsSkipReason: 'native-abort' }
1469
+ : {}),
1390
1470
  isPrimaryBranch: !placement.branch,
1391
1471
  };
1392
1472
  }
@@ -2242,7 +2322,10 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
2242
2322
  : undefined,
2243
2323
  ]
2244
2324
  .filter((e) => e !== undefined)
2245
- .map(({ reason, label }) => diagnoseExtensionLoad(reason, label).remedy);
2325
+ .map(({ reason, label }) => {
2326
+ const inspectPath = extractExtensionPath(reason);
2327
+ return diagnoseExtensionLoad(reason, label, inspectPath, label === 'FTS' ? resolveFtsVersionPair(inspectPath) : undefined).remedy;
2328
+ });
2246
2329
  log(`Incremental: ${escalationCauses.join('; and ')} — switching to a full DB write ` +
2247
2330
  `(wipe + bulk COPY) for this run; file-level incremental bookkeeping is unaffected.` +
2248
2331
  (degradedEffects.length > 0
@@ -2499,12 +2582,6 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
2499
2582
  await restoreDerivedRels(derivedSnapshot);
2500
2583
  }
2501
2584
  }
2502
- // Boundary drain (#2409): checkpoint at the end of the incremental
2503
- // writeback so the WAL it accumulated never lingers into the FTS and
2504
- // embedding phases — a later crash leaves only post-checkpoint WAL for
2505
- // the next open to replay. Near-instant when the periodic driver has
2506
- // kept up; rides the driver's bounded retry via runCheckpointWithRetry.
2507
- await checkpointOnce();
2508
2585
  }
2509
2586
  else {
2510
2587
  // ── Full rebuild ───────────────────────────────────────────────
@@ -2518,6 +2595,41 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
2518
2595
  progress('lbug', pct, msg);
2519
2596
  }, pipelineResult.pdgEmitManifest, pipelineResult.graphEmitManifest, contentRetention);
2520
2597
  }
2598
+ // Converged graph-boundary drain: incremental used to checkpoint here;
2599
+ // full rebuild did not. One site so FTS always starts after a settled
2600
+ // plan (post-escalation `buildPath`) and a recorded checkpoint outcome.
2601
+ const ftsWritePlan = resolveFtsWritePlan(buildPath, lbugPath);
2602
+ let boundaryCheckpointSucceeded = false;
2603
+ try {
2604
+ boundaryCheckpointSucceeded = await checkpointOnce();
2605
+ }
2606
+ catch (error) {
2607
+ if (isBoundaryCheckpointFatal(ftsWritePlan)) {
2608
+ throw error;
2609
+ }
2610
+ const detail = error instanceof Error ? error.message : String(error);
2611
+ log(`Boundary WAL checkpoint failed on the in-place FTS path (best-effort): ${detail}. ` +
2612
+ 'Continuing; recovery will treat the graph-boundary checkpoint as unsuccessful.');
2613
+ }
2614
+ if (shouldStampFtsDirtyPhase(ftsWritePlan)) {
2615
+ // Lift the prior-meta precondition: a first-ever in-place run (Windows
2616
+ // full rebuild, or any in-place incremental) must stamp too. Staging
2617
+ // never stamps — an abort there abandons the unpublished file.
2618
+ const latestMeta = (await loadMeta(metaDir)) ?? existingMeta;
2619
+ const base = latestMeta ?? {
2620
+ repoPath,
2621
+ lastCommit: '',
2622
+ indexedAt: new Date().toISOString(),
2623
+ };
2624
+ await saveMeta(metaDir, {
2625
+ ...base,
2626
+ incrementalInProgress: buildFtsDirtyStamp({
2627
+ prior: base.incrementalInProgress,
2628
+ writePlan: 'in-place',
2629
+ checkpointSucceeded: boundaryCheckpointSucceeded,
2630
+ }),
2631
+ });
2632
+ }
2521
2633
  // ── Phase 3: FTS (85–90%) ─────────────────────────────────────────
2522
2634
  // The analyze (write) path owns building the search indexes, so it uses
2523
2635
  // the `auto` install policy (LOAD-first, then one bounded INSTALL) —
@@ -2541,12 +2653,28 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
2541
2653
  let ftsReady = ftsAvailable;
2542
2654
  // Why FTS ended up skipped (#2658 review L2): an explicit opt-out
2543
2655
  // (`disabled-by-flag` / `disabled-by-env`, #3091) when one was recorded,
2656
+ // else tuple-missing when the loader reported no packaged artifact,
2544
2657
  // else extension-unavailable up front, or build-failed in the degrade
2545
2658
  // branch below.
2659
+ const tupleMissing = !ftsAvailable &&
2660
+ !ftsDisabledReason &&
2661
+ /no packaged FTS artifact/i.test(getFtsCapability()?.reason ?? '');
2546
2662
  let ftsSkipReason = ftsAvailable
2547
2663
  ? undefined
2548
- : (ftsDisabledReason ?? 'extension-unavailable');
2549
- if (ftsAvailable) {
2664
+ : (ftsDisabledReason ?? (tupleMissing ? 'tuple-missing' : 'extension-unavailable'));
2665
+ if (ftsAvailable && priorFtsNativeAbort && !requestedRepairFts) {
2666
+ // KTD6 / R11: do not retry CREATE_FTS_INDEX after a native abort —
2667
+ // the same content can kill the process again. `--repair-fts` is the
2668
+ // explicit retry: its early-return path never reaches this branch,
2669
+ // and a retention-mismatch rewrite that started as `--repair-fts`
2670
+ // still creates indexes.
2671
+ ftsReady = false;
2672
+ ftsSkipReason = 'native-abort';
2673
+ log('FTS index build skipped — a previous analyze aborted while building search indexes. ' +
2674
+ 'Graph analysis completed. Run `gitnexus analyze --repair-fts` to retry.');
2675
+ progress('fts', 90, 'Search indexes skipped (previous native abort)');
2676
+ }
2677
+ else if (ftsAvailable) {
2550
2678
  // Degrade rather than throw: createSearchFTSIndexes re-tokenizes every
2551
2679
  // stored row on every run, so a native tokenizer error on a single
2552
2680
  // pre-existing row (#2544/#2546) must not discard this run's otherwise-
@@ -2598,8 +2726,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
2598
2726
  // Same #2383 mock seam as the repair path above — keep the exported
2599
2727
  // `getExtensionCapabilities()` lookup here.
2600
2728
  const ftsReason = getExtensionCapabilities().find((c) => c.name === 'fts')?.reason;
2601
- const { kind, remedy } = diagnoseExtensionLoad(ftsReason);
2602
- log(kind === 'missing_dependency'
2729
+ const inspectPath = extractExtensionPath(ftsReason);
2730
+ const { kind, remedy } = diagnoseExtensionLoad(ftsReason, 'FTS', inspectPath, resolveFtsVersionPair(inspectPath));
2731
+ log(usesClassifiedLoadRemedy(kind)
2603
2732
  ? `${FTS_UNAVAILABLE_LEAD} ${remedy}`
2604
2733
  : FTS_UNAVAILABLE_MESSAGE);
2605
2734
  progress('fts', 90, 'Search indexes skipped (FTS unavailable)');
@@ -3252,6 +3381,14 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
3252
3381
  // 'unavailable', and the next run does it again. Stamping the
3253
3382
  // discriminator the run already computed makes the read exact instead.
3254
3383
  skipReason: ftsReady ? undefined : ftsSkipReason,
3384
+ // Keep the abort write plan after persist cleared the dirty flag
3385
+ // so a leftover live WAL is still refuse-able. Successful FTS
3386
+ // drops it with skipReason.
3387
+ ...(!ftsReady &&
3388
+ ftsSkipReason === 'native-abort' &&
3389
+ existingMeta?.capabilities?.fts?.writePlan
3390
+ ? { writePlan: existingMeta.capabilities.fts.writePlan }
3391
+ : {}),
3255
3392
  },
3256
3393
  vectorSearch: {
3257
3394
  provider: effectiveSemanticMode === 'vector-index' ? 'ladybugdb-vector' : 'exact-scan',
@@ -0,0 +1,59 @@
1
+ /**
2
+ * FTS-phase dirty-flag policy (KTD4 / KTD6).
3
+ *
4
+ * A native abort during CREATE_FTS_INDEX kills the process before JS can
5
+ * persist a skip reason. The next run infers `native-abort` from this phase
6
+ * value, or from a persisted `capabilities.fts.skipReason` of `native-abort`
7
+ * after recovery clears the dirty flag. Tests induce the flag through
8
+ * saveMeta — they cannot be a green real-abort of analyze.
9
+ */
10
+ import type { RepoMeta } from '../../storage/repo-meta.js';
11
+ export declare const FTS_DIRTY_PHASE: "fts";
12
+ export type FtsWritePlan = 'in-place' | 'staging';
13
+ export type IncrementalDirtyState = NonNullable<RepoMeta['incrementalInProgress']>;
14
+ export declare const resolveFtsWritePlan: (buildPath: string, livePath: string) => FtsWritePlan;
15
+ export declare const shouldStampFtsDirtyPhase: (writePlan: FtsWritePlan) => boolean;
16
+ /** Staging throws (abandons the unpublished file). In-place is best-effort. */
17
+ export declare const isBoundaryCheckpointFatal: (writePlan: FtsWritePlan) => boolean;
18
+ export declare const isFtsDirtyPhase: (dirty: RepoMeta["incrementalInProgress"] | undefined) => dirty is IncrementalDirtyState & {
19
+ phase: typeof FTS_DIRTY_PHASE;
20
+ };
21
+ /**
22
+ * Half-written graph, or FTS-phase without a successful checkpoint, blocks
23
+ * `--repair-fts`. FTS-phase + `checkpointSucceeded === true` is admitted
24
+ * (in-place or staging). Staging still must not park.
25
+ */
26
+ export declare const shouldRefuseRepairFtsWhileDirty: (dirty: RepoMeta["incrementalInProgress"] | undefined) => boolean;
27
+ export declare const inferNativeAbortSkip: (dirty: RepoMeta["incrementalInProgress"] | undefined, skipReason?: string) => boolean;
28
+ /**
29
+ * KTD5 conjunctive warrant for parking a live WAL: FTS phase, in-place
30
+ * write plan (Windows full rebuild, POSIX incremental, escalated-in-place),
31
+ * and a successful graph-boundary checkpoint. Staging never qualifies —
32
+ * the live index next to an unpublished staging file must keep its WAL.
33
+ */
34
+ export declare const allowsFtsCrashWalPark: (dirty: RepoMeta["incrementalInProgress"] | undefined) => boolean;
35
+ export declare const isInPlaceFtsDirty: (dirty: RepoMeta["incrementalInProgress"] | undefined) => dirty is IncrementalDirtyState & {
36
+ phase: typeof FTS_DIRTY_PHASE;
37
+ writePlan: "in-place";
38
+ };
39
+ /** Persisted FTS capability fields used as crash evidence after the dirty flag is cleared. */
40
+ export type PersistedFtsCrashEvidence = {
41
+ skipReason?: string;
42
+ writePlan?: FtsWritePlan;
43
+ };
44
+ export declare const hasRecoveredInPlaceFtsAbort: (fts: PersistedFtsCrashEvidence | undefined) => boolean;
45
+ /**
46
+ * Reader / `--repair-fts` refuse-or-park warrant. Any in-place FTS dirty
47
+ * flag is enough — a failed graph-boundary checkpoint still leaves CREATE
48
+ * able to abort with a live WAL. After persist clears the flag, a
49
+ * `native-abort` skip plus persisted `writePlan: 'in-place'` is the same
50
+ * evidence. Staging persist also writes `native-abort` and must not match.
51
+ */
52
+ export declare const shouldRefuseFtsCrashWal: (dirty: RepoMeta["incrementalInProgress"] | undefined, fts?: PersistedFtsCrashEvidence) => boolean;
53
+ export declare const isFtsStagingDirty: (dirty: RepoMeta["incrementalInProgress"] | undefined) => boolean;
54
+ export declare const buildFtsDirtyStamp: (args: {
55
+ prior?: IncrementalDirtyState;
56
+ now?: number;
57
+ writePlan: "in-place";
58
+ checkpointSucceeded: boolean;
59
+ }) => IncrementalDirtyState;
@@ -0,0 +1,55 @@
1
+ export const FTS_DIRTY_PHASE = 'fts';
2
+ export const resolveFtsWritePlan = (buildPath, livePath) => buildPath === livePath ? 'in-place' : 'staging';
3
+ export const shouldStampFtsDirtyPhase = (writePlan) => writePlan === 'in-place';
4
+ /** Staging throws (abandons the unpublished file). In-place is best-effort. */
5
+ export const isBoundaryCheckpointFatal = (writePlan) => writePlan === 'staging';
6
+ export const isFtsDirtyPhase = (dirty) => dirty?.phase === FTS_DIRTY_PHASE;
7
+ /**
8
+ * Half-written graph, or FTS-phase without a successful checkpoint, blocks
9
+ * `--repair-fts`. FTS-phase + `checkpointSucceeded === true` is admitted
10
+ * (in-place or staging). Staging still must not park.
11
+ */
12
+ export const shouldRefuseRepairFtsWhileDirty = (dirty) => dirty != null && (!isFtsDirtyPhase(dirty) || dirty.checkpointSucceeded !== true);
13
+ export const inferNativeAbortSkip = (dirty, skipReason) => isFtsDirtyPhase(dirty) || skipReason === 'native-abort';
14
+ /**
15
+ * KTD5 conjunctive warrant for parking a live WAL: FTS phase, in-place
16
+ * write plan (Windows full rebuild, POSIX incremental, escalated-in-place),
17
+ * and a successful graph-boundary checkpoint. Staging never qualifies —
18
+ * the live index next to an unpublished staging file must keep its WAL.
19
+ */
20
+ export const allowsFtsCrashWalPark = (dirty) => isFtsDirtyPhase(dirty) && dirty.writePlan === 'in-place' && dirty.checkpointSucceeded === true;
21
+ export const isInPlaceFtsDirty = (dirty) => isFtsDirtyPhase(dirty) && dirty.writePlan === 'in-place';
22
+ export const hasRecoveredInPlaceFtsAbort = (fts) => fts?.skipReason === 'native-abort' && fts.writePlan === 'in-place';
23
+ /**
24
+ * Reader / `--repair-fts` refuse-or-park warrant. Any in-place FTS dirty
25
+ * flag is enough — a failed graph-boundary checkpoint still leaves CREATE
26
+ * able to abort with a live WAL. After persist clears the flag, a
27
+ * `native-abort` skip plus persisted `writePlan: 'in-place'` is the same
28
+ * evidence. Staging persist also writes `native-abort` and must not match.
29
+ */
30
+ export const shouldRefuseFtsCrashWal = (dirty, fts) => isInPlaceFtsDirty(dirty) || hasRecoveredInPlaceFtsAbort(fts);
31
+ export const isFtsStagingDirty = (dirty) => isFtsDirtyPhase(dirty) && dirty.writePlan === 'staging';
32
+ export const buildFtsDirtyStamp = (args) => {
33
+ const now = args.now ?? Date.now();
34
+ const prior = args.prior;
35
+ return {
36
+ startedAt: prior?.startedAt ?? now,
37
+ updatedAt: now,
38
+ toWriteCount: prior?.toWriteCount ?? 0,
39
+ phase: FTS_DIRTY_PHASE,
40
+ writePlan: args.writePlan,
41
+ checkpointSucceeded: args.checkpointSucceeded,
42
+ ...(prior?.directWriteCount !== undefined ? { directWriteCount: prior.directWriteCount } : {}),
43
+ ...(prior?.importerExpansion !== undefined
44
+ ? { importerExpansion: prior.importerExpansion }
45
+ : {}),
46
+ ...(prior?.effectiveWriteCount !== undefined
47
+ ? { effectiveWriteCount: prior.effectiveWriteCount }
48
+ : {}),
49
+ ...(prior?.deleteCount !== undefined ? { deleteCount: prior.deleteCount } : {}),
50
+ ...(prior?.shadowSeedCount !== undefined ? { shadowSeedCount: prior.shadowSeedCount } : {}),
51
+ ...(prior?.droppedImporterChunks !== undefined
52
+ ? { droppedImporterChunks: prior.droppedImporterChunks }
53
+ : {}),
54
+ };
55
+ };
@@ -1,7 +1,7 @@
1
1
  import { createFTSIndex, dropFTSIndex, indexRowName, indexRowTable, resolveGateRows, DEFAULT_FTS_STEMMER, } from '../lbug/lbug-adapter.js';
2
2
  import { getFtsCapability } from '../lbug/extension-loader.js';
3
3
  import { FTS_DISABLED_MESSAGE } from './fts-policy.js';
4
- import { classifyExtensionLoadError } from '../lbug/extension-load-error.js';
4
+ import { classifyExtensionLoadError, usesClassifiedLoadRemedy, } from '../lbug/extension-load-error.js';
5
5
  import { FTS_INDEXES } from './fts-schema.js';
6
6
  /**
7
7
  * Strip filesystem paths from a LadybugDB error before it reaches the HTTP
@@ -59,7 +59,7 @@ export const ftsDegradedWarning = (context, disabledReason) => {
59
59
  // per-request path (HTTP /api/search + MCP query) does NO file I/O (#2383 F3);
60
60
  // fall back to the pure, no-I/O string classifier if it is somehow absent.
61
61
  const { kind, remedy } = fts.diagnosis ?? classifyExtensionLoadError(fts.reason);
62
- const tail = kind === 'missing_dependency'
62
+ const tail = usesClassifiedLoadRemedy(kind)
63
63
  ? ` ${remedy}`
64
64
  : '. Run `gitnexus doctor` for details, then `gitnexus analyze --repair-fts` with network access to reinstall.';
65
65
  return ('FTS extension failed to load — keyword search degraded' +
@@ -1,6 +1,11 @@
1
1
  import type { RepoMeta } from '../../storage/repo-meta.js';
2
2
  export type FtsDisabledReason = 'disabled-by-flag' | 'disabled-by-env';
3
- export type FtsSkipReason = FtsDisabledReason | 'extension-unavailable' | 'build-failed';
3
+ /** Failure-like skip reasons. Must not join `FtsDisabledReason` — that
4
+ * predicate is a hand-written `string` check the compiler does not verify. */
5
+ export type FtsFailureSkipReason = 'extension-unavailable' | 'build-failed' | 'native-abort' | 'tuple-missing';
6
+ export type FtsSkipReason = FtsDisabledReason | FtsFailureSkipReason;
7
+ /** Runtime list for storage/core parity tests. Order is not significant. */
8
+ export declare const FTS_SKIP_REASONS: readonly FtsSkipReason[];
4
9
  type RepoCapabilities = NonNullable<RepoMeta['capabilities']>;
5
10
  export declare const DEFAULT_GRAPH_CAPABILITY: RepoCapabilities['graph'];
6
11
  export declare const DEFAULT_VECTOR_SEARCH_CAPABILITY: RepoCapabilities['vectorSearch'];
@@ -15,4 +20,9 @@ export declare function getFtsDisabledReason(capability: RepoCapabilities['fts']
15
20
  */
16
21
  export declare function withExplicitFtsDisablement(meta: RepoMeta, reason: FtsDisabledReason | undefined): RepoMeta;
17
22
  export declare const FTS_DISABLED_MESSAGE: string;
23
+ /**
24
+ * CLI analyze summary copy for a skipped FTS build. Total switch so a new
25
+ * member cannot silently inherit the network-install remedy.
26
+ */
27
+ export declare const formatAnalyzeFtsSkipSummary: (reason: FtsSkipReason | undefined) => string;
18
28
  export {};
@@ -1,3 +1,14 @@
1
+ const _ftsSkipReasonStorageParity = true;
2
+ void _ftsSkipReasonStorageParity;
3
+ /** Runtime list for storage/core parity tests. Order is not significant. */
4
+ export const FTS_SKIP_REASONS = [
5
+ 'disabled-by-flag',
6
+ 'disabled-by-env',
7
+ 'extension-unavailable',
8
+ 'build-failed',
9
+ 'native-abort',
10
+ 'tuple-missing',
11
+ ];
1
12
  export const DEFAULT_GRAPH_CAPABILITY = {
2
13
  provider: 'ladybugdb',
3
14
  status: 'available',
@@ -54,3 +65,39 @@ export function withExplicitFtsDisablement(meta, reason) {
54
65
  }
55
66
  export const FTS_DISABLED_MESSAGE = 'FTS disabled for this index. To enable keyword search, run gitnexus analyze ' +
56
67
  'without --skip-fts and with GITNEXUS_SKIP_FTS unset.';
68
+ const FTS_BUILD_FAILED_MESSAGE = 'Warning: full-text/BM25 search is disabled — the search index build failed this run.\n' +
69
+ ' The FTS extension is available; rerun `gitnexus analyze --repair-fts`. If it persists,\n' +
70
+ ' check the disk for space or corruption. Run `gitnexus doctor` for details.';
71
+ const FTS_EXTENSION_UNAVAILABLE_MESSAGE = 'Warning: full-text/BM25 search is disabled — the LadybugDB FTS extension was unavailable.\n' +
72
+ ' Install it once with network access (GITNEXUS_LBUG_EXTENSION_INSTALL=auto), then run\n' +
73
+ ' `gitnexus analyze --repair-fts` to build the search indexes. Run `gitnexus doctor` for details.';
74
+ const FTS_NATIVE_ABORT_MESSAGE = 'Warning: full-text/BM25 search is disabled — a previous analyze aborted while building the search indexes.\n' +
75
+ ' Rerun `gitnexus analyze --repair-fts` to recover. Run `gitnexus doctor` for details.';
76
+ const FTS_TUPLE_MISSING_MESSAGE = 'Warning: full-text/BM25 search is disabled — no packaged FTS artifact is available for this platform.\n' +
77
+ ' Keyword search is unavailable on this host. Run `gitnexus doctor` for details.';
78
+ const assertNever = (value) => {
79
+ throw new Error(`unhandled FTS skip reason: ${String(value)}`);
80
+ };
81
+ /**
82
+ * CLI analyze summary copy for a skipped FTS build. Total switch so a new
83
+ * member cannot silently inherit the network-install remedy.
84
+ */
85
+ export const formatAnalyzeFtsSkipSummary = (reason) => {
86
+ if (reason === undefined)
87
+ return FTS_EXTENSION_UNAVAILABLE_MESSAGE;
88
+ switch (reason) {
89
+ case 'disabled-by-flag':
90
+ case 'disabled-by-env':
91
+ return FTS_DISABLED_MESSAGE;
92
+ case 'build-failed':
93
+ return FTS_BUILD_FAILED_MESSAGE;
94
+ case 'native-abort':
95
+ return FTS_NATIVE_ABORT_MESSAGE;
96
+ case 'tuple-missing':
97
+ return FTS_TUPLE_MISSING_MESSAGE;
98
+ case 'extension-unavailable':
99
+ return FTS_EXTENSION_UNAVAILABLE_MESSAGE;
100
+ default:
101
+ return assertNever(reason);
102
+ }
103
+ };