@phren/cli 0.1.40 → 0.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/profiles.js +59 -5
- package/dist/cli/actions.js +11 -3
- package/dist/cli/config-access.js +8 -1
- package/dist/cli/extract.d.ts +27 -1
- package/dist/cli/extract.js +243 -9
- package/dist/cli/govern.js +6 -87
- package/dist/cli/hooks-context.d.ts +1 -1
- package/dist/cli/hooks-context.js +1 -1
- package/dist/cli/namespaces-review.js +14 -1
- package/dist/cli/session-git.d.ts +18 -0
- package/dist/cli/session-git.js +47 -0
- package/dist/cli/session-start.js +2 -2
- package/dist/cli/session-stop.d.ts +11 -0
- package/dist/cli/session-stop.js +141 -33
- package/dist/cli/session-tool-hook.js +17 -5
- package/dist/cli-registry.d.ts +1 -1
- package/dist/cli-registry.js +1 -0
- package/dist/content/dedup.d.ts +1 -4
- package/dist/content/dedup.js +7 -63
- package/dist/content/learning.d.ts +20 -2
- package/dist/content/learning.js +21 -3
- package/dist/content/quality.d.ts +23 -0
- package/dist/content/quality.js +123 -0
- package/dist/content/secrets.d.ts +52 -0
- package/dist/content/secrets.js +193 -0
- package/dist/data/access.d.ts +56 -2
- package/dist/data/access.js +246 -24
- package/dist/entrypoint.js +5 -1
- package/dist/finding/impact.js +170 -43
- package/dist/finding/lifecycle.d.ts +19 -2
- package/dist/finding/lifecycle.js +15 -3
- package/dist/git-worktree.d.ts +19 -0
- package/dist/git-worktree.js +117 -0
- package/dist/governance/policy.d.ts +84 -4
- package/dist/governance/policy.js +209 -14
- package/dist/governance/rbac.d.ts +8 -0
- package/dist/governance/rbac.js +26 -0
- package/dist/hooks.d.ts +20 -0
- package/dist/hooks.js +113 -3
- package/dist/index.js +5 -1
- package/dist/init/guard-globals.d.ts +20 -0
- package/dist/init/guard-globals.js +72 -6
- package/dist/init/init-configure.js +6 -6
- package/dist/init/init.js +10 -6
- package/dist/init/setup.d.ts +4 -3
- package/dist/init/setup.js +120 -16
- package/dist/init/store-gitignore.d.ts +29 -0
- package/dist/init/store-gitignore.js +48 -0
- package/dist/link/doctor.js +86 -20
- package/dist/phren-core.d.ts +35 -8
- package/dist/phren-core.js +35 -8
- package/dist/phren-paths.d.ts +90 -1
- package/dist/phren-paths.js +150 -5
- package/dist/proactivity.js +1 -1
- package/dist/shared/embedding-cache.d.ts +16 -0
- package/dist/shared/embedding-cache.js +24 -0
- package/dist/shared/fragment-graph.js +10 -0
- package/dist/shared/index.d.ts +6 -0
- package/dist/shared/index.js +523 -138
- package/dist/shared/ollama.d.ts +14 -2
- package/dist/shared/ollama.js +84 -5
- package/dist/shared/retrieval.d.ts +22 -1
- package/dist/shared/retrieval.js +30 -1
- package/dist/shared/search-fallback.js +15 -3
- package/dist/shared/vector-index.d.ts +20 -1
- package/dist/shared/vector-index.js +18 -4
- package/dist/shared.d.ts +1 -1
- package/dist/shared.js +1 -1
- package/dist/status.js +18 -1
- package/dist/store-registry.d.ts +22 -0
- package/dist/store-registry.js +36 -3
- package/dist/task/lifecycle.js +31 -0
- package/dist/tools/config.js +16 -7
- package/dist/tools/data.js +1 -1
- package/dist/tools/extract.js +2 -2
- package/dist/tools/finding.js +14 -3
- package/dist/tools/graph.js +9 -1
- package/dist/tools/memory.js +1 -1
- package/dist/tools/notes.js +3 -3
- package/dist/tools/ops.js +19 -18
- package/dist/tools/search.js +2 -2
- package/dist/tools/session.js +2 -2
- package/dist/tools/tasks.js +2 -2
- package/dist/tools/types.d.ts +32 -2
- package/dist/tools/types.js +41 -8
- package/dist/ui/scripts.js +2 -2
- package/dist/ui/server.js +5 -0
- package/package.json +1 -1
- package/dist/cli-hooks-git.d.ts +0 -50
- package/dist/cli-hooks-git.js +0 -288
- package/starter/my-api/.claude/skills/README.md +0 -7
- package/starter/my-api/CLAUDE.md +0 -33
- package/starter/my-api/FINDINGS.md +0 -9
- package/starter/my-api/summary.md +0 -7
- package/starter/my-api/tasks.md +0 -7
- package/starter/my-first-project/.claude/skills/README.md +0 -7
- package/starter/my-first-project/CLAUDE.md +0 -49
- package/starter/my-first-project/FINDINGS.md +0 -24
- package/starter/my-first-project/summary.md +0 -11
- package/starter/my-first-project/tasks.md +0 -25
- package/starter/my-frontend/.claude/skills/README.md +0 -7
- package/starter/my-frontend/CLAUDE.md +0 -33
- package/starter/my-frontend/FINDINGS.md +0 -9
- package/starter/my-frontend/summary.md +0 -7
- package/starter/my-frontend/tasks.md +0 -7
package/dist/shared/index.js
CHANGED
|
@@ -73,6 +73,7 @@ import { beginUserFragmentBuildCache, endUserFragmentBuildCache, extractAndLinkF
|
|
|
73
73
|
import { bootstrapSqlJs } from "./sqljs.js";
|
|
74
74
|
import { spawnDetachedChild } from "./process.js";
|
|
75
75
|
import { getProjectOwnershipMode, getProjectSourcePath, readProjectConfig } from "../project-config.js";
|
|
76
|
+
import { resolveRepoRootForPath } from "../git-worktree.js";
|
|
76
77
|
import { buildSourceDocKey, decodeStringRow, getDocSourceKey, queryDocRows, queryRows, } from "../index-query.js";
|
|
77
78
|
import { classifyTopicForText, readProjectTopics, } from "../project-topics.js";
|
|
78
79
|
export { porterStem } from "./stemmer.js";
|
|
@@ -159,6 +160,9 @@ const FILE_TYPE_MAP = {
|
|
|
159
160
|
"tasks.md": "task",
|
|
160
161
|
"changelog.md": "changelog",
|
|
161
162
|
"truths.md": "canonical",
|
|
163
|
+
// review.md is indexed on purpose so `search_knowledge` can answer "why is this in my
|
|
164
|
+
// queue?", but `review-queue` is in NON_INJECTABLE_TYPES (shared/retrieval.ts): it is
|
|
165
|
+
// unreviewed, quarantined content and must never be pushed into a prompt automatically.
|
|
162
166
|
"review.md": "review-queue",
|
|
163
167
|
};
|
|
164
168
|
function pathHasSegment(relPath, segment) {
|
|
@@ -249,17 +253,10 @@ function _resolveImportsRecursive(content, phrenPath, seen, depth) {
|
|
|
249
253
|
export function resolveImports(content, phrenPath) {
|
|
250
254
|
return _resolveImportsRecursive(content, phrenPath, new Set(), 0);
|
|
251
255
|
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
257
|
-
fs.writeFileSync(sentinelPath, Date.now().toString());
|
|
258
|
-
}
|
|
259
|
-
catch (err) {
|
|
260
|
-
logger.debug("touchSentinel", errorMessage(err));
|
|
261
|
-
}
|
|
262
|
-
}
|
|
256
|
+
// `touchSentinel()` used to write `.runtime/phren-sentinel` here. Nothing ever
|
|
257
|
+
// read that file — index freshness is decided by index-sentinel.json (directory
|
|
258
|
+
// mtimes + a re-hash of the recorded file list), and a content write already
|
|
259
|
+
// changes the file's own mtime, so the stat-hash invalidates on its own.
|
|
263
260
|
function computePhrenHash(phrenPath, profile, preGlobbed) {
|
|
264
261
|
const policy = getIndexPolicy(phrenPath);
|
|
265
262
|
const hash = crypto.createHash("sha1");
|
|
@@ -390,10 +387,59 @@ const INDEX_HASHES_FILENAME = "index-hashes.json";
|
|
|
390
387
|
const INDEX_SCHEMA_VERSION = 3; // bump when FTS schema changes to force full rebuild
|
|
391
388
|
// Deletion share above which an incremental update falls back to a full rebuild.
|
|
392
389
|
const MAX_INCREMENTAL_DELETE_RATIO = 0.5;
|
|
393
|
-
function
|
|
394
|
-
const content = fs.readFileSync(filePath, "utf-8");
|
|
390
|
+
function hashContent(content) {
|
|
395
391
|
return crypto.createHash("sha256").update(content).digest("hex");
|
|
396
392
|
}
|
|
393
|
+
function hashFileContent(filePath) {
|
|
394
|
+
return hashContent(fs.readFileSync(filePath, "utf-8"));
|
|
395
|
+
}
|
|
396
|
+
/** Read a file's text, or null when it is gone/unreadable (races with deletion are normal). */
|
|
397
|
+
function readFileOrNull(filePath) {
|
|
398
|
+
try {
|
|
399
|
+
return fs.readFileSync(filePath, "utf-8");
|
|
400
|
+
}
|
|
401
|
+
catch (err) {
|
|
402
|
+
logger.debug("readFileOrNull", errorMessage(err));
|
|
403
|
+
return null;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Carries a file's bytes from the pass that hashes it to the pass that indexes
|
|
408
|
+
* it, so one rebuild reads each file exactly once instead of 2x (hash + insert)
|
|
409
|
+
* or 3x (hash + insert + fragment extraction on findings files).
|
|
410
|
+
*
|
|
411
|
+
* Bounded on purpose: the incremental path can, in the worst case, see every
|
|
412
|
+
* file as changed, and holding a whole store's markdown in memory to save a
|
|
413
|
+
* re-read is a bad trade. Past the budget the cache simply stops accepting
|
|
414
|
+
* entries and the consumer falls back to reading from disk — slower, never
|
|
415
|
+
* wrong.
|
|
416
|
+
*/
|
|
417
|
+
const BUILD_CONTENT_BUDGET_BYTES = 32 * 1024 * 1024;
|
|
418
|
+
class BuildContentCache {
|
|
419
|
+
entries = new Map();
|
|
420
|
+
bytes = 0;
|
|
421
|
+
put(filePath, content) {
|
|
422
|
+
if (this.bytes + content.length > BUILD_CONTENT_BUDGET_BYTES)
|
|
423
|
+
return;
|
|
424
|
+
if (this.entries.has(filePath))
|
|
425
|
+
return;
|
|
426
|
+
this.entries.set(filePath, content);
|
|
427
|
+
this.bytes += content.length;
|
|
428
|
+
}
|
|
429
|
+
/** Read-and-forget: content is consumed exactly once, then released. */
|
|
430
|
+
take(filePath) {
|
|
431
|
+
const hit = this.entries.get(filePath);
|
|
432
|
+
if (hit === undefined)
|
|
433
|
+
return undefined;
|
|
434
|
+
this.entries.delete(filePath);
|
|
435
|
+
this.bytes -= hit.length;
|
|
436
|
+
return hit;
|
|
437
|
+
}
|
|
438
|
+
clear() {
|
|
439
|
+
this.entries.clear();
|
|
440
|
+
this.bytes = 0;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
397
443
|
/** Errors expected from idempotent schema migrations run against older cached DBs. */
|
|
398
444
|
function isExpectedMigrationError(err) {
|
|
399
445
|
const msg = errorMessage(err).toLowerCase();
|
|
@@ -572,7 +618,7 @@ export function normalizeIndexedContent(content, type, phrenPath, maxChars) {
|
|
|
572
618
|
}
|
|
573
619
|
function insertFileIntoIndex(db, entry, phrenPath, opts) {
|
|
574
620
|
try {
|
|
575
|
-
const raw = fs.readFileSync(entry.fullPath, "utf-8");
|
|
621
|
+
const raw = opts?.content ?? fs.readFileSync(entry.fullPath, "utf-8");
|
|
576
622
|
const content = normalizeIndexedContent(raw, entry.type, phrenPath);
|
|
577
623
|
const indexedContent = applyReferenceTopicHints(entry, content, phrenPath);
|
|
578
624
|
db.run("INSERT INTO docs (project, filename, type, content, path) VALUES (?, ?, ?, ?, ?)", [entry.project, entry.filename, entry.type, indexedContent, entry.fullPath]);
|
|
@@ -596,10 +642,47 @@ function extractLegacyTopicSlug(entry) {
|
|
|
596
642
|
return null;
|
|
597
643
|
return match[1].toLowerCase();
|
|
598
644
|
}
|
|
645
|
+
/**
|
|
646
|
+
* Build-scoped memo for `readProjectTopics()`.
|
|
647
|
+
*
|
|
648
|
+
* `readProjectTopics` derives adaptive topics by reading and tokenising the
|
|
649
|
+
* *whole* project corpus — CLAUDE.md, FINDINGS.md and every reference/*.md —
|
|
650
|
+
* on each call (see `buildTopicContentSignal` in project-topics.ts). It is
|
|
651
|
+
* called once per reference document, so a project with R reference docs read
|
|
652
|
+
* and tokenised its own corpus R times per rebuild. Measured on a 1892-file
|
|
653
|
+
* store (46 projects x 34 reference docs): 60,180 markdown reads for 1892
|
|
654
|
+
* files, ~85% of a 9.4s cold build spent in topic re-derivation.
|
|
655
|
+
*
|
|
656
|
+
* The answer is per-project, not per-document, and the corpus cannot change
|
|
657
|
+
* mid-build, so one derivation per project per build is exactly equivalent.
|
|
658
|
+
* The memo is armed only for the duration of `buildIndexImpl`; single-file
|
|
659
|
+
* callers such as `updateFileInIndex` keep reading through.
|
|
660
|
+
*/
|
|
661
|
+
const _buildTopicCache = new Map();
|
|
662
|
+
let _buildTopicCacheActive = false;
|
|
663
|
+
function beginTopicBuildCache() {
|
|
664
|
+
_buildTopicCacheActive = true;
|
|
665
|
+
_buildTopicCache.clear();
|
|
666
|
+
}
|
|
667
|
+
function endTopicBuildCache() {
|
|
668
|
+
_buildTopicCacheActive = false;
|
|
669
|
+
_buildTopicCache.clear();
|
|
670
|
+
}
|
|
671
|
+
function readProjectTopicsForBuild(phrenPath, project) {
|
|
672
|
+
if (!_buildTopicCacheActive)
|
|
673
|
+
return readProjectTopics(phrenPath, project);
|
|
674
|
+
const key = `${phrenPath}${project}`;
|
|
675
|
+
const hit = _buildTopicCache.get(key);
|
|
676
|
+
if (hit)
|
|
677
|
+
return hit;
|
|
678
|
+
const resolved = readProjectTopics(phrenPath, project);
|
|
679
|
+
_buildTopicCache.set(key, resolved);
|
|
680
|
+
return resolved;
|
|
681
|
+
}
|
|
599
682
|
function detectReferenceTopics(entry, content, phrenPath) {
|
|
600
683
|
if (entry.type !== "reference")
|
|
601
684
|
return [];
|
|
602
|
-
const { topics } =
|
|
685
|
+
const { topics } = readProjectTopicsForBuild(phrenPath, entry.project);
|
|
603
686
|
if (!topics.length)
|
|
604
687
|
return [];
|
|
605
688
|
const topicBySlug = new Map(topics.map((topic) => [topic.slug, topic]));
|
|
@@ -694,12 +777,13 @@ export function updateFileInIndex(db, filePath, phrenPath) {
|
|
|
694
777
|
const relFile = rel.split(path.sep).slice(1).join(path.sep);
|
|
695
778
|
const type = classifyFile(filename, relFile);
|
|
696
779
|
const entry = { fullPath: resolvedPath, project, filename, type, relFile };
|
|
697
|
-
|
|
780
|
+
// Single read feeds the insert, fragment extraction and the content hash.
|
|
781
|
+
const raw = readFileOrNull(resolvedPath);
|
|
782
|
+
if (raw !== null && insertFileIntoIndex(db, entry, phrenPath, { scheduleEmbeddings: true, content: raw })) {
|
|
698
783
|
// Re-extract fragments for finding files
|
|
699
784
|
if (type === "findings") {
|
|
700
785
|
try {
|
|
701
|
-
|
|
702
|
-
extractAndLinkFragments(db, content, getEntrySourceDocKey(entry, phrenPath), phrenPath);
|
|
786
|
+
extractAndLinkFragments(db, raw, getEntrySourceDocKey(entry, phrenPath), phrenPath);
|
|
703
787
|
}
|
|
704
788
|
catch (err) {
|
|
705
789
|
logger.debug("updateFileInIndex entityExtraction", errorMessage(err));
|
|
@@ -709,7 +793,7 @@ export function updateFileInIndex(db, filePath, phrenPath) {
|
|
|
709
793
|
// Update hash map for this file
|
|
710
794
|
try {
|
|
711
795
|
const hashData = loadHashMap(phrenPath);
|
|
712
|
-
hashData.hashes[resolvedPath] = hashFileContent(resolvedPath);
|
|
796
|
+
hashData.hashes[resolvedPath] = raw !== null ? hashContent(raw) : hashFileContent(resolvedPath);
|
|
713
797
|
const stat = fs.statSync(resolvedPath);
|
|
714
798
|
saveHashMap(phrenPath, hashData.hashes, undefined, { [resolvedPath]: { mtimeMs: stat.mtimeMs, size: stat.size } });
|
|
715
799
|
}
|
|
@@ -731,59 +815,217 @@ export function updateFileInIndex(db, filePath, phrenPath) {
|
|
|
731
815
|
}
|
|
732
816
|
})();
|
|
733
817
|
}
|
|
734
|
-
touchSentinel(phrenPath);
|
|
735
818
|
invalidateDfCache();
|
|
736
819
|
}
|
|
737
|
-
|
|
738
|
-
|
|
820
|
+
// ── Index freshness sentinel ────────────────────────────────────────────────
|
|
821
|
+
//
|
|
822
|
+
// The sentinel exists to skip `globAllFiles()` — the single most expensive part
|
|
823
|
+
// of a warm `buildIndex` (≈14ms on a 400-file store, ≈46ms on a 1900-file one,
|
|
824
|
+
// versus ≈0.7ms/4.3ms to re-stat the same files).
|
|
825
|
+
//
|
|
826
|
+
// It works in two steps, and BOTH are required for correctness:
|
|
827
|
+
//
|
|
828
|
+
// 1. Directory-mtime scan. POSIX bumps a directory's mtime whenever an entry
|
|
829
|
+
// inside it is created, removed or renamed, so if every directory in the
|
|
830
|
+
// indexed tree still has the exact mtime it had at build time, the *set*
|
|
831
|
+
// of indexed files provably has not changed and the glob can be skipped.
|
|
832
|
+
// 2. Re-hash the recorded file list. A directory's mtime does NOT change when
|
|
833
|
+
// a file inside it is edited in place, so step 1 alone would happily serve
|
|
834
|
+
// a stale index. `computePhrenHash()` over the recorded paths re-stats
|
|
835
|
+
// every file and catches edits — the same hash the slow path computes.
|
|
836
|
+
//
|
|
837
|
+
// `.runtime` is deliberately excluded from step 1 (see SENTINEL_SKIP_DIRS).
|
|
838
|
+
/**
|
|
839
|
+
* Directories that never hold indexed content and must stay out of the mtime
|
|
840
|
+
* scan.
|
|
841
|
+
*
|
|
842
|
+
* `.runtime` is the load-bearing entry. It is phren's own scratch dir, and
|
|
843
|
+
* `_buildIndexGuarded()` creates `.runtime/index-rebuild.lock` on entry and
|
|
844
|
+
* unlinks it on exit — i.e. *after* the sentinel is written. Any scan that
|
|
845
|
+
* includes `.runtime` therefore sees a directory that is unconditionally newer
|
|
846
|
+
* than the sentinel it is being compared against, on every single call. That is
|
|
847
|
+
* why the fast path had never once fired. `updateRuntimeHealth()` and the debug
|
|
848
|
+
* log dirty it too, but the rebuild lock alone is enough to guarantee a miss.
|
|
849
|
+
*/
|
|
850
|
+
const SENTINEL_SKIP_DIRS = new Set([
|
|
851
|
+
".runtime",
|
|
852
|
+
".sessions",
|
|
853
|
+
".git",
|
|
854
|
+
"node_modules",
|
|
855
|
+
"dist",
|
|
856
|
+
"build",
|
|
857
|
+
]);
|
|
858
|
+
const INDEX_SENTINEL_VERSION = 3;
|
|
859
|
+
function indexSentinelPath(phrenPath) {
|
|
860
|
+
return runtimeFile(phrenPath, "index-sentinel.json");
|
|
861
|
+
}
|
|
862
|
+
function statIndexPolicy(phrenPath) {
|
|
863
|
+
try {
|
|
864
|
+
const stat = fs.statSync(path.join(phrenPath, ".config", "index-policy.json"));
|
|
865
|
+
return [stat.mtimeMs, stat.size];
|
|
866
|
+
}
|
|
867
|
+
catch {
|
|
868
|
+
return [-1, -1];
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
/**
|
|
872
|
+
* Every directory whose entry list can change which files end up in the index:
|
|
873
|
+
* the phren root, its config/profiles dirs, each store root, the full tree
|
|
874
|
+
* under every project dir, the global skills tree, and the native agent memory
|
|
875
|
+
* dirs. A dir-only walk — no per-file stats, no glob pattern matching.
|
|
876
|
+
*/
|
|
877
|
+
function collectContentDirs(phrenPath, projectDirs) {
|
|
878
|
+
const out = new Set([
|
|
879
|
+
phrenPath,
|
|
880
|
+
path.join(phrenPath, ".config"),
|
|
881
|
+
path.join(phrenPath, "profiles"),
|
|
882
|
+
]);
|
|
883
|
+
const walk = (dir) => {
|
|
884
|
+
out.add(dir);
|
|
885
|
+
let entries;
|
|
886
|
+
try {
|
|
887
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
888
|
+
}
|
|
889
|
+
catch {
|
|
890
|
+
return;
|
|
891
|
+
}
|
|
892
|
+
for (const entry of entries) {
|
|
893
|
+
if (!entry.isDirectory() || SENTINEL_SKIP_DIRS.has(entry.name))
|
|
894
|
+
continue;
|
|
895
|
+
walk(path.join(dir, entry.name));
|
|
896
|
+
}
|
|
897
|
+
};
|
|
898
|
+
for (const dir of projectDirs) {
|
|
899
|
+
out.add(path.dirname(dir)); // store root
|
|
900
|
+
walk(dir);
|
|
901
|
+
}
|
|
902
|
+
walk(path.join(phrenPath, "global", "skills"));
|
|
903
|
+
// Native agent memory (~/.claude/projects/*/memory) is indexed too, but the
|
|
904
|
+
// surrounding project dirs are huge — watch only the two levels that matter.
|
|
905
|
+
const nativeRoot = path.join(homeDir(), ".claude", "projects");
|
|
906
|
+
out.add(nativeRoot);
|
|
907
|
+
try {
|
|
908
|
+
for (const entry of fs.readdirSync(nativeRoot, { withFileTypes: true })) {
|
|
909
|
+
if (entry.isDirectory())
|
|
910
|
+
out.add(path.join(nativeRoot, entry.name, "memory"));
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
catch {
|
|
914
|
+
// no native memory dirs — the nativeRoot entry above still catches creation
|
|
915
|
+
}
|
|
916
|
+
return [...out];
|
|
917
|
+
}
|
|
918
|
+
function snapshotContentDirs(phrenPath, projectDirs) {
|
|
919
|
+
const snapshot = {};
|
|
920
|
+
for (const dir of collectContentDirs(phrenPath, projectDirs)) {
|
|
921
|
+
try {
|
|
922
|
+
snapshot[dir] = fs.statSync(dir).mtimeMs;
|
|
923
|
+
}
|
|
924
|
+
catch {
|
|
925
|
+
snapshot[dir] = -1; // absent now; reappearing is itself a change
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
return snapshot;
|
|
929
|
+
}
|
|
930
|
+
function readIndexSentinel(phrenPath) {
|
|
739
931
|
try {
|
|
740
|
-
const sentinelPath =
|
|
932
|
+
const sentinelPath = indexSentinelPath(phrenPath);
|
|
741
933
|
if (!fs.existsSync(sentinelPath))
|
|
742
934
|
return null;
|
|
743
935
|
const data = JSON.parse(fs.readFileSync(sentinelPath, "utf-8"));
|
|
744
|
-
if (
|
|
745
|
-
|
|
936
|
+
if (data.version === INDEX_SENTINEL_VERSION &&
|
|
937
|
+
typeof data.hash === "string" &&
|
|
938
|
+
typeof data.computedAt === "number" &&
|
|
939
|
+
typeof data.profile === "string" &&
|
|
940
|
+
Array.isArray(data.policy) &&
|
|
941
|
+
Array.isArray(data.projectDirs) &&
|
|
942
|
+
Array.isArray(data.files) &&
|
|
943
|
+
data.dirs && typeof data.dirs === "object") {
|
|
944
|
+
return data;
|
|
746
945
|
}
|
|
747
946
|
}
|
|
748
947
|
catch (err) {
|
|
749
|
-
logger.debug("
|
|
948
|
+
logger.debug("readIndexSentinel", errorMessage(err));
|
|
750
949
|
}
|
|
751
950
|
return null;
|
|
752
951
|
}
|
|
753
|
-
function
|
|
952
|
+
function writeIndexSentinel(phrenPath, entry) {
|
|
754
953
|
try {
|
|
755
|
-
const
|
|
756
|
-
|
|
954
|
+
const sentinel = {
|
|
955
|
+
version: INDEX_SENTINEL_VERSION,
|
|
956
|
+
hash: entry.hash,
|
|
957
|
+
computedAt: Date.now(),
|
|
958
|
+
profile: entry.profile ?? "",
|
|
959
|
+
policy: statIndexPolicy(phrenPath),
|
|
960
|
+
projectDirs: entry.projectDirs,
|
|
961
|
+
files: entry.files,
|
|
962
|
+
dirs: entry.dirs,
|
|
963
|
+
};
|
|
964
|
+
fs.writeFileSync(indexSentinelPath(phrenPath), JSON.stringify(sentinel));
|
|
757
965
|
}
|
|
758
966
|
catch (err) {
|
|
759
|
-
logger.debug("
|
|
967
|
+
logger.debug("writeIndexSentinel", errorMessage(err));
|
|
760
968
|
}
|
|
761
969
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
970
|
+
/**
|
|
971
|
+
* True when nothing that feeds the index hash can have changed since the
|
|
972
|
+
* sentinel was written. Deliberately conservative: any unreadable directory,
|
|
973
|
+
* any mtime that differs *at all* (not just "newer"), a profile or project-set
|
|
974
|
+
* change, or an index-policy edit all report stale.
|
|
975
|
+
*/
|
|
976
|
+
function isSentinelFresh(phrenPath, sentinel, profile, projectDirs) {
|
|
977
|
+
if (sentinel.profile !== (profile ?? ""))
|
|
978
|
+
return false;
|
|
979
|
+
const [policyMtime, policySize] = statIndexPolicy(phrenPath);
|
|
980
|
+
if (sentinel.policy[0] !== policyMtime || sentinel.policy[1] !== policySize)
|
|
981
|
+
return false;
|
|
982
|
+
if (sentinel.projectDirs.length !== projectDirs.length)
|
|
983
|
+
return false;
|
|
984
|
+
const currentDirs = new Set(projectDirs);
|
|
985
|
+
for (const dir of sentinel.projectDirs) {
|
|
986
|
+
if (!currentDirs.has(dir))
|
|
987
|
+
return false;
|
|
774
988
|
}
|
|
775
|
-
for (const dir of
|
|
989
|
+
for (const [dir, mtimeMs] of Object.entries(sentinel.dirs)) {
|
|
990
|
+
let current;
|
|
776
991
|
try {
|
|
777
|
-
|
|
778
|
-
if (stat.mtimeMs > sentinel.computedAt)
|
|
779
|
-
return false;
|
|
992
|
+
current = fs.statSync(dir).mtimeMs;
|
|
780
993
|
}
|
|
781
|
-
catch
|
|
782
|
-
|
|
994
|
+
catch {
|
|
995
|
+
current = -1;
|
|
783
996
|
}
|
|
997
|
+
if (current !== mtimeMs)
|
|
998
|
+
return false;
|
|
784
999
|
}
|
|
785
1000
|
return true;
|
|
786
1001
|
}
|
|
1002
|
+
/**
|
|
1003
|
+
* The slow path: snapshot directory mtimes, glob, hash, and seal a new sentinel.
|
|
1004
|
+
*
|
|
1005
|
+
* The snapshot is taken *before* the glob on purpose. Anything written during or
|
|
1006
|
+
* after the glob leaves the recorded mtime behind the real one, so the next
|
|
1007
|
+
* freshness check reports stale — the conservative direction.
|
|
1008
|
+
*/
|
|
1009
|
+
function globAndSealSentinel(phrenPath, profile, projectDirs) {
|
|
1010
|
+
const dirs = snapshotContentDirs(phrenPath, projectDirs);
|
|
1011
|
+
const globResult = globAllFiles(phrenPath, profile);
|
|
1012
|
+
const hash = computePhrenHash(phrenPath, profile, globResult.filePaths);
|
|
1013
|
+
writeIndexSentinel(phrenPath, { hash, profile, projectDirs, files: globResult.filePaths, dirs });
|
|
1014
|
+
return { globResult, hash };
|
|
1015
|
+
}
|
|
1016
|
+
/**
|
|
1017
|
+
* Resolve the current index hash without globbing when the sentinel proves the
|
|
1018
|
+
* file set is unchanged. Returns null when the caller must fall back to a glob.
|
|
1019
|
+
*/
|
|
1020
|
+
function hashFromFreshSentinel(phrenPath, profile, projectDirs) {
|
|
1021
|
+
const sentinel = readIndexSentinel(phrenPath);
|
|
1022
|
+
if (!sentinel || !isSentinelFresh(phrenPath, sentinel, profile, projectDirs))
|
|
1023
|
+
return null;
|
|
1024
|
+
// File set is proven unchanged; re-stat those files so an in-place edit
|
|
1025
|
+
// (which leaves every directory mtime untouched) still busts the cache.
|
|
1026
|
+
const hash = computePhrenHash(phrenPath, profile, sentinel.files);
|
|
1027
|
+
return hash === sentinel.hash ? hash : null;
|
|
1028
|
+
}
|
|
787
1029
|
/**
|
|
788
1030
|
* Attempt to restore the fragment graph (entities, entity_links, global_entities) from a
|
|
789
1031
|
* previously persisted JSON snapshot. Returns true if the graph was loaded, false if the
|
|
@@ -945,39 +1187,25 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
945
1187
|
await refreshStoreProjectDirs(phrenPath, profile);
|
|
946
1188
|
const projectDirs = getAllStoreProjectDirs(phrenPath, profile);
|
|
947
1189
|
beginUserFragmentBuildCache(phrenPath, projectDirs.map(dir => path.basename(dir)));
|
|
1190
|
+
beginTopicBuildCache();
|
|
1191
|
+
const contentCache = new BuildContentCache();
|
|
948
1192
|
try {
|
|
949
1193
|
// ── Cache dir + hash sentinel ─────────────────────────────────────────────
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
}
|
|
954
|
-
catch (err) {
|
|
955
|
-
logger.debug("buildIndexImpl userInfo", errorMessage(err));
|
|
956
|
-
userSuffix = crypto.createHash("sha1").update(homeDir()).digest("hex").slice(0, 12);
|
|
957
|
-
}
|
|
958
|
-
const cacheDir = path.join(os.tmpdir(), `phren-fts-${userSuffix}`);
|
|
959
|
-
// Fast path: if the sentinel is fresh, skip the expensive glob + hash computation
|
|
960
|
-
const sentinel = readHashSentinel(phrenPath);
|
|
1194
|
+
pruneFtsCacheRoot(storeCacheKey(phrenPath, profile));
|
|
1195
|
+
const cacheDir = ftsCacheDir(phrenPath, profile);
|
|
1196
|
+
// Fast path: if the sentinel is fresh, skip the expensive glob computation.
|
|
961
1197
|
let hash;
|
|
962
1198
|
let globResult = null;
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
// Sentinel cache hit — defer glob; only load it if incremental path needs it
|
|
968
|
-
globResult = null;
|
|
969
|
-
}
|
|
970
|
-
else {
|
|
971
|
-
// Cache file was cleaned up, fall through to full computation
|
|
972
|
-
globResult = globAllFiles(phrenPath, profile);
|
|
973
|
-
hash = computePhrenHash(phrenPath, profile, globResult.filePaths);
|
|
974
|
-
writeHashSentinel(phrenPath, hash);
|
|
975
|
-
}
|
|
1199
|
+
const sentinelHash = hashFromFreshSentinel(phrenPath, profile, projectDirs);
|
|
1200
|
+
if (sentinelHash && fs.existsSync(path.join(cacheDir, `${sentinelHash}.db`))) {
|
|
1201
|
+
// Sentinel cache hit — defer glob; only load it if incremental path needs it
|
|
1202
|
+
hash = sentinelHash;
|
|
976
1203
|
}
|
|
977
1204
|
else {
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
1205
|
+
// Sentinel stale, content changed, or the cache file was cleaned up.
|
|
1206
|
+
const sealed = globAndSealSentinel(phrenPath, profile, projectDirs);
|
|
1207
|
+
globResult = sealed.globResult;
|
|
1208
|
+
hash = sealed.hash;
|
|
981
1209
|
}
|
|
982
1210
|
const cacheFile = path.join(cacheDir, `${hash}.db`);
|
|
983
1211
|
const SQL = await bootstrapSqlJs();
|
|
@@ -1008,6 +1236,9 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1008
1236
|
appendIndexEvent(phrenPath, {
|
|
1009
1237
|
event: "build_index",
|
|
1010
1238
|
cache: "hit",
|
|
1239
|
+
// The glob was skipped entirely. Recorded so "is the fast path
|
|
1240
|
+
// firing?" is answerable from the event log instead of by reasoning.
|
|
1241
|
+
sentinel: true,
|
|
1011
1242
|
hash: hash.slice(0, 12),
|
|
1012
1243
|
elapsedMs: Date.now() - t0,
|
|
1013
1244
|
profile: profile || "",
|
|
@@ -1021,9 +1252,9 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1021
1252
|
debugLog(`sentinel-hit DB load failed, falling back to full rebuild: ${errorMessage(err)}`);
|
|
1022
1253
|
}
|
|
1023
1254
|
// Need glob for rebuild
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1255
|
+
const sealed = globAndSealSentinel(phrenPath, profile, projectDirs);
|
|
1256
|
+
globResult = sealed.globResult;
|
|
1257
|
+
hash = sealed.hash;
|
|
1027
1258
|
}
|
|
1028
1259
|
try {
|
|
1029
1260
|
const cached = fs.readFileSync(cacheFile);
|
|
@@ -1065,7 +1296,17 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1065
1296
|
const prevMeta = savedMeta[entry.fullPath];
|
|
1066
1297
|
const statUnchanged = prevHash !== undefined && prevMeta !== undefined
|
|
1067
1298
|
&& prevMeta.mtimeMs === stat.mtimeMs && prevMeta.size === stat.size;
|
|
1068
|
-
|
|
1299
|
+
let fileHash;
|
|
1300
|
+
if (statUnchanged) {
|
|
1301
|
+
fileHash = prevHash;
|
|
1302
|
+
}
|
|
1303
|
+
else {
|
|
1304
|
+
// The only read of this file in this rebuild: hand the bytes to the
|
|
1305
|
+
// insert pass below instead of letting it read them again.
|
|
1306
|
+
const raw = fs.readFileSync(entry.fullPath, "utf-8");
|
|
1307
|
+
fileHash = hashContent(raw);
|
|
1308
|
+
contentCache.put(entry.fullPath, raw);
|
|
1309
|
+
}
|
|
1069
1310
|
currentHashes[entry.fullPath] = fileHash;
|
|
1070
1311
|
currentMeta[entry.fullPath] = { mtimeMs: stat.mtimeMs, size: stat.size };
|
|
1071
1312
|
if (!(entry.fullPath in savedHashes)) {
|
|
@@ -1097,6 +1338,7 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1097
1338
|
appendIndexEvent(phrenPath, {
|
|
1098
1339
|
event: "build_index",
|
|
1099
1340
|
cache: "hit",
|
|
1341
|
+
sentinel: false,
|
|
1100
1342
|
hash: hash.slice(0, 12),
|
|
1101
1343
|
elapsedMs: Date.now() - t0,
|
|
1102
1344
|
profile: profile || "",
|
|
@@ -1152,12 +1394,15 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1152
1394
|
}
|
|
1153
1395
|
db.run("DELETE FROM docs WHERE path = ?", [entry.fullPath]);
|
|
1154
1396
|
}
|
|
1155
|
-
|
|
1397
|
+
// Reuse the bytes read during hashing; only fall back to disk when
|
|
1398
|
+
// the carry cache declined the file (budget) or never saw it.
|
|
1399
|
+
const carried = contentCache.take(entry.fullPath);
|
|
1400
|
+
const raw = carried ?? readFileOrNull(entry.fullPath);
|
|
1401
|
+
if (raw !== null && insertFileIntoIndex(db, entry, phrenPath, { scheduleEmbeddings: true, content: raw })) {
|
|
1156
1402
|
updatedCount++;
|
|
1157
1403
|
if (entry.type === "findings") {
|
|
1158
1404
|
try {
|
|
1159
|
-
|
|
1160
|
-
extractAndLinkFragments(db, content, getEntrySourceDocKey(entry, phrenPath), phrenPath);
|
|
1405
|
+
extractAndLinkFragments(db, raw, getEntrySourceDocKey(entry, phrenPath), phrenPath);
|
|
1161
1406
|
}
|
|
1162
1407
|
catch (err) {
|
|
1163
1408
|
debugLog(`fragment extraction failed: ${errorMessage(err)}`);
|
|
@@ -1177,7 +1422,6 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1177
1422
|
}
|
|
1178
1423
|
}
|
|
1179
1424
|
saveHashMap(phrenPath, currentHashes, new Set(Object.keys(currentHashes)), currentMeta);
|
|
1180
|
-
touchSentinel(phrenPath);
|
|
1181
1425
|
invalidateDfCache();
|
|
1182
1426
|
// Save updated cache
|
|
1183
1427
|
try {
|
|
@@ -1215,9 +1459,9 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1215
1459
|
// ── Full rebuild ──────────────────────────────────────────────────────────
|
|
1216
1460
|
// Ensure glob data is available for full rebuild (may be null from sentinel fast-path fallback)
|
|
1217
1461
|
if (!globResult) {
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1462
|
+
const sealed = globAndSealSentinel(phrenPath, profile, projectDirs);
|
|
1463
|
+
globResult = sealed.globResult;
|
|
1464
|
+
hash = sealed.hash;
|
|
1221
1465
|
}
|
|
1222
1466
|
const db = new SQL.Database();
|
|
1223
1467
|
db.run(`
|
|
@@ -1238,28 +1482,49 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1238
1482
|
// Try loading cached fragment graph
|
|
1239
1483
|
const graphPath = runtimeFile(phrenPath, 'entity-graph.json');
|
|
1240
1484
|
const entityGraphLoaded = loadCachedEntityGraph(db, graphPath, allFiles, phrenPath);
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1485
|
+
// One read per file: the bytes feed the content hash, the FTS insert and (for
|
|
1486
|
+
// findings) fragment extraction. Every insert is batched into a single
|
|
1487
|
+
// transaction so sql.js does not open and commit one per statement.
|
|
1488
|
+
db.run("BEGIN");
|
|
1489
|
+
let fullRebuildCommitted = false;
|
|
1490
|
+
try {
|
|
1491
|
+
for (const entry of allFiles) {
|
|
1492
|
+
const raw = readFileOrNull(entry.fullPath);
|
|
1493
|
+
if (raw === null)
|
|
1494
|
+
continue;
|
|
1495
|
+
try {
|
|
1496
|
+
newHashes[entry.fullPath] = hashContent(raw);
|
|
1497
|
+
const stat = fs.statSync(entry.fullPath);
|
|
1498
|
+
newMeta[entry.fullPath] = { mtimeMs: stat.mtimeMs, size: stat.size };
|
|
1499
|
+
}
|
|
1500
|
+
catch (err) {
|
|
1501
|
+
logger.debug("buildIndex statFile", errorMessage(err));
|
|
1502
|
+
}
|
|
1503
|
+
if (insertFileIntoIndex(db, entry, phrenPath, { scheduleEmbeddings: true, content: raw })) {
|
|
1504
|
+
fileCount++;
|
|
1505
|
+
// Extract fragments from finding files (if not loaded from cache)
|
|
1506
|
+
if (!entityGraphLoaded && entry.type === "findings") {
|
|
1507
|
+
try {
|
|
1508
|
+
extractAndLinkFragments(db, raw, getEntrySourceDocKey(entry, phrenPath), phrenPath);
|
|
1509
|
+
}
|
|
1510
|
+
catch (err) {
|
|
1511
|
+
debugLog(`fragment extraction failed: ${errorMessage(err)}`);
|
|
1512
|
+
}
|
|
1260
1513
|
}
|
|
1261
1514
|
}
|
|
1262
1515
|
}
|
|
1516
|
+
db.run("COMMIT");
|
|
1517
|
+
fullRebuildCommitted = true;
|
|
1518
|
+
}
|
|
1519
|
+
finally {
|
|
1520
|
+
if (!fullRebuildCommitted) {
|
|
1521
|
+
try {
|
|
1522
|
+
db.run("ROLLBACK");
|
|
1523
|
+
}
|
|
1524
|
+
catch (err) {
|
|
1525
|
+
logger.debug("buildIndex fullRebuildRollback", errorMessage(err));
|
|
1526
|
+
}
|
|
1527
|
+
}
|
|
1263
1528
|
}
|
|
1264
1529
|
// Persist fragment graph for next build
|
|
1265
1530
|
if (!entityGraphLoaded) {
|
|
@@ -1279,7 +1544,6 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1279
1544
|
mergeManualLinks(db, phrenPath);
|
|
1280
1545
|
// ── Finalize: persist hashes, save cache, log ─────────────────────────────
|
|
1281
1546
|
saveHashMap(phrenPath, newHashes, new Set(Object.keys(newHashes)), newMeta);
|
|
1282
|
-
touchSentinel(phrenPath);
|
|
1283
1547
|
invalidateDfCache();
|
|
1284
1548
|
const buildMs = Date.now() - t0;
|
|
1285
1549
|
debugLog(`Built FTS index: ${fileCount} files from ${projectDirs.length} projects in ${buildMs}ms`);
|
|
@@ -1316,6 +1580,8 @@ async function buildIndexImpl(phrenPath, profile) {
|
|
|
1316
1580
|
}
|
|
1317
1581
|
finally {
|
|
1318
1582
|
endUserFragmentBuildCache(phrenPath);
|
|
1583
|
+
endTopicBuildCache();
|
|
1584
|
+
contentCache.clear();
|
|
1319
1585
|
}
|
|
1320
1586
|
}
|
|
1321
1587
|
function createEmptyIndexDb(SQL) {
|
|
@@ -1344,19 +1610,12 @@ function isRebuildLockHeld(phrenPath) {
|
|
|
1344
1610
|
return false;
|
|
1345
1611
|
}
|
|
1346
1612
|
}
|
|
1347
|
-
async function loadIndexSnapshotOrEmpty(phrenPath, profile) {
|
|
1613
|
+
async function loadIndexSnapshotOrEmpty(phrenPath, profile, knownHash) {
|
|
1348
1614
|
const SQL = await bootstrapSqlJs();
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
catch (err) {
|
|
1354
|
-
logger.debug("loadIndexSnapshotOrEmpty userInfo", errorMessage(err));
|
|
1355
|
-
userSuffix = crypto.createHash("sha1").update(homeDir()).digest("hex").slice(0, 12);
|
|
1356
|
-
}
|
|
1357
|
-
const cacheDir = path.join(os.tmpdir(), `phren-fts-${userSuffix}`);
|
|
1358
|
-
const globResult = globAllFiles(phrenPath, profile);
|
|
1359
|
-
const hash = computePhrenHash(phrenPath, profile, globResult.filePaths);
|
|
1615
|
+
const cacheDir = ftsCacheDir(phrenPath, profile);
|
|
1616
|
+
// `knownHash` lets loadIndexForHook skip a second glob for a hash it just
|
|
1617
|
+
// computed (and already knows misses).
|
|
1618
|
+
const hash = knownHash ?? computePhrenHash(phrenPath, profile, globAllFiles(phrenPath, profile).filePaths);
|
|
1360
1619
|
const cacheFile = path.join(cacheDir, `${hash}.db`);
|
|
1361
1620
|
if (fs.existsSync(cacheFile)) {
|
|
1362
1621
|
try {
|
|
@@ -1366,7 +1625,10 @@ async function loadIndexSnapshotOrEmpty(phrenPath, profile) {
|
|
|
1366
1625
|
debugLog(`Failed to open cached FTS snapshot while rebuild lock held: ${errorMessage(err)}`);
|
|
1367
1626
|
}
|
|
1368
1627
|
}
|
|
1369
|
-
// Before returning empty,
|
|
1628
|
+
// Before returning empty, fall back to an older snapshot *of this store*.
|
|
1629
|
+
// cacheDir is store+profile scoped, so nothing here can belong to another
|
|
1630
|
+
// store — serving a neighbour's index would inject its knowledge into this
|
|
1631
|
+
// store's prompts.
|
|
1370
1632
|
try {
|
|
1371
1633
|
if (fs.existsSync(cacheDir)) {
|
|
1372
1634
|
const cacheFiles = fs.readdirSync(cacheDir)
|
|
@@ -1436,18 +1698,121 @@ export async function buildIndex(phrenPath, profile) {
|
|
|
1436
1698
|
_lastBuildKey = buildKey;
|
|
1437
1699
|
return db;
|
|
1438
1700
|
}
|
|
1439
|
-
/**
|
|
1440
|
-
function
|
|
1701
|
+
/** Per-user FTS cache root (os.tmpdir()/phren-fts-<uid>). */
|
|
1702
|
+
function ftsCacheRoot() {
|
|
1441
1703
|
let userSuffix;
|
|
1442
1704
|
try {
|
|
1443
1705
|
userSuffix = String(os.userInfo().uid);
|
|
1444
1706
|
}
|
|
1445
1707
|
catch (err) {
|
|
1446
|
-
logger.debug("
|
|
1708
|
+
logger.debug("ftsCacheRoot userInfo", errorMessage(err));
|
|
1447
1709
|
userSuffix = crypto.createHash("sha1").update(homeDir()).digest("hex").slice(0, 12);
|
|
1448
1710
|
}
|
|
1449
1711
|
return path.join(os.tmpdir(), `phren-fts-${userSuffix}`);
|
|
1450
1712
|
}
|
|
1713
|
+
/**
|
|
1714
|
+
* Store identity, independent of content. A snapshot's filename is a *content*
|
|
1715
|
+
* hash, so an older snapshot of this store has a different name than the
|
|
1716
|
+
* current one — there is no way to recognise your own stale caches by hash.
|
|
1717
|
+
* Identity therefore has to live in the path.
|
|
1718
|
+
*/
|
|
1719
|
+
function storeCacheKey(phrenPath, profile) {
|
|
1720
|
+
return crypto.createHash("sha1")
|
|
1721
|
+
.update(`${path.resolve(phrenPath)}|${profile ?? ""}`)
|
|
1722
|
+
.digest("hex")
|
|
1723
|
+
.slice(0, 16);
|
|
1724
|
+
}
|
|
1725
|
+
/**
|
|
1726
|
+
* FTS cache directory for one store+profile: os.tmpdir()/phren-fts-<uid>/<storeKey>/.
|
|
1727
|
+
*
|
|
1728
|
+
* The per-user root used to hold every store's `<contentHash>.db` side by side,
|
|
1729
|
+
* and the "serve a stale snapshot" fallback picked the newest .db in it without
|
|
1730
|
+
* checking whose it was. With two stores configured — phren's own documented
|
|
1731
|
+
* personal + team setup — one store's knowledge could be injected into the
|
|
1732
|
+
* other's agent prompt whenever the second store's hash missed. Scoping the
|
|
1733
|
+
* directory makes every scan trivially store-local. It also keeps the
|
|
1734
|
+
* "prune older snapshots" sweep from evicting a sibling store's cache and
|
|
1735
|
+
* forcing it into a full rebuild on its next build.
|
|
1736
|
+
*/
|
|
1737
|
+
function ftsCacheDir(phrenPath, profile) {
|
|
1738
|
+
return path.join(ftsCacheRoot(), storeCacheKey(phrenPath, profile));
|
|
1739
|
+
}
|
|
1740
|
+
/** Store cache dirs older than this are dropped; a cold rebuild recreates them. */
|
|
1741
|
+
const FTS_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
1742
|
+
/** Hard cap on retained store dirs, so a burst of transient stores cannot pile up. */
|
|
1743
|
+
const FTS_CACHE_MAX_STORES = 64;
|
|
1744
|
+
/**
|
|
1745
|
+
* Keep the cache root bounded, once per process.
|
|
1746
|
+
*
|
|
1747
|
+
* The old flat layout self-limited to a single snapshot because every full
|
|
1748
|
+
* rebuild unlinked all the others — cross-store eviction was the (accidental)
|
|
1749
|
+
* garbage collector. Per-store directories fix the contamination that caused,
|
|
1750
|
+
* but nothing would ever reclaim a directory again: one test-suite run alone
|
|
1751
|
+
* left 524 store dirs / 36MB behind. So prune explicitly.
|
|
1752
|
+
*
|
|
1753
|
+
* Also removes pre-0.1.41 snapshots sitting flat in the root: they are not
|
|
1754
|
+
* attributable to any store, so they can only ever be mis-served.
|
|
1755
|
+
*/
|
|
1756
|
+
let _ftsCachePruned = false;
|
|
1757
|
+
/**
|
|
1758
|
+
* @internal Exported for tests. The prune is a one-shot per process, so a test
|
|
1759
|
+
* that needs to set up state *after* a build (which consumes it) has no other
|
|
1760
|
+
* way to exercise the pruning logic a second time.
|
|
1761
|
+
*/
|
|
1762
|
+
export function __resetFtsCachePruneGuardForTests() {
|
|
1763
|
+
_ftsCachePruned = false;
|
|
1764
|
+
}
|
|
1765
|
+
function pruneFtsCacheRoot(keepKey) {
|
|
1766
|
+
if (_ftsCachePruned)
|
|
1767
|
+
return;
|
|
1768
|
+
_ftsCachePruned = true;
|
|
1769
|
+
try {
|
|
1770
|
+
const root = ftsCacheRoot();
|
|
1771
|
+
if (!fs.existsSync(root))
|
|
1772
|
+
return;
|
|
1773
|
+
const now = Date.now();
|
|
1774
|
+
const stores = [];
|
|
1775
|
+
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
|
1776
|
+
const full = path.join(root, entry.name);
|
|
1777
|
+
if (entry.isFile()) {
|
|
1778
|
+
if (entry.name.endsWith(".db")) {
|
|
1779
|
+
try {
|
|
1780
|
+
fs.unlinkSync(full);
|
|
1781
|
+
}
|
|
1782
|
+
catch { /* best effort */ }
|
|
1783
|
+
}
|
|
1784
|
+
continue;
|
|
1785
|
+
}
|
|
1786
|
+
if (!entry.isDirectory() || entry.name === keepKey)
|
|
1787
|
+
continue;
|
|
1788
|
+
let mtimeMs = 0;
|
|
1789
|
+
try {
|
|
1790
|
+
mtimeMs = fs.statSync(full).mtimeMs;
|
|
1791
|
+
}
|
|
1792
|
+
catch { /* treat as ancient */ }
|
|
1793
|
+
if (now - mtimeMs > FTS_CACHE_TTL_MS) {
|
|
1794
|
+
try {
|
|
1795
|
+
fs.rmSync(full, { recursive: true, force: true });
|
|
1796
|
+
}
|
|
1797
|
+
catch { /* best effort */ }
|
|
1798
|
+
continue;
|
|
1799
|
+
}
|
|
1800
|
+
stores.push({ name: entry.name, mtimeMs });
|
|
1801
|
+
}
|
|
1802
|
+
if (stores.length <= FTS_CACHE_MAX_STORES)
|
|
1803
|
+
return;
|
|
1804
|
+
stores.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
1805
|
+
for (const store of stores.slice(FTS_CACHE_MAX_STORES)) {
|
|
1806
|
+
try {
|
|
1807
|
+
fs.rmSync(path.join(root, store.name), { recursive: true, force: true });
|
|
1808
|
+
}
|
|
1809
|
+
catch { /* best effort */ }
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
catch (err) {
|
|
1813
|
+
logger.debug("pruneFtsCacheRoot", errorMessage(err));
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1451
1816
|
/**
|
|
1452
1817
|
* Non-blocking index loader for the UserPromptSubmit / PostToolUse hooks.
|
|
1453
1818
|
* Unlike buildIndex(), this NEVER blocks a hook on a 2-12s rebuild:
|
|
@@ -1462,9 +1827,18 @@ function ftsCacheDir() {
|
|
|
1462
1827
|
* misses → a rebuild is scheduled → the following prompt hits the new cache.
|
|
1463
1828
|
*/
|
|
1464
1829
|
export async function loadIndexForHook(phrenPath, profile) {
|
|
1465
|
-
|
|
1466
|
-
const
|
|
1467
|
-
|
|
1830
|
+
pruneFtsCacheRoot(storeCacheKey(phrenPath, profile));
|
|
1831
|
+
const cacheDir = ftsCacheDir(phrenPath, profile);
|
|
1832
|
+
// Resolve team stores the same way buildIndex does. Without this the hook's
|
|
1833
|
+
// file set (and therefore its hash) can never match the one buildIndex sealed,
|
|
1834
|
+
// so every prompt would miss the cache and spawn a redundant reindex.
|
|
1835
|
+
await refreshStoreProjectDirs(phrenPath, profile);
|
|
1836
|
+
const projectDirs = getAllStoreProjectDirs(phrenPath, profile);
|
|
1837
|
+
// Skip the glob when the sentinel proves nothing changed (see the sentinel
|
|
1838
|
+
// block above): ~20x cheaper than re-globbing, and yields the same hash.
|
|
1839
|
+
const sentinelHash = hashFromFreshSentinel(phrenPath, profile, projectDirs);
|
|
1840
|
+
const hash = sentinelHash
|
|
1841
|
+
?? computePhrenHash(phrenPath, profile, globAllFiles(phrenPath, profile).filePaths);
|
|
1468
1842
|
const cacheFile = path.join(cacheDir, `${hash}.db`);
|
|
1469
1843
|
const SQL = await bootstrapSqlJs();
|
|
1470
1844
|
// Fast path: exact stat-hash hit (no file changed mtime/size since the build).
|
|
@@ -1473,7 +1847,7 @@ export async function loadIndexForHook(phrenPath, profile) {
|
|
|
1473
1847
|
const db = new SQL.Database(fs.readFileSync(cacheFile));
|
|
1474
1848
|
const docCount = db.exec("SELECT COUNT(*) FROM docs")?.[0]?.values?.[0]?.[0] ?? 0;
|
|
1475
1849
|
if (docCount > 0) {
|
|
1476
|
-
appendIndexEvent(phrenPath, { event: "build_index", cache: "hit", hash: hash.slice(0, 12), elapsedMs: 0, profile: profile || "" });
|
|
1850
|
+
appendIndexEvent(phrenPath, { event: "build_index", cache: "hit", sentinel: sentinelHash !== null, hash: hash.slice(0, 12), elapsedMs: 0, profile: profile || "" });
|
|
1477
1851
|
return db;
|
|
1478
1852
|
}
|
|
1479
1853
|
db.close();
|
|
@@ -1482,7 +1856,9 @@ export async function loadIndexForHook(phrenPath, profile) {
|
|
|
1482
1856
|
debugLog(`loadIndexForHook fast-path load failed: ${errorMessage(err)}`);
|
|
1483
1857
|
}
|
|
1484
1858
|
}
|
|
1485
|
-
// Miss. Serve stale immediately if
|
|
1859
|
+
// Miss. Serve stale immediately if *this store* has a snapshot; else
|
|
1860
|
+
// cold-build once. cacheDir is store+profile scoped, so another store's
|
|
1861
|
+
// snapshot can never satisfy this check.
|
|
1486
1862
|
let hasStale = false;
|
|
1487
1863
|
try {
|
|
1488
1864
|
hasStale = fs.existsSync(cacheDir) && fs.readdirSync(cacheDir).some(f => f.endsWith(".db"));
|
|
@@ -1491,14 +1867,18 @@ export async function loadIndexForHook(phrenPath, profile) {
|
|
|
1491
1867
|
logger.debug("loadIndexForHook staleScan", errorMessage(err));
|
|
1492
1868
|
}
|
|
1493
1869
|
if (!hasStale) {
|
|
1870
|
+
// Cold start: no snapshot exists at all. Block once rather than inject an
|
|
1871
|
+
// empty context — "instant but empty" is worse than "slow but right".
|
|
1494
1872
|
return buildIndex(phrenPath, profile);
|
|
1495
1873
|
}
|
|
1496
1874
|
// Schedule a detached rebuild for next time (rebuild lock dedups concurrent hooks).
|
|
1875
|
+
let scheduled = false;
|
|
1497
1876
|
if (!isRebuildLockHeld(phrenPath)) {
|
|
1498
1877
|
const entry = process.argv[1];
|
|
1499
1878
|
if (entry && /index\.(c?js|mjs|ts)$/.test(entry) && fs.existsSync(entry)) {
|
|
1500
1879
|
try {
|
|
1501
1880
|
spawnDetachedChild([entry, "background-reindex"], { phrenPath }).unref();
|
|
1881
|
+
scheduled = true;
|
|
1502
1882
|
debugLog(`loadIndexForHook: scheduled detached background-reindex (stat-hash miss ${hash.slice(0, 8)})`);
|
|
1503
1883
|
}
|
|
1504
1884
|
catch (err) {
|
|
@@ -1506,7 +1886,18 @@ export async function loadIndexForHook(phrenPath, profile) {
|
|
|
1506
1886
|
}
|
|
1507
1887
|
}
|
|
1508
1888
|
}
|
|
1509
|
-
|
|
1889
|
+
// The prompt is about to be answered from a snapshot that predates the latest
|
|
1890
|
+
// write. That trade is deliberate, but it must not be invisible: record it so
|
|
1891
|
+
// "why did phren miss the finding I just wrote?" is answerable from the log.
|
|
1892
|
+
appendIndexEvent(phrenPath, {
|
|
1893
|
+
event: "build_index",
|
|
1894
|
+
cache: "stale",
|
|
1895
|
+
hash: hash.slice(0, 12),
|
|
1896
|
+
rebuildScheduled: scheduled,
|
|
1897
|
+
elapsedMs: 0,
|
|
1898
|
+
profile: profile || "",
|
|
1899
|
+
});
|
|
1900
|
+
return loadIndexSnapshotOrEmpty(phrenPath, profile, hash);
|
|
1510
1901
|
}
|
|
1511
1902
|
async function _buildIndexGuarded(phrenPath, profile) {
|
|
1512
1903
|
const lockTarget = runtimeFile(phrenPath, "index-rebuild");
|
|
@@ -1538,15 +1929,7 @@ async function _buildIndexGuarded(phrenPath, profile) {
|
|
|
1538
1929
|
}
|
|
1539
1930
|
/** Find the FTS cache file for a specific phrenPath+profile. Returns exists + size. */
|
|
1540
1931
|
export function findFtsCacheForPath(phrenPath, profile) {
|
|
1541
|
-
|
|
1542
|
-
try {
|
|
1543
|
-
userSuffix = String(os.userInfo().uid);
|
|
1544
|
-
}
|
|
1545
|
-
catch (err) {
|
|
1546
|
-
logger.debug("findFtsCacheForPath userInfo", errorMessage(err));
|
|
1547
|
-
userSuffix = crypto.createHash("sha1").update(homeDir()).digest("hex").slice(0, 12);
|
|
1548
|
-
}
|
|
1549
|
-
const cacheDir = path.join(os.tmpdir(), `phren-fts-${userSuffix}`);
|
|
1932
|
+
const cacheDir = ftsCacheDir(phrenPath, profile);
|
|
1550
1933
|
try {
|
|
1551
1934
|
const globResult = globAllFiles(phrenPath, profile);
|
|
1552
1935
|
const hash = computePhrenHash(phrenPath, profile, globResult.filePaths);
|
|
@@ -1567,7 +1950,9 @@ export function detectProject(phrenPath, cwd, profile) {
|
|
|
1567
1950
|
return manifest.primaryProject || null;
|
|
1568
1951
|
}
|
|
1569
1952
|
const projectDirs = getAllStoreProjectDirs(phrenPath, profile);
|
|
1570
|
-
|
|
1953
|
+
// A session running inside a git worktree belongs to the repository the
|
|
1954
|
+
// worktree came from — its own path never matches any registered sourcePath.
|
|
1955
|
+
const resolvedCwd = resolveRepoRootForPath(cwd);
|
|
1571
1956
|
let bestMatch = null;
|
|
1572
1957
|
for (const dir of projectDirs) {
|
|
1573
1958
|
const projectName = path.basename(dir);
|