@drewpayment/mink 0.11.0-beta.4 → 0.12.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/out/404.html +1 -1
- package/dashboard/out/action-log.html +1 -1
- package/dashboard/out/action-log.txt +1 -1
- package/dashboard/out/activity.html +1 -1
- package/dashboard/out/activity.txt +1 -1
- package/dashboard/out/bugs.html +1 -1
- package/dashboard/out/bugs.txt +1 -1
- package/dashboard/out/capture.html +1 -1
- package/dashboard/out/capture.txt +1 -1
- package/dashboard/out/config.html +1 -1
- package/dashboard/out/config.txt +1 -1
- package/dashboard/out/daemon.html +1 -1
- package/dashboard/out/daemon.txt +1 -1
- package/dashboard/out/design.html +1 -1
- package/dashboard/out/design.txt +1 -1
- package/dashboard/out/discord.html +1 -1
- package/dashboard/out/discord.txt +1 -1
- package/dashboard/out/file-index.html +1 -1
- package/dashboard/out/file-index.txt +1 -1
- package/dashboard/out/index.html +1 -1
- package/dashboard/out/index.txt +1 -1
- package/dashboard/out/insights.html +1 -1
- package/dashboard/out/insights.txt +1 -1
- package/dashboard/out/learning.html +1 -1
- package/dashboard/out/learning.txt +1 -1
- package/dashboard/out/overview.html +1 -1
- package/dashboard/out/overview.txt +1 -1
- package/dashboard/out/scheduler.html +1 -1
- package/dashboard/out/scheduler.txt +1 -1
- package/dashboard/out/sync.html +1 -1
- package/dashboard/out/sync.txt +1 -1
- package/dashboard/out/tokens.html +1 -1
- package/dashboard/out/tokens.txt +1 -1
- package/dashboard/out/waste.html +1 -1
- package/dashboard/out/waste.txt +1 -1
- package/dashboard/out/wiki.html +1 -1
- package/dashboard/out/wiki.txt +1 -1
- package/dist/cli.bun.js +90615 -0
- package/dist/{cli.js → cli.node.js} +2227 -758
- package/package.json +14 -4
- package/scripts/build.mjs +47 -0
- package/src/commands/bug-search.ts +2 -4
- package/src/commands/detect-waste.ts +24 -32
- package/src/commands/post-read.ts +10 -11
- package/src/commands/post-write.ts +13 -19
- package/src/commands/pre-read.ts +19 -24
- package/src/commands/scan.ts +103 -40
- package/src/commands/status.ts +45 -26
- package/src/core/bug-memory.ts +32 -34
- package/src/core/dashboard-api.ts +44 -22
- package/src/core/index-store.ts +23 -0
- package/src/core/paths.ts +7 -0
- package/src/core/scanner.ts +8 -4
- package/src/core/state-aggregator.ts +64 -7
- package/src/core/state-counters.ts +11 -31
- package/src/core/sync-merge-drivers.ts +164 -1
- package/src/core/sync.ts +9 -0
- package/src/core/token-ledger.ts +50 -4
- package/src/repositories/bug-memory-repo.ts +268 -0
- package/src/repositories/counters-repo.ts +88 -0
- package/src/repositories/file-index-repo.ts +238 -0
- package/src/repositories/token-ledger-repo.ts +412 -0
- package/src/storage/db.ts +121 -0
- package/src/storage/driver.bun.ts +99 -0
- package/src/storage/driver.node.ts +107 -0
- package/src/storage/driver.ts +76 -0
- package/src/storage/migrate-json.ts +415 -0
- package/src/storage/schema.ts +207 -0
- package/src/types/file-index.ts +9 -0
- /package/dashboard/out/_next/static/{fci7mSuW5y3ri6IlmLojm → 9ElzGFcXpcjLq-QSQslWY}/_buildManifest.js +0 -0
- /package/dashboard/out/_next/static/{fci7mSuW5y3ri6IlmLojm → 9ElzGFcXpcjLq-QSQslWY}/_ssgManifest.js +0 -0
package/package.json
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drewpayment/mink",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0-beta.1",
|
|
4
4
|
"description": "A hidden presence that moves alongside the developer — token efficiency and cross-project wiki for AI coding assistants",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
|
-
"mink": "./dist/cli.js"
|
|
7
|
+
"mink": "./dist/cli.node.js",
|
|
8
|
+
"mink-bun": "./dist/cli.bun.js"
|
|
9
|
+
},
|
|
10
|
+
"engines": {
|
|
11
|
+
"node": ">=22.5.0"
|
|
8
12
|
},
|
|
9
13
|
"scripts": {
|
|
10
|
-
"build": "
|
|
14
|
+
"build": "node scripts/build.mjs",
|
|
11
15
|
"postinstall": "bun run build 2>/dev/null || true",
|
|
12
16
|
"typecheck": "bunx tsc --noEmit",
|
|
13
17
|
"test": "bun test",
|
|
@@ -17,7 +21,9 @@
|
|
|
17
21
|
},
|
|
18
22
|
"files": [
|
|
19
23
|
"src/**/*.ts",
|
|
20
|
-
"dist/cli.js",
|
|
24
|
+
"dist/cli.node.js",
|
|
25
|
+
"dist/cli.bun.js",
|
|
26
|
+
"scripts/build.mjs",
|
|
21
27
|
"skills/**/*",
|
|
22
28
|
"agents/**/*",
|
|
23
29
|
"dashboard/out"
|
|
@@ -25,6 +31,10 @@
|
|
|
25
31
|
"publishConfig": {
|
|
26
32
|
"access": "public"
|
|
27
33
|
},
|
|
34
|
+
"repository": {
|
|
35
|
+
"type": "git",
|
|
36
|
+
"url": "git+https://github.com/drewpayment/mink.git"
|
|
37
|
+
},
|
|
28
38
|
"license": "MIT",
|
|
29
39
|
"dependencies": {
|
|
30
40
|
"puppeteer-core": "^24.0.0"
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Build both runtime bundles. Each invocation feeds `bun build` a
|
|
3
|
+
// `--define MINK_RUNTIME=...` value that the storage driver dispatcher in
|
|
4
|
+
// `src/storage/driver.ts` constant-folds, so the unused branch's
|
|
5
|
+
// `require("bun:sqlite")` / `require("node:sqlite")` is never executed at
|
|
6
|
+
// runtime — even though both strings are present in the bundle source.
|
|
7
|
+
//
|
|
8
|
+
// Outputs:
|
|
9
|
+
// dist/cli.bun.js — #!/usr/bin/env bun (faster startup, recommended)
|
|
10
|
+
// dist/cli.node.js — #!/usr/bin/env node (works wherever Node ≥22.5 is)
|
|
11
|
+
//
|
|
12
|
+
// `package.json:bin` maps the user-visible `mink` command to the Node
|
|
13
|
+
// bundle (broadest compat) and `mink-bun` to the Bun bundle.
|
|
14
|
+
|
|
15
|
+
import { execFileSync } from "node:child_process";
|
|
16
|
+
import { chmodSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
17
|
+
import { dirname } from "node:path";
|
|
18
|
+
|
|
19
|
+
const SRC = "src/cli.ts";
|
|
20
|
+
|
|
21
|
+
const TARGETS = [
|
|
22
|
+
{ runtime: "bun", outfile: "dist/cli.bun.js", target: "bun", shebang: "#!/usr/bin/env bun" },
|
|
23
|
+
{ runtime: "node", outfile: "dist/cli.node.js", target: "node", shebang: "#!/usr/bin/env node" },
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
function run(cmd, args) {
|
|
27
|
+
execFileSync(cmd, args, { stdio: "inherit" });
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
for (const t of TARGETS) {
|
|
31
|
+
mkdirSync(dirname(t.outfile), { recursive: true });
|
|
32
|
+
run("bun", [
|
|
33
|
+
"build", SRC,
|
|
34
|
+
"--outfile", t.outfile,
|
|
35
|
+
"--target", t.target,
|
|
36
|
+
"--format", "esm",
|
|
37
|
+
"--define", `MINK_RUNTIME="${t.runtime}"`,
|
|
38
|
+
]);
|
|
39
|
+
|
|
40
|
+
// `bun build` may emit its own shebang depending on the target. Strip any
|
|
41
|
+
// existing line beginning with `#!` and prepend the canonical one.
|
|
42
|
+
let body = readFileSync(t.outfile, "utf-8");
|
|
43
|
+
body = body.replace(/^#!.*\n/, "");
|
|
44
|
+
writeFileSync(t.outfile, `${t.shebang}\n${body}`);
|
|
45
|
+
chmodSync(t.outfile, 0o755);
|
|
46
|
+
console.log(`built ${t.outfile} (${t.runtime})`);
|
|
47
|
+
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { aggregateBugMemory } from "../core/state-aggregator";
|
|
1
|
+
import { BugMemoryRepo } from "../repositories/bug-memory-repo";
|
|
3
2
|
|
|
4
3
|
export function bugSearch(cwd: string, query: string): void {
|
|
5
4
|
if (!query) {
|
|
@@ -7,8 +6,7 @@ export function bugSearch(cwd: string, query: string): void {
|
|
|
7
6
|
process.exit(1);
|
|
8
7
|
}
|
|
9
8
|
|
|
10
|
-
const
|
|
11
|
-
const results = searchBugs(memory, query);
|
|
9
|
+
const results = BugMemoryRepo.for(cwd).searchBugs(query);
|
|
12
10
|
|
|
13
11
|
if (results.length === 0) {
|
|
14
12
|
console.log("No matching bugs found.");
|
|
@@ -1,24 +1,18 @@
|
|
|
1
1
|
import { statSync } from "fs";
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
learningMemoryPath,
|
|
6
|
-
} from "../core/paths";
|
|
7
|
-
import { loadLedger, saveLedger } from "../core/token-ledger";
|
|
8
|
-
import { isFileIndex, createEmptyIndex } from "../core/index-store";
|
|
2
|
+
import { learningMemoryPath } from "../core/paths";
|
|
3
|
+
import { FileIndexRepo } from "../repositories/file-index-repo";
|
|
4
|
+
import { TokenLedgerRepo } from "../repositories/token-ledger-repo";
|
|
9
5
|
import {
|
|
10
6
|
aggregateTokenLedger,
|
|
11
7
|
aggregateActionLog,
|
|
12
8
|
} from "../core/state-aggregator";
|
|
13
9
|
import { loadCounters } from "../core/state-counters";
|
|
14
|
-
import { safeReadJson } from "../core/fs-utils";
|
|
15
10
|
import { runDetection } from "../core/waste-detection";
|
|
16
11
|
import { getOrCreateDeviceId } from "../core/device";
|
|
17
12
|
import type { TokenLedger } from "../types/token-ledger";
|
|
18
|
-
import type {
|
|
13
|
+
import type { FileIndexEntry } from "../types/file-index";
|
|
19
14
|
|
|
20
15
|
export function detectWaste(cwd: string): void {
|
|
21
|
-
const idxPath = fileIndexPath(cwd);
|
|
22
16
|
const lmPath = learningMemoryPath(cwd);
|
|
23
17
|
|
|
24
18
|
// Aggregated ledger (across all device shards + legacy). Aggregator returns
|
|
@@ -27,14 +21,14 @@ export function detectWaste(cwd: string): void {
|
|
|
27
21
|
// already logged by loadLedger.
|
|
28
22
|
const ledger: TokenLedger = aggregateTokenLedger(cwd);
|
|
29
23
|
|
|
30
|
-
// Load file index
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
24
|
+
// Load file index — read every entry into the map shape the waste
|
|
25
|
+
// detector expects. listAll() walks the table once; under 20k rows it
|
|
26
|
+
// returns in single-digit ms.
|
|
27
|
+
const repo = FileIndexRepo.for(cwd);
|
|
28
|
+
const entries: Record<string, FileIndexEntry> = {};
|
|
29
|
+
for (const e of repo.listAll()) entries[e.filePath] = e;
|
|
30
|
+
const totalFiles = repo.totalFiles();
|
|
31
|
+
const lastScanTimestamp = repo.getLastScanTimestamp();
|
|
38
32
|
|
|
39
33
|
// Aggregated action log content (across all device shards + legacy)
|
|
40
34
|
const actionLogContent = aggregateActionLog(cwd);
|
|
@@ -47,32 +41,30 @@ export function detectWaste(cwd: string): void {
|
|
|
47
41
|
// File missing — will be flagged as stale
|
|
48
42
|
}
|
|
49
43
|
|
|
50
|
-
// Pull hit/miss telemetry from the
|
|
51
|
-
// the
|
|
52
|
-
//
|
|
44
|
+
// Pull hit/miss telemetry from the SQLite counters table. We synthesize
|
|
45
|
+
// the header shape the detector expects (lifetimeHits/Misses lived in
|
|
46
|
+
// the JSON header pre-migration).
|
|
53
47
|
const counters = loadCounters(cwd);
|
|
54
48
|
const headerForDetection = {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
49
|
+
lastScanTimestamp,
|
|
50
|
+
totalFiles,
|
|
51
|
+
lifetimeHits: counters.fileIndexHits,
|
|
52
|
+
lifetimeMisses: counters.fileIndexMisses,
|
|
58
53
|
};
|
|
59
54
|
|
|
60
55
|
// Run detection on the aggregated cross-device view
|
|
61
56
|
const flags = runDetection(
|
|
62
57
|
ledger,
|
|
63
|
-
|
|
58
|
+
entries,
|
|
64
59
|
headerForDetection,
|
|
65
60
|
actionLogContent,
|
|
66
61
|
learningMemoryMtimeMs
|
|
67
62
|
);
|
|
68
63
|
|
|
69
|
-
// Persist flags in THIS device's
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
|
|
73
|
-
const shardLedger = loadLedger(shardLedgerPath);
|
|
74
|
-
shardLedger.wasteFlags = flags;
|
|
75
|
-
saveLedger(shardLedgerPath, shardLedger);
|
|
64
|
+
// Persist flags in THIS device's waste_flags rows. The merge driver
|
|
65
|
+
// unions across devices on sync; replaceWasteFlagsForDevice clears
|
|
66
|
+
// and rewrites this device's set on every detection run.
|
|
67
|
+
TokenLedgerRepo.for(cwd).replaceWasteFlagsForDevice(getOrCreateDeviceId(), flags);
|
|
76
68
|
|
|
77
69
|
// Output summary
|
|
78
70
|
if (flags.length === 0) {
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { relative } from "path";
|
|
2
2
|
import { readStdinJson } from "../core/stdin";
|
|
3
|
-
import { sessionPath,
|
|
3
|
+
import { sessionPath, actionLogShardPath } from "../core/paths";
|
|
4
4
|
import { safeReadJson, atomicWriteJson } from "../core/fs-utils";
|
|
5
5
|
import { createSessionState, isSessionState, recordRead } from "../core/session";
|
|
6
|
-
import {
|
|
6
|
+
import { FileIndexRepo } from "../repositories/file-index-repo";
|
|
7
7
|
import { estimateTokens, isBinaryFile } from "../core/token-estimate";
|
|
8
8
|
import { createActionLogWriter } from "../core/action-log";
|
|
9
9
|
import { getOrCreateDeviceId } from "../core/device";
|
|
10
10
|
import type { SessionState } from "../types/session";
|
|
11
|
-
import type {
|
|
11
|
+
import type { IndexLookup } from "../types/file-index";
|
|
12
12
|
import type { PostToolUseInput } from "../types/hook-input";
|
|
13
13
|
|
|
14
14
|
export interface PostReadResult {
|
|
@@ -20,17 +20,17 @@ export interface PostReadResult {
|
|
|
20
20
|
export function analyzePostRead(
|
|
21
21
|
filePath: string,
|
|
22
22
|
content: string | null,
|
|
23
|
-
index:
|
|
23
|
+
index: IndexLookup | null
|
|
24
24
|
): PostReadResult {
|
|
25
25
|
// Binary file — skip token estimation
|
|
26
26
|
if (isBinaryFile(filePath, content ?? undefined)) {
|
|
27
|
-
const entry = index ? lookupEntry(
|
|
27
|
+
const entry = index ? index.lookupEntry(filePath) : null;
|
|
28
28
|
return { estimatedTokens: 0, indexHit: !!entry, source: "none" };
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
// Content available — estimate from actual content
|
|
32
32
|
if (content !== null && content.length > 0) {
|
|
33
|
-
const entry = index ? lookupEntry(
|
|
33
|
+
const entry = index ? index.lookupEntry(filePath) : null;
|
|
34
34
|
return {
|
|
35
35
|
estimatedTokens: estimateTokens(content, filePath),
|
|
36
36
|
indexHit: !!entry,
|
|
@@ -40,7 +40,7 @@ export function analyzePostRead(
|
|
|
40
40
|
|
|
41
41
|
// No content — try file index fallback
|
|
42
42
|
if (index) {
|
|
43
|
-
const entry = lookupEntry(
|
|
43
|
+
const entry = index.lookupEntry(filePath);
|
|
44
44
|
if (entry) {
|
|
45
45
|
return {
|
|
46
46
|
estimatedTokens: entry.estimatedTokens,
|
|
@@ -89,14 +89,13 @@ export async function postRead(cwd: string): Promise<void> {
|
|
|
89
89
|
? rawState
|
|
90
90
|
: createSessionState();
|
|
91
91
|
|
|
92
|
-
//
|
|
93
|
-
const
|
|
94
|
-
const index: FileIndex | null = isFileIndex(rawIndex) ? rawIndex : null;
|
|
92
|
+
// File index repository — one-key lookup, no whole-index load.
|
|
93
|
+
const repo = FileIndexRepo.for(cwd);
|
|
95
94
|
|
|
96
95
|
// Extract content from tool output
|
|
97
96
|
const content = extractContent(input);
|
|
98
97
|
|
|
99
|
-
const result = analyzePostRead(filePath, content,
|
|
98
|
+
const result = analyzePostRead(filePath, content, repo);
|
|
100
99
|
|
|
101
100
|
// Record the read in session state
|
|
102
101
|
recordRead(state, filePath, result.estimatedTokens, result.indexHit);
|
|
@@ -1,22 +1,17 @@
|
|
|
1
1
|
import { relative } from "path";
|
|
2
2
|
import { readFileSync } from "fs";
|
|
3
3
|
import { readStdinJson } from "../core/stdin";
|
|
4
|
-
import { sessionPath,
|
|
4
|
+
import { sessionPath, actionLogShardPath } from "../core/paths";
|
|
5
5
|
import { getOrCreateDeviceId } from "../core/device";
|
|
6
6
|
import { safeReadJson, atomicWriteJson } from "../core/fs-utils";
|
|
7
7
|
import { createSessionState, isSessionState, recordWrite } from "../core/session";
|
|
8
|
-
import {
|
|
9
|
-
isFileIndex,
|
|
10
|
-
lookupEntry,
|
|
11
|
-
upsertEntry,
|
|
12
|
-
createEmptyIndex,
|
|
13
|
-
} from "../core/index-store";
|
|
8
|
+
import { FileIndexRepo } from "../repositories/file-index-repo";
|
|
14
9
|
import { extractDescription } from "../core/description";
|
|
15
10
|
import { estimateTokens, isBinaryFile } from "../core/token-estimate";
|
|
16
11
|
import { isWriteExcluded } from "../core/write-exclusions";
|
|
17
12
|
import { createActionLogWriter } from "../core/action-log";
|
|
18
13
|
import type { SessionState } from "../types/session";
|
|
19
|
-
import type {
|
|
14
|
+
import type { FileIndexEntry, IndexLookup } from "../types/file-index";
|
|
20
15
|
import type { PostToolUseInput } from "../types/hook-input";
|
|
21
16
|
|
|
22
17
|
export interface PostWriteResult {
|
|
@@ -30,7 +25,7 @@ export interface PostWriteResult {
|
|
|
30
25
|
export function analyzePostWrite(
|
|
31
26
|
filePath: string,
|
|
32
27
|
fileContent: string | null,
|
|
33
|
-
index:
|
|
28
|
+
index: IndexLookup | null
|
|
34
29
|
): PostWriteResult {
|
|
35
30
|
// Check exclusions
|
|
36
31
|
if (isWriteExcluded(filePath)) {
|
|
@@ -43,8 +38,9 @@ export function analyzePostWrite(
|
|
|
43
38
|
};
|
|
44
39
|
}
|
|
45
40
|
|
|
46
|
-
// Determine action from index presence
|
|
47
|
-
|
|
41
|
+
// Determine action from index presence (one-key lookup; never loads the
|
|
42
|
+
// whole index — important for 20k-file projects).
|
|
43
|
+
const existingEntry = index ? index.lookupEntry(filePath) : null;
|
|
48
44
|
const action: "create" | "edit" = existingEntry ? "edit" : "create";
|
|
49
45
|
|
|
50
46
|
// Handle binary or unreadable content
|
|
@@ -117,17 +113,16 @@ export async function postWrite(cwd: string): Promise<void> {
|
|
|
117
113
|
? rawState
|
|
118
114
|
: createSessionState();
|
|
119
115
|
|
|
120
|
-
//
|
|
121
|
-
const
|
|
122
|
-
const index: FileIndex = isFileIndex(rawIndex) ? rawIndex : createEmptyIndex();
|
|
116
|
+
// File index repository — one-key lookup, no whole-index load.
|
|
117
|
+
const repo = FileIndexRepo.for(cwd);
|
|
123
118
|
|
|
124
|
-
const result = analyzePostWrite(filePath, fileContent,
|
|
119
|
+
const result = analyzePostWrite(filePath, fileContent, repo);
|
|
125
120
|
|
|
126
121
|
if (result.excluded) return;
|
|
127
122
|
|
|
128
|
-
// 1. File index update
|
|
123
|
+
// 1. File index update — single-row upsert.
|
|
129
124
|
if (result.indexEntry) {
|
|
130
|
-
|
|
125
|
+
repo.upsert(result.indexEntry);
|
|
131
126
|
}
|
|
132
127
|
|
|
133
128
|
// 2. Action log entry — write to this device's shard
|
|
@@ -149,9 +144,8 @@ export async function postWrite(cwd: string): Promise<void> {
|
|
|
149
144
|
// 3. Session state update
|
|
150
145
|
recordWrite(state, filePath, result.action, result.estimatedTokens);
|
|
151
146
|
|
|
152
|
-
// Persist
|
|
147
|
+
// Persist session — file index already committed via repo.upsert.
|
|
153
148
|
atomicWriteJson(sessionPath(cwd), state);
|
|
154
|
-
atomicWriteJson(fileIndexPath(cwd), index);
|
|
155
149
|
} catch {
|
|
156
150
|
// Never crash — exit silently
|
|
157
151
|
} finally {
|
package/src/commands/pre-read.ts
CHANGED
|
@@ -1,15 +1,12 @@
|
|
|
1
1
|
import { relative } from "path";
|
|
2
2
|
import { readStdinJson } from "../core/stdin";
|
|
3
|
-
import { sessionPath
|
|
3
|
+
import { sessionPath } from "../core/paths";
|
|
4
4
|
import { safeReadJson, atomicWriteJson } from "../core/fs-utils";
|
|
5
5
|
import { createSessionState, isSessionState } from "../core/session";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
incrementFileIndexHit,
|
|
9
|
-
incrementFileIndexMiss,
|
|
10
|
-
} from "../core/state-counters";
|
|
6
|
+
import { FileIndexRepo } from "../repositories/file-index-repo";
|
|
7
|
+
import { CountersRepo } from "../repositories/counters-repo";
|
|
11
8
|
import type { SessionState } from "../types/session";
|
|
12
|
-
import type {
|
|
9
|
+
import type { FileIndexEntry, IndexLookup } from "../types/file-index";
|
|
13
10
|
import type { PreToolUseInput } from "../types/hook-input";
|
|
14
11
|
|
|
15
12
|
export interface PreReadResult {
|
|
@@ -22,7 +19,7 @@ export interface PreReadResult {
|
|
|
22
19
|
export function analyzePreRead(
|
|
23
20
|
filePath: string,
|
|
24
21
|
state: SessionState,
|
|
25
|
-
index:
|
|
22
|
+
index: IndexLookup | null
|
|
26
23
|
): PreReadResult {
|
|
27
24
|
const warnings: string[] = [];
|
|
28
25
|
let repeatedRead = false;
|
|
@@ -39,12 +36,12 @@ export function analyzePreRead(
|
|
|
39
36
|
state.counters.repeatedReadWarnings++;
|
|
40
37
|
}
|
|
41
38
|
|
|
42
|
-
// File index lookup. Hit/miss telemetry is persisted by the caller
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
//
|
|
39
|
+
// File index lookup. Hit/miss telemetry is persisted by the caller into
|
|
40
|
+
// the counters table, not by mutating the index — keeps the file_index
|
|
41
|
+
// row's last_modified stable so the sync merge driver doesn't churn it
|
|
42
|
+
// on every read.
|
|
46
43
|
if (index) {
|
|
47
|
-
entry = lookupEntry(
|
|
44
|
+
entry = index.lookupEntry(filePath);
|
|
48
45
|
if (entry) {
|
|
49
46
|
indexHit = true;
|
|
50
47
|
warnings.push(
|
|
@@ -84,11 +81,10 @@ export async function preRead(cwd: string): Promise<void> {
|
|
|
84
81
|
? rawState
|
|
85
82
|
: createSessionState();
|
|
86
83
|
|
|
87
|
-
//
|
|
88
|
-
const
|
|
89
|
-
const index: FileIndex | null = isFileIndex(rawIndex) ? rawIndex : null;
|
|
84
|
+
// File index repository — one-key lookup per hook.
|
|
85
|
+
const repo = FileIndexRepo.for(cwd);
|
|
90
86
|
|
|
91
|
-
const result = analyzePreRead(filePath, state,
|
|
87
|
+
const result = analyzePreRead(filePath, state, repo);
|
|
92
88
|
|
|
93
89
|
// Emit warnings to stderr
|
|
94
90
|
for (const warning of result.warnings) {
|
|
@@ -97,13 +93,12 @@ export async function preRead(cwd: string): Promise<void> {
|
|
|
97
93
|
|
|
98
94
|
// Persist state changes
|
|
99
95
|
atomicWriteJson(sessionPath(cwd), state);
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
}
|
|
96
|
+
try {
|
|
97
|
+
const counters = CountersRepo.for(cwd);
|
|
98
|
+
if (result.indexHit) counters.incrementHit();
|
|
99
|
+
else counters.incrementMiss();
|
|
100
|
+
} catch {
|
|
101
|
+
// Counter table is best-effort telemetry — never block the read hook
|
|
107
102
|
}
|
|
108
103
|
} catch {
|
|
109
104
|
// Never crash — exit silently
|
package/src/commands/scan.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFileSync } from "fs";
|
|
2
|
+
import { createHash } from "crypto";
|
|
2
3
|
import { join, relative } from "path";
|
|
3
|
-
import {
|
|
4
|
-
import { atomicWriteJson, safeReadJson } from "../core/fs-utils";
|
|
4
|
+
import { configPath } from "../core/paths";
|
|
5
5
|
import {
|
|
6
6
|
scanProject,
|
|
7
7
|
scanProjectWithStats,
|
|
@@ -10,45 +10,39 @@ import {
|
|
|
10
10
|
} from "../core/scanner";
|
|
11
11
|
import { extractDescription } from "../core/description";
|
|
12
12
|
import { estimateTokens } from "../core/token-estimate";
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
isFileIndex,
|
|
16
|
-
upsertEntry,
|
|
17
|
-
checkStaleness,
|
|
18
|
-
} from "../core/index-store";
|
|
19
|
-
import type { FileIndex, FileIndexEntry } from "../types/file-index";
|
|
13
|
+
import { FileIndexRepo } from "../repositories/file-index-repo";
|
|
14
|
+
import type { FileIndexEntry } from "../types/file-index";
|
|
20
15
|
|
|
21
16
|
function configRelativePath(cfgPath: string, cwd: string): string {
|
|
22
17
|
const rel = relative(cwd, cfgPath);
|
|
23
18
|
return rel.startsWith("..") ? cfgPath : rel;
|
|
24
19
|
}
|
|
25
20
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
}
|
|
32
|
-
return createEmptyIndex();
|
|
21
|
+
// Truncated SHA-1 — 16 hex chars is plenty to distinguish content
|
|
22
|
+
// versions of the same file. Cheaper than cryptographic strength;
|
|
23
|
+
// we only use it for change detection, never as a security boundary.
|
|
24
|
+
function contentHashOf(content: string): string {
|
|
25
|
+
return createHash("sha1").update(content).digest("hex").slice(0, 16);
|
|
33
26
|
}
|
|
34
27
|
|
|
35
28
|
export function scan(cwd: string, options: { check: boolean }): void {
|
|
36
|
-
const idxPath = fileIndexPath(cwd);
|
|
37
29
|
const cfgPath = configPath(cwd);
|
|
38
30
|
const config = loadConfig(cfgPath);
|
|
39
31
|
const excludes = getExcludes(config);
|
|
40
|
-
|
|
32
|
+
// No default cap as of Phase 5 — per-row write cost is flat in SQLite.
|
|
33
|
+
// Users who still want a cap set `maxFiles` in config.json.
|
|
34
|
+
const maxFiles = config.maxFiles;
|
|
35
|
+
const repo = FileIndexRepo.for(cwd);
|
|
41
36
|
|
|
42
37
|
if (options.check) {
|
|
43
|
-
|
|
44
|
-
if (!isFileIndex(existing)) {
|
|
38
|
+
if (repo.totalFiles() === 0) {
|
|
45
39
|
console.error("[mink] no index found — run mink scan first");
|
|
46
40
|
process.exit(1);
|
|
47
41
|
}
|
|
48
42
|
|
|
49
43
|
const scanned = scanProject(cwd, excludes, maxFiles);
|
|
50
44
|
const scannedPaths = scanned.map((f) => f.relativePath);
|
|
51
|
-
const report = checkStaleness(
|
|
45
|
+
const report = repo.checkStaleness(scannedPaths);
|
|
52
46
|
|
|
53
47
|
if (!report.isStale) {
|
|
54
48
|
console.log("[mink] index is up to date");
|
|
@@ -70,45 +64,103 @@ export function scan(cwd: string, options: { check: boolean }): void {
|
|
|
70
64
|
process.exit(1);
|
|
71
65
|
}
|
|
72
66
|
|
|
73
|
-
//
|
|
67
|
+
// Incremental scan — the actual 20k-file win.
|
|
68
|
+
// 1. Walk the tree (cheap; just stat + readdir).
|
|
69
|
+
// 2. Repo.staleSet(scanned) returns only paths whose mtime differs
|
|
70
|
+
// from what's stored (or never indexed at all). Everything else
|
|
71
|
+
// gets skipped — no readFileSync, no description extract, no
|
|
72
|
+
// token estimate.
|
|
73
|
+
// 3. For stale paths, read content + compute a content hash. If the
|
|
74
|
+
// stored content_hash matches, the file was just touched
|
|
75
|
+
// without an edit — skip the description/tokens re-extract and
|
|
76
|
+
// only refresh mtime/last_indexed. Otherwise do the full
|
|
77
|
+
// re-extract.
|
|
78
|
+
// 4. Bulk upsert in a single transaction, then prune orphans.
|
|
74
79
|
const start = Date.now();
|
|
75
|
-
const index = loadExistingIndex(idxPath);
|
|
76
80
|
|
|
77
81
|
const stats = scanProjectWithStats(cwd, excludes, maxFiles);
|
|
78
82
|
const scanned = stats.files;
|
|
79
83
|
|
|
80
|
-
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
+
const stalePaths = new Set(repo.staleSet(scanned));
|
|
85
|
+
const batch: Array<{
|
|
86
|
+
entry: FileIndexEntry;
|
|
87
|
+
opts: { mtimeMs: number; contentHash: string | null; sizeBytes: number };
|
|
88
|
+
}> = [];
|
|
89
|
+
let touchOnlyCount = 0;
|
|
90
|
+
let extractedCount = 0;
|
|
84
91
|
|
|
85
92
|
for (const file of scanned) {
|
|
93
|
+
if (!stalePaths.has(file.relativePath)) {
|
|
94
|
+
// mtime matches what we have — nothing to do for this file.
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
|
|
86
98
|
const fullPath = join(cwd, file.relativePath);
|
|
87
99
|
let content: string;
|
|
88
100
|
try {
|
|
89
101
|
content = readFileSync(fullPath, "utf-8");
|
|
90
102
|
} catch {
|
|
91
|
-
continue; // Skip unreadable files
|
|
103
|
+
continue; // Skip unreadable files (permissions, race condition)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const hash = contentHashOf(content);
|
|
107
|
+
const existing = repo.lookupEntry(file.relativePath);
|
|
108
|
+
const existingHash = existing ? repo.contentHashFor(file.relativePath) : null;
|
|
109
|
+
|
|
110
|
+
if (existing && existingHash === hash) {
|
|
111
|
+
// Touched but unchanged — bump mtime/last_indexed only, keep the
|
|
112
|
+
// stored description + token estimate.
|
|
113
|
+
batch.push({
|
|
114
|
+
entry: {
|
|
115
|
+
filePath: file.relativePath,
|
|
116
|
+
description: existing.description,
|
|
117
|
+
estimatedTokens: existing.estimatedTokens,
|
|
118
|
+
lastModified: new Date(file.mtimeMs).toISOString(),
|
|
119
|
+
lastIndexed: new Date().toISOString(),
|
|
120
|
+
},
|
|
121
|
+
opts: {
|
|
122
|
+
mtimeMs: Math.floor(file.mtimeMs),
|
|
123
|
+
contentHash: hash,
|
|
124
|
+
sizeBytes: content.length,
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
touchOnlyCount++;
|
|
128
|
+
continue;
|
|
92
129
|
}
|
|
93
130
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
131
|
+
// Full re-extract.
|
|
132
|
+
batch.push({
|
|
133
|
+
entry: {
|
|
134
|
+
filePath: file.relativePath,
|
|
135
|
+
description: extractDescription(file.relativePath, content),
|
|
136
|
+
estimatedTokens: estimateTokens(content, file.relativePath),
|
|
137
|
+
lastModified: new Date(file.mtimeMs).toISOString(),
|
|
138
|
+
lastIndexed: new Date().toISOString(),
|
|
139
|
+
},
|
|
140
|
+
opts: {
|
|
141
|
+
mtimeMs: Math.floor(file.mtimeMs),
|
|
142
|
+
contentHash: hash,
|
|
143
|
+
sizeBytes: content.length,
|
|
144
|
+
},
|
|
145
|
+
});
|
|
146
|
+
extractedCount++;
|
|
102
147
|
}
|
|
103
148
|
|
|
104
|
-
|
|
149
|
+
// Single transaction — ~50x faster than per-file commits at 20k files.
|
|
150
|
+
if (batch.length > 0) repo.upsertMany(batch);
|
|
151
|
+
|
|
152
|
+
// Prune orphans: every entry whose file is no longer on disk.
|
|
153
|
+
const removed = repo.retainOnly(scanned.map((f) => f.relativePath));
|
|
105
154
|
|
|
106
|
-
|
|
155
|
+
repo.setLastScanTimestamp(new Date().toISOString());
|
|
107
156
|
|
|
108
157
|
const elapsed = Date.now() - start;
|
|
158
|
+
const indexed = repo.totalFiles();
|
|
159
|
+
const skipped = scanned.length - stalePaths.size;
|
|
160
|
+
|
|
109
161
|
if (stats.truncated > 0) {
|
|
110
162
|
console.log(
|
|
111
|
-
`[mink] scanned ${stats.totalScanned} files; indexed ${
|
|
163
|
+
`[mink] scanned ${stats.totalScanned} files; indexed ${indexed} most recent in ${elapsed}ms`
|
|
112
164
|
);
|
|
113
165
|
console.log(
|
|
114
166
|
` ${stats.truncated} files past maxFiles=${maxFiles} were not indexed`
|
|
@@ -116,9 +168,20 @@ export function scan(cwd: string, options: { check: boolean }): void {
|
|
|
116
168
|
console.log(
|
|
117
169
|
` raise the cap by setting "maxFiles" in ${configRelativePath(cfgPath, cwd)}`
|
|
118
170
|
);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (skipped === scanned.length && extractedCount === 0 && touchOnlyCount === 0 && removed === 0) {
|
|
175
|
+
console.log(`[mink] indexed ${indexed} files in ${elapsed}ms (no changes)`);
|
|
119
176
|
} else {
|
|
177
|
+
const parts: string[] = [];
|
|
178
|
+
if (extractedCount > 0) parts.push(`${extractedCount} re-indexed`);
|
|
179
|
+
if (touchOnlyCount > 0) parts.push(`${touchOnlyCount} touch-only`);
|
|
180
|
+
if (removed > 0) parts.push(`${removed} pruned`);
|
|
181
|
+
if (skipped > 0) parts.push(`${skipped} unchanged`);
|
|
120
182
|
console.log(
|
|
121
|
-
`[mink] indexed ${
|
|
183
|
+
`[mink] indexed ${indexed} files in ${elapsed}ms (${parts.join(", ")})`
|
|
122
184
|
);
|
|
123
185
|
}
|
|
124
186
|
}
|
|
187
|
+
|