grepmax 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/NOTICE +2 -2
- package/README.md +72 -72
- package/dist/commands/claude-code.js +6 -6
- package/dist/commands/codex.js +17 -17
- package/dist/commands/doctor.js +6 -5
- package/dist/commands/droid.js +22 -22
- package/dist/commands/index.js +1 -1
- package/dist/commands/list.js +82 -19
- package/dist/commands/mcp.js +177 -158
- package/dist/commands/opencode.js +26 -26
- package/dist/commands/search.js +23 -13
- package/dist/commands/serve.js +30 -30
- package/dist/commands/setup.js +51 -40
- package/dist/commands/skeleton.js +19 -13
- package/dist/commands/symbols.js +40 -2
- package/dist/commands/verify.js +1 -1
- package/dist/commands/watch.js +206 -0
- package/dist/config.js +37 -7
- package/dist/eval.js +14 -14
- package/dist/index.js +11 -7
- package/dist/lib/core/languages.js +28 -0
- package/dist/lib/index/chunker.js +6 -3
- package/dist/lib/index/grammar-loader.js +2 -2
- package/dist/lib/index/ignore-patterns.js +1 -1
- package/dist/lib/index/index-config.js +50 -10
- package/dist/lib/index/sync-helpers.js +1 -1
- package/dist/lib/index/syncer.js +67 -45
- package/dist/lib/index/walker.js +3 -3
- package/dist/lib/index/watcher.js +4 -4
- package/dist/lib/output/formatter.js +1 -1
- package/dist/lib/search/searcher.js +9 -9
- package/dist/lib/setup/model-loader.js +3 -3
- package/dist/lib/setup/setup-helpers.js +2 -4
- package/dist/lib/skeleton/body-fields.js +20 -0
- package/dist/lib/skeleton/retriever.js +1 -1
- package/dist/lib/skeleton/skeletonizer.js +8 -2
- package/dist/lib/skeleton/summary-formatter.js +1 -4
- package/dist/lib/store/meta-cache.js +28 -3
- package/dist/lib/store/vector-db.js +17 -9
- package/dist/lib/utils/formatter.js +3 -3
- package/dist/lib/utils/lock.js +1 -1
- package/dist/lib/utils/project-registry.js +83 -0
- package/dist/lib/utils/project-root.js +32 -57
- package/dist/lib/utils/watcher-registry.js +100 -0
- package/dist/lib/workers/colbert-math.js +2 -2
- package/dist/lib/workers/download-worker.js +2 -2
- package/dist/lib/workers/embeddings/colbert.js +2 -2
- package/dist/lib/workers/embeddings/granite.js +4 -4
- package/dist/lib/workers/embeddings/mlx-client.js +1 -1
- package/dist/lib/workers/orchestrator.js +8 -8
- package/dist/lib/workers/pool.js +1 -1
- package/dist/lib/workers/worker.js +4 -1
- package/package.json +20 -21
- package/plugins/{osgrep → grepmax}/.claude-plugin/plugin.json +4 -4
- package/plugins/grepmax/hooks/start.js +63 -0
- package/plugins/grepmax/hooks/stop.js +3 -0
- package/plugins/{osgrep/skills/osgrep → grepmax/skills/gmax}/SKILL.md +11 -11
- package/plugins/osgrep/hooks/start.js +0 -90
- package/plugins/osgrep/hooks/stop.js +0 -3
- /package/plugins/{osgrep → grepmax}/hooks.json +0 -0
|
@@ -102,6 +102,31 @@ class MetaCache {
|
|
|
102
102
|
return keys;
|
|
103
103
|
});
|
|
104
104
|
}
|
|
105
|
+
getKeysWithPrefix(prefix) {
|
|
106
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
107
|
+
var _a, e_2, _b, _c;
|
|
108
|
+
const keys = new Set();
|
|
109
|
+
try {
|
|
110
|
+
for (var _d = true, _e = __asyncValues(this.db.getRange({ start: prefix })), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
|
|
111
|
+
_c = _f.value;
|
|
112
|
+
_d = false;
|
|
113
|
+
const { key } = _c;
|
|
114
|
+
const k = String(key);
|
|
115
|
+
if (!k.startsWith(prefix))
|
|
116
|
+
break;
|
|
117
|
+
keys.add(k);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
121
|
+
finally {
|
|
122
|
+
try {
|
|
123
|
+
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
|
|
124
|
+
}
|
|
125
|
+
finally { if (e_2) throw e_2.error; }
|
|
126
|
+
}
|
|
127
|
+
return keys;
|
|
128
|
+
});
|
|
129
|
+
}
|
|
105
130
|
put(filePath, entry) {
|
|
106
131
|
this.db.put(filePath, entry);
|
|
107
132
|
}
|
|
@@ -110,7 +135,7 @@ class MetaCache {
|
|
|
110
135
|
}
|
|
111
136
|
entries() {
|
|
112
137
|
return __asyncGenerator(this, arguments, function* entries_1() {
|
|
113
|
-
var _a,
|
|
138
|
+
var _a, e_3, _b, _c;
|
|
114
139
|
try {
|
|
115
140
|
for (var _d = true, _e = __asyncValues(this.db.getRange()), _f; _f = yield __await(_e.next()), _a = _f.done, !_a; _d = true) {
|
|
116
141
|
_c = _f.value;
|
|
@@ -121,12 +146,12 @@ class MetaCache {
|
|
|
121
146
|
yield yield __await({ path: String(key), entry: value });
|
|
122
147
|
}
|
|
123
148
|
}
|
|
124
|
-
catch (
|
|
149
|
+
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
125
150
|
finally {
|
|
126
151
|
try {
|
|
127
152
|
if (!_d && !_a && (_b = _e.return)) yield __await(_b.call(_e));
|
|
128
153
|
}
|
|
129
|
-
finally { if (
|
|
154
|
+
finally { if (e_3) throw e_3.error; }
|
|
130
155
|
}
|
|
131
156
|
});
|
|
132
157
|
}
|
|
@@ -50,10 +50,11 @@ const config_1 = require("../../config");
|
|
|
50
50
|
const cleanup_1 = require("../utils/cleanup");
|
|
51
51
|
const TABLE_NAME = "chunks";
|
|
52
52
|
class VectorDB {
|
|
53
|
-
constructor(lancedbDir) {
|
|
53
|
+
constructor(lancedbDir, vectorDim) {
|
|
54
54
|
this.lancedbDir = lancedbDir;
|
|
55
55
|
this.db = null;
|
|
56
56
|
this.closed = false;
|
|
57
|
+
this.vectorDim = vectorDim !== null && vectorDim !== void 0 ? vectorDim : config_1.CONFIG.VECTOR_DIM;
|
|
57
58
|
this.unregisterCleanup = (0, cleanup_1.registerCleanup)(() => this.close());
|
|
58
59
|
}
|
|
59
60
|
getDb() {
|
|
@@ -84,7 +85,7 @@ class VectorDB {
|
|
|
84
85
|
chunk_type: "",
|
|
85
86
|
complexity: 0,
|
|
86
87
|
is_exported: false,
|
|
87
|
-
vector: Array(
|
|
88
|
+
vector: Array(this.vectorDim).fill(0),
|
|
88
89
|
colbert: Buffer.alloc(0),
|
|
89
90
|
colbert_scale: 1,
|
|
90
91
|
pooled_colbert_48d: Array(config_1.CONFIG.COLBERT_DIM).fill(0),
|
|
@@ -105,7 +106,7 @@ class VectorDB {
|
|
|
105
106
|
const required = ["complexity", "is_exported"];
|
|
106
107
|
const missing = required.filter((r) => !fields.has(r));
|
|
107
108
|
if (missing.length > 0) {
|
|
108
|
-
throw new Error(`[vector-db] schema missing fields (${missing.join(", ")}). Please run "
|
|
109
|
+
throw new Error(`[vector-db] schema missing fields (${missing.join(", ")}). Please run "gmax index --reset" to rebuild the index.`);
|
|
109
110
|
}
|
|
110
111
|
});
|
|
111
112
|
}
|
|
@@ -118,7 +119,7 @@ class VectorDB {
|
|
|
118
119
|
new apache_arrow_1.Field("display_text", new apache_arrow_1.Utf8(), false),
|
|
119
120
|
new apache_arrow_1.Field("start_line", new apache_arrow_1.Int32(), false),
|
|
120
121
|
new apache_arrow_1.Field("end_line", new apache_arrow_1.Int32(), false),
|
|
121
|
-
new apache_arrow_1.Field("vector", new apache_arrow_1.FixedSizeList(
|
|
122
|
+
new apache_arrow_1.Field("vector", new apache_arrow_1.FixedSizeList(this.vectorDim, new apache_arrow_1.Field("item", new apache_arrow_1.Float32(), false)), false),
|
|
122
123
|
new apache_arrow_1.Field("chunk_index", new apache_arrow_1.Int32(), true),
|
|
123
124
|
new apache_arrow_1.Field("is_anchor", new apache_arrow_1.Bool(), true),
|
|
124
125
|
new apache_arrow_1.Field("context_prev", new apache_arrow_1.Utf8(), true),
|
|
@@ -203,11 +204,11 @@ class VectorDB {
|
|
|
203
204
|
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
204
205
|
const vec = (() => {
|
|
205
206
|
const arr = toNumberArray(rec.vector);
|
|
206
|
-
if (arr.length <
|
|
207
|
-
arr.push(...Array(
|
|
207
|
+
if (arr.length < this.vectorDim) {
|
|
208
|
+
arr.push(...Array(this.vectorDim - arr.length).fill(0));
|
|
208
209
|
}
|
|
209
|
-
else if (arr.length >
|
|
210
|
-
arr.length =
|
|
210
|
+
else if (arr.length > this.vectorDim) {
|
|
211
|
+
arr.length = this.vectorDim;
|
|
211
212
|
}
|
|
212
213
|
return arr;
|
|
213
214
|
})();
|
|
@@ -250,7 +251,7 @@ class VectorDB {
|
|
|
250
251
|
if (msg.toLowerCase().includes("found field not in schema")) {
|
|
251
252
|
const schema = yield table.schema();
|
|
252
253
|
const schemaFields = schema.fields.map((f) => f.name);
|
|
253
|
-
throw new Error(`[vector-db] schema mismatch detected (fields: ${schemaFields.join(", ")}). Please run "
|
|
254
|
+
throw new Error(`[vector-db] schema mismatch detected (fields: ${schemaFields.join(", ")}). Please run "gmax index --reset" to rebuild the index.`);
|
|
254
255
|
}
|
|
255
256
|
throw err;
|
|
256
257
|
}
|
|
@@ -310,6 +311,13 @@ class VectorDB {
|
|
|
310
311
|
}
|
|
311
312
|
});
|
|
312
313
|
}
|
|
314
|
+
deletePathsWithPrefix(prefix) {
|
|
315
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
316
|
+
const table = yield this.ensureTable();
|
|
317
|
+
const escaped = prefix.replace(/'/g, "''");
|
|
318
|
+
yield table.delete(`path LIKE '${escaped}%'`);
|
|
319
|
+
});
|
|
320
|
+
}
|
|
313
321
|
drop() {
|
|
314
322
|
return __awaiter(this, void 0, void 0, function* () {
|
|
315
323
|
const db = yield this.getDb();
|
|
@@ -89,7 +89,7 @@ function cleanSnippetLines(snippet) {
|
|
|
89
89
|
function formatTextResults(results, query, root, options) {
|
|
90
90
|
var _a, _b, _c;
|
|
91
91
|
if (results.length === 0)
|
|
92
|
-
return `
|
|
92
|
+
return `gmax: No results found for "${query}".`;
|
|
93
93
|
// --- MODE: COMPACT (File paths only) ---
|
|
94
94
|
if (options.compact) {
|
|
95
95
|
const uniquePaths = Array.from(new Set(results.map((r) => r.path))).sort();
|
|
@@ -139,7 +139,7 @@ function formatTextResults(results, query, root, options) {
|
|
|
139
139
|
});
|
|
140
140
|
output += "\n";
|
|
141
141
|
});
|
|
142
|
-
output += `
|
|
142
|
+
output += `gmax results (${results.length} matches across ${fileCount} files)`;
|
|
143
143
|
return output.trim();
|
|
144
144
|
}
|
|
145
145
|
// --- MODE B: HUMAN (Pretty) ---
|
|
@@ -175,7 +175,7 @@ function formatTextResults(results, query, root, options) {
|
|
|
175
175
|
mergedGroups.push({ filePath, merged });
|
|
176
176
|
}
|
|
177
177
|
const displayedCount = mergedGroups.reduce((sum, g) => sum + g.merged.length, 0);
|
|
178
|
-
let output = `\n${style.bold(`
|
|
178
|
+
let output = `\n${style.bold(`gmax results (query: "${query}", ${displayedCount} matches across ${fileCount} files)`)}\n`;
|
|
179
179
|
let rank = 1;
|
|
180
180
|
for (const { filePath, merged } of mergedGroups) {
|
|
181
181
|
const relPath = path.relative(root, filePath);
|
package/dist/lib/utils/lock.js
CHANGED
|
@@ -108,7 +108,7 @@ function acquireWriterLock(lockDir) {
|
|
|
108
108
|
const holderDesc = pid
|
|
109
109
|
? `${pid}${startedAt ? ` @ ${startedAt}` : ""}`
|
|
110
110
|
: "unknown";
|
|
111
|
-
throw new Error(`.
|
|
111
|
+
throw new Error(`.gmax lock already held (${holderDesc}). Another indexing process is running or the lock must be cleared.`);
|
|
112
112
|
}
|
|
113
113
|
}
|
|
114
114
|
return {
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Global project registry — tracks all indexed projects for
|
|
4
|
+
* cross-project search and dimension compatibility checking.
|
|
5
|
+
*
|
|
6
|
+
* Stored in ~/.gmax/projects.json
|
|
7
|
+
*/
|
|
8
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
|
+
if (k2 === undefined) k2 = k;
|
|
10
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
11
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
12
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
13
|
+
}
|
|
14
|
+
Object.defineProperty(o, k2, desc);
|
|
15
|
+
}) : (function(o, m, k, k2) {
|
|
16
|
+
if (k2 === undefined) k2 = k;
|
|
17
|
+
o[k2] = m[k];
|
|
18
|
+
}));
|
|
19
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
20
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
21
|
+
}) : function(o, v) {
|
|
22
|
+
o["default"] = v;
|
|
23
|
+
});
|
|
24
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
25
|
+
var ownKeys = function(o) {
|
|
26
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
27
|
+
var ar = [];
|
|
28
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
29
|
+
return ar;
|
|
30
|
+
};
|
|
31
|
+
return ownKeys(o);
|
|
32
|
+
};
|
|
33
|
+
return function (mod) {
|
|
34
|
+
if (mod && mod.__esModule) return mod;
|
|
35
|
+
var result = {};
|
|
36
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
37
|
+
__setModuleDefault(result, mod);
|
|
38
|
+
return result;
|
|
39
|
+
};
|
|
40
|
+
})();
|
|
41
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
+
exports.registerProject = registerProject;
|
|
43
|
+
exports.listProjects = listProjects;
|
|
44
|
+
exports.getProject = getProject;
|
|
45
|
+
exports.removeProject = removeProject;
|
|
46
|
+
const fs = __importStar(require("node:fs"));
|
|
47
|
+
const path = __importStar(require("node:path"));
|
|
48
|
+
const config_1 = require("../../config");
|
|
49
|
+
const REGISTRY_PATH = path.join(config_1.PATHS.globalRoot, "projects.json");
|
|
50
|
+
function loadRegistry() {
|
|
51
|
+
try {
|
|
52
|
+
const raw = fs.readFileSync(REGISTRY_PATH, "utf-8");
|
|
53
|
+
return JSON.parse(raw);
|
|
54
|
+
}
|
|
55
|
+
catch (_a) {
|
|
56
|
+
return [];
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function saveRegistry(entries) {
|
|
60
|
+
fs.mkdirSync(path.dirname(REGISTRY_PATH), { recursive: true });
|
|
61
|
+
fs.writeFileSync(REGISTRY_PATH, `${JSON.stringify(entries, null, 2)}\n`);
|
|
62
|
+
}
|
|
63
|
+
function registerProject(entry) {
|
|
64
|
+
const entries = loadRegistry();
|
|
65
|
+
const idx = entries.findIndex((e) => e.root === entry.root);
|
|
66
|
+
if (idx >= 0) {
|
|
67
|
+
entries[idx] = entry;
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
entries.push(entry);
|
|
71
|
+
}
|
|
72
|
+
saveRegistry(entries);
|
|
73
|
+
}
|
|
74
|
+
function listProjects() {
|
|
75
|
+
return loadRegistry();
|
|
76
|
+
}
|
|
77
|
+
function getProject(root) {
|
|
78
|
+
return loadRegistry().find((e) => e.root === root);
|
|
79
|
+
}
|
|
80
|
+
function removeProject(root) {
|
|
81
|
+
const entries = loadRegistry().filter((e) => e.root !== root);
|
|
82
|
+
saveRegistry(entries);
|
|
83
|
+
}
|
|
@@ -38,70 +38,45 @@ exports.ensureProjectPaths = ensureProjectPaths;
|
|
|
38
38
|
const fs = __importStar(require("node:fs"));
|
|
39
39
|
const path = __importStar(require("node:path"));
|
|
40
40
|
const config_1 = require("../../config");
|
|
41
|
+
/**
|
|
42
|
+
* Find the project root for a given directory.
|
|
43
|
+
* Looks for .git to determine the project boundary.
|
|
44
|
+
* Falls back to the directory itself if no .git found.
|
|
45
|
+
*/
|
|
41
46
|
function findProjectRoot(startDir = process.cwd()) {
|
|
42
47
|
const start = path.resolve(startDir);
|
|
43
|
-
//
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
48
|
+
// Walk up to find .git
|
|
49
|
+
let dir = start;
|
|
50
|
+
while (true) {
|
|
51
|
+
if (fs.existsSync(path.join(dir, ".git")))
|
|
52
|
+
return dir;
|
|
53
|
+
const parent = path.dirname(dir);
|
|
54
|
+
if (parent === dir)
|
|
55
|
+
break; // reached filesystem root
|
|
56
|
+
dir = parent;
|
|
49
57
|
}
|
|
50
|
-
//
|
|
58
|
+
// No .git found — treat startDir as root
|
|
51
59
|
return start;
|
|
52
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Returns centralized paths for storage.
|
|
63
|
+
* The `root` field is the directory being indexed/searched.
|
|
64
|
+
* All storage paths point to ~/.gmax/ (centralized).
|
|
65
|
+
*/
|
|
53
66
|
function ensureProjectPaths(startDir = process.cwd(), options) {
|
|
54
|
-
|
|
55
|
-
const root = (_a = findProjectRoot(startDir)) !== null && _a !== void 0 ? _a : path.resolve(startDir);
|
|
56
|
-
const osgrepDir = path.join(root, ".osgrep");
|
|
57
|
-
const lancedbDir = path.join(osgrepDir, "lancedb");
|
|
58
|
-
const cacheDir = path.join(osgrepDir, "cache");
|
|
59
|
-
const lmdbPath = path.join(cacheDir, "meta.lmdb");
|
|
60
|
-
const configPath = path.join(osgrepDir, "config.json");
|
|
67
|
+
const root = findProjectRoot(startDir);
|
|
61
68
|
if (!(options === null || options === void 0 ? void 0 : options.dryRun)) {
|
|
62
|
-
|
|
69
|
+
// Ensure centralized directories exist
|
|
70
|
+
for (const dir of [config_1.PATHS.lancedbDir, config_1.PATHS.cacheDir]) {
|
|
63
71
|
fs.mkdirSync(dir, { recursive: true });
|
|
64
|
-
}
|
|
65
|
-
ensureGitignoreEntry(root);
|
|
72
|
+
}
|
|
66
73
|
}
|
|
67
|
-
return {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
.includes(entry);
|
|
76
|
-
}
|
|
77
|
-
catch (_a) {
|
|
78
|
-
return false;
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
function ensureGitignoreEntry(root) {
|
|
82
|
-
// Only add when inside a git repo.
|
|
83
|
-
if (!fs.existsSync(path.join(root, ".git")))
|
|
84
|
-
return;
|
|
85
|
-
const entry = ".osgrep";
|
|
86
|
-
// Check .git/info/exclude first
|
|
87
|
-
const excludePath = path.join(root, ".git", "info", "exclude");
|
|
88
|
-
if (fileContainsEntry(excludePath, entry))
|
|
89
|
-
return;
|
|
90
|
-
// Check .gitignore
|
|
91
|
-
const gitignorePath = path.join(root, ".gitignore");
|
|
92
|
-
if (fileContainsEntry(gitignorePath, entry))
|
|
93
|
-
return;
|
|
94
|
-
// Add to .gitignore
|
|
95
|
-
let contents = "";
|
|
96
|
-
try {
|
|
97
|
-
contents = fs.readFileSync(gitignorePath, "utf-8");
|
|
98
|
-
}
|
|
99
|
-
catch (_a) {
|
|
100
|
-
// ignore missing file; will create below
|
|
101
|
-
}
|
|
102
|
-
const needsNewline = contents.length > 0 && !contents.endsWith("\n");
|
|
103
|
-
const prefix = needsNewline ? "\n" : "";
|
|
104
|
-
fs.writeFileSync(gitignorePath, `${contents}${prefix}${entry}\n`, {
|
|
105
|
-
encoding: "utf-8",
|
|
106
|
-
});
|
|
74
|
+
return {
|
|
75
|
+
root,
|
|
76
|
+
dataDir: config_1.PATHS.globalRoot,
|
|
77
|
+
lancedbDir: config_1.PATHS.lancedbDir,
|
|
78
|
+
cacheDir: config_1.PATHS.cacheDir,
|
|
79
|
+
lmdbPath: config_1.PATHS.lmdbPath,
|
|
80
|
+
configPath: config_1.PATHS.configPath,
|
|
81
|
+
};
|
|
107
82
|
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Watcher registry — tracks background watcher processes per project.
|
|
4
|
+
* Ensures only one watcher runs per project root.
|
|
5
|
+
*
|
|
6
|
+
* Stored in ~/.gmax/watchers.json
|
|
7
|
+
*/
|
|
8
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
|
+
if (k2 === undefined) k2 = k;
|
|
10
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
11
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
12
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
13
|
+
}
|
|
14
|
+
Object.defineProperty(o, k2, desc);
|
|
15
|
+
}) : (function(o, m, k, k2) {
|
|
16
|
+
if (k2 === undefined) k2 = k;
|
|
17
|
+
o[k2] = m[k];
|
|
18
|
+
}));
|
|
19
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
20
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
21
|
+
}) : function(o, v) {
|
|
22
|
+
o["default"] = v;
|
|
23
|
+
});
|
|
24
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
25
|
+
var ownKeys = function(o) {
|
|
26
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
27
|
+
var ar = [];
|
|
28
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
29
|
+
return ar;
|
|
30
|
+
};
|
|
31
|
+
return ownKeys(o);
|
|
32
|
+
};
|
|
33
|
+
return function (mod) {
|
|
34
|
+
if (mod && mod.__esModule) return mod;
|
|
35
|
+
var result = {};
|
|
36
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
37
|
+
__setModuleDefault(result, mod);
|
|
38
|
+
return result;
|
|
39
|
+
};
|
|
40
|
+
})();
|
|
41
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
+
exports.isProcessRunning = isProcessRunning;
|
|
43
|
+
exports.registerWatcher = registerWatcher;
|
|
44
|
+
exports.unregisterWatcher = unregisterWatcher;
|
|
45
|
+
exports.getWatcherForProject = getWatcherForProject;
|
|
46
|
+
exports.listWatchers = listWatchers;
|
|
47
|
+
const fs = __importStar(require("node:fs"));
|
|
48
|
+
const path = __importStar(require("node:path"));
|
|
49
|
+
const config_1 = require("../../config");
|
|
50
|
+
const REGISTRY_PATH = path.join(config_1.PATHS.globalRoot, "watchers.json");
|
|
51
|
+
function loadRegistry() {
|
|
52
|
+
try {
|
|
53
|
+
const raw = fs.readFileSync(REGISTRY_PATH, "utf-8");
|
|
54
|
+
return JSON.parse(raw);
|
|
55
|
+
}
|
|
56
|
+
catch (_a) {
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function saveRegistry(entries) {
|
|
61
|
+
fs.mkdirSync(path.dirname(REGISTRY_PATH), { recursive: true });
|
|
62
|
+
fs.writeFileSync(REGISTRY_PATH, `${JSON.stringify(entries, null, 2)}\n`);
|
|
63
|
+
}
|
|
64
|
+
function isProcessRunning(pid) {
|
|
65
|
+
try {
|
|
66
|
+
process.kill(pid, 0);
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
69
|
+
catch (_a) {
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function registerWatcher(info) {
|
|
74
|
+
const entries = loadRegistry().filter((e) => e.projectRoot !== info.projectRoot);
|
|
75
|
+
entries.push(info);
|
|
76
|
+
saveRegistry(entries);
|
|
77
|
+
}
|
|
78
|
+
function unregisterWatcher(pid) {
|
|
79
|
+
const entries = loadRegistry().filter((e) => e.pid !== pid);
|
|
80
|
+
saveRegistry(entries);
|
|
81
|
+
}
|
|
82
|
+
function getWatcherForProject(projectRoot) {
|
|
83
|
+
const entries = loadRegistry();
|
|
84
|
+
const match = entries.find((e) => e.projectRoot === projectRoot);
|
|
85
|
+
if (match && isProcessRunning(match.pid))
|
|
86
|
+
return match;
|
|
87
|
+
// Clean stale entry
|
|
88
|
+
if (match) {
|
|
89
|
+
saveRegistry(entries.filter((e) => e.pid !== match.pid));
|
|
90
|
+
}
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
function listWatchers() {
|
|
94
|
+
const entries = loadRegistry();
|
|
95
|
+
const active = entries.filter((e) => isProcessRunning(e.pid));
|
|
96
|
+
if (active.length !== entries.length) {
|
|
97
|
+
saveRegistry(active);
|
|
98
|
+
}
|
|
99
|
+
return active;
|
|
100
|
+
}
|
|
@@ -44,8 +44,8 @@ function loadSkipIds() {
|
|
|
44
44
|
if (SKIP_IDS)
|
|
45
45
|
return SKIP_IDS;
|
|
46
46
|
// Check local models first (same logic as orchestrator)
|
|
47
|
-
const PROJECT_ROOT = process.env.
|
|
48
|
-
? path.resolve(process.env.
|
|
47
|
+
const PROJECT_ROOT = process.env.GMAX_PROJECT_ROOT
|
|
48
|
+
? path.resolve(process.env.GMAX_PROJECT_ROOT)
|
|
49
49
|
: process.cwd();
|
|
50
50
|
const localModels = path.join(PROJECT_ROOT, "models");
|
|
51
51
|
const localColbert = path.join(localModels, ...config_1.MODEL_IDS.colbert.split("/"));
|
|
@@ -50,7 +50,7 @@ const transformers_1 = require("@huggingface/transformers");
|
|
|
50
50
|
const config_1 = require("../../config");
|
|
51
51
|
// Configuration
|
|
52
52
|
const HOMEDIR = os.homedir();
|
|
53
|
-
const CACHE_DIR = path.join(HOMEDIR, ".
|
|
53
|
+
const CACHE_DIR = path.join(HOMEDIR, ".gmax", "models");
|
|
54
54
|
transformers_1.env.cacheDir = CACHE_DIR;
|
|
55
55
|
transformers_1.env.allowLocalModels = true;
|
|
56
56
|
transformers_1.env.allowRemoteModels = true;
|
|
@@ -91,7 +91,7 @@ function downloadModelWithTimeout(modelId, dtype) {
|
|
|
91
91
|
function downloadExtraFile(modelId, filename) {
|
|
92
92
|
return __awaiter(this, void 0, void 0, function* () {
|
|
93
93
|
const url = `https://huggingface.co/${modelId}/resolve/main/${filename}`;
|
|
94
|
-
// Construct path: ~/.
|
|
94
|
+
// Construct path: ~/.gmax/models/ryandono/gmax-colbert-q8/skiplist.json
|
|
95
95
|
const destDir = path.join(CACHE_DIR, ...modelId.split("/"));
|
|
96
96
|
const destPath = path.join(destDir, filename);
|
|
97
97
|
if (!fs.existsSync(destDir)) {
|
|
@@ -50,8 +50,8 @@ const config_1 = require("../../../config");
|
|
|
50
50
|
const colbert_tokenizer_1 = require("../colbert-tokenizer");
|
|
51
51
|
const CACHE_DIR = config_1.PATHS.models;
|
|
52
52
|
const ONNX_THREADS = 1;
|
|
53
|
-
const LOG_MODELS = process.env.
|
|
54
|
-
process.env.
|
|
53
|
+
const LOG_MODELS = process.env.GMAX_DEBUG_MODELS === "1" ||
|
|
54
|
+
process.env.GMAX_DEBUG_MODELS === "true";
|
|
55
55
|
const log = (...args) => {
|
|
56
56
|
if (LOG_MODELS)
|
|
57
57
|
console.log(...args);
|
|
@@ -50,17 +50,17 @@ const ort = __importStar(require("onnxruntime-node"));
|
|
|
50
50
|
const config_1 = require("../../../config");
|
|
51
51
|
const CACHE_DIR = config_1.PATHS.models;
|
|
52
52
|
const ONNX_THREADS = 1;
|
|
53
|
-
const LOG_MODELS = process.env.
|
|
54
|
-
process.env.
|
|
53
|
+
const LOG_MODELS = process.env.GMAX_DEBUG_MODELS === "1" ||
|
|
54
|
+
process.env.GMAX_DEBUG_MODELS === "true";
|
|
55
55
|
const log = (...args) => {
|
|
56
56
|
if (LOG_MODELS)
|
|
57
57
|
console.log(...args);
|
|
58
58
|
};
|
|
59
59
|
class GraniteModel {
|
|
60
|
-
constructor() {
|
|
60
|
+
constructor(vectorDim) {
|
|
61
61
|
this.session = null;
|
|
62
62
|
this.tokenizer = null;
|
|
63
|
-
this.vectorDimensions = config_1.CONFIG.VECTOR_DIM;
|
|
63
|
+
this.vectorDimensions = vectorDim !== null && vectorDim !== void 0 ? vectorDim : config_1.CONFIG.VECTOR_DIM;
|
|
64
64
|
}
|
|
65
65
|
resolvePaths() {
|
|
66
66
|
const basePath = path.join(CACHE_DIR, config_1.MODEL_IDS.embed);
|
|
@@ -53,7 +53,7 @@ const http = __importStar(require("node:http"));
|
|
|
53
53
|
const MLX_PORT = parseInt(process.env.MLX_EMBED_PORT || "8100", 10);
|
|
54
54
|
const MLX_HOST = "127.0.0.1";
|
|
55
55
|
const MLX_TIMEOUT_MS = 10000;
|
|
56
|
-
const EMBED_MODE = process.env.
|
|
56
|
+
const EMBED_MODE = process.env.GMAX_EMBED_MODE || "auto";
|
|
57
57
|
let mlxAvailable = null;
|
|
58
58
|
let lastCheck = 0;
|
|
59
59
|
const CHECK_INTERVAL_MS = 30000;
|
|
@@ -57,8 +57,8 @@ const colbert_1 = require("./embeddings/colbert");
|
|
|
57
57
|
const granite_1 = require("./embeddings/granite");
|
|
58
58
|
const mlx_client_1 = require("./embeddings/mlx-client");
|
|
59
59
|
const CACHE_DIR = config_1.PATHS.models;
|
|
60
|
-
const LOG_MODELS = process.env.
|
|
61
|
-
process.env.
|
|
60
|
+
const LOG_MODELS = process.env.GMAX_DEBUG_MODELS === "1" ||
|
|
61
|
+
process.env.GMAX_DEBUG_MODELS === "true";
|
|
62
62
|
const log = (...args) => {
|
|
63
63
|
if (LOG_MODELS)
|
|
64
64
|
console.log(...args);
|
|
@@ -66,8 +66,8 @@ const log = (...args) => {
|
|
|
66
66
|
transformers_1.env.cacheDir = CACHE_DIR;
|
|
67
67
|
transformers_1.env.allowLocalModels = true;
|
|
68
68
|
transformers_1.env.allowRemoteModels = true;
|
|
69
|
-
const PROJECT_ROOT = process.env.
|
|
70
|
-
? path.resolve(process.env.
|
|
69
|
+
const PROJECT_ROOT = process.env.GMAX_PROJECT_ROOT
|
|
70
|
+
? path.resolve(process.env.GMAX_PROJECT_ROOT)
|
|
71
71
|
: process.cwd();
|
|
72
72
|
const LOCAL_MODELS = path.join(PROJECT_ROOT, "models");
|
|
73
73
|
if (fs.existsSync(LOCAL_MODELS)) {
|
|
@@ -75,13 +75,13 @@ if (fs.existsSync(LOCAL_MODELS)) {
|
|
|
75
75
|
log(`Worker: Using local models from ${LOCAL_MODELS}`);
|
|
76
76
|
}
|
|
77
77
|
class WorkerOrchestrator {
|
|
78
|
-
constructor() {
|
|
79
|
-
this.granite = new granite_1.GraniteModel();
|
|
78
|
+
constructor(vectorDim) {
|
|
80
79
|
this.colbert = new colbert_1.ColbertModel();
|
|
81
80
|
this.chunker = new chunker_1.TreeSitterChunker();
|
|
82
81
|
this.skeletonizer = new skeleton_1.Skeletonizer();
|
|
83
82
|
this.initPromise = null;
|
|
84
|
-
this.vectorDimensions = config_1.CONFIG.VECTOR_DIM;
|
|
83
|
+
this.vectorDimensions = vectorDim !== null && vectorDim !== void 0 ? vectorDim : config_1.CONFIG.VECTOR_DIM;
|
|
84
|
+
this.granite = new granite_1.GraniteModel(this.vectorDimensions);
|
|
85
85
|
}
|
|
86
86
|
ensureReady() {
|
|
87
87
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -110,7 +110,7 @@ class WorkerOrchestrator {
|
|
|
110
110
|
return [];
|
|
111
111
|
yield this.ensureReady();
|
|
112
112
|
const results = [];
|
|
113
|
-
const envBatch = Number.parseInt((_a = process.env.
|
|
113
|
+
const envBatch = Number.parseInt((_a = process.env.GMAX_WORKER_BATCH_SIZE) !== null && _a !== void 0 ? _a : "", 10);
|
|
114
114
|
const BATCH_SIZE = Number.isFinite(envBatch) && envBatch > 0
|
|
115
115
|
? Math.max(4, Math.min(16, envBatch))
|
|
116
116
|
: 16;
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -76,7 +76,7 @@ function reviveProcessFileResult(result) {
|
|
|
76
76
|
}
|
|
77
77
|
const TASK_TIMEOUT_MS = (() => {
|
|
78
78
|
var _a;
|
|
79
|
-
const fromEnv = Number.parseInt((_a = process.env.
|
|
79
|
+
const fromEnv = Number.parseInt((_a = process.env.GMAX_WORKER_TASK_TIMEOUT_MS) !== null && _a !== void 0 ? _a : "", 10);
|
|
80
80
|
if (Number.isFinite(fromEnv) && fromEnv > 0)
|
|
81
81
|
return fromEnv;
|
|
82
82
|
return 120000;
|
|
@@ -13,7 +13,10 @@ exports.default = processFile;
|
|
|
13
13
|
exports.encodeQuery = encodeQuery;
|
|
14
14
|
exports.rerank = rerank;
|
|
15
15
|
const orchestrator_1 = require("./orchestrator");
|
|
16
|
-
const
|
|
16
|
+
const vectorDim = process.env.GMAX_VECTOR_DIM
|
|
17
|
+
? Number.parseInt(process.env.GMAX_VECTOR_DIM, 10)
|
|
18
|
+
: undefined;
|
|
19
|
+
const orchestrator = new orchestrator_1.WorkerOrchestrator(vectorDim);
|
|
17
20
|
function processFile(input, onProgress) {
|
|
18
21
|
return __awaiter(this, void 0, void 0, function* () {
|
|
19
22
|
return orchestrator.processFile(input, onProgress);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "grepmax",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"author": "Robert Owens <robowens@me.com>",
|
|
5
5
|
"homepage": "https://github.com/reowens/grepmax",
|
|
6
6
|
"bugs": {
|
|
@@ -12,27 +12,10 @@
|
|
|
12
12
|
},
|
|
13
13
|
"main": "index.js",
|
|
14
14
|
"bin": {
|
|
15
|
-
"
|
|
16
|
-
},
|
|
17
|
-
"scripts": {
|
|
18
|
-
"prebuild": "mkdir -p dist",
|
|
19
|
-
"build": "tsc",
|
|
20
|
-
"postbuild": "chmod +x dist/index.js",
|
|
21
|
-
"dev": "npx tsc && node dist/index.js",
|
|
22
|
-
"test": "vitest run",
|
|
23
|
-
"test:watch": "vitest",
|
|
24
|
-
"benchmark": "./run-benchmark.sh",
|
|
25
|
-
"benchmark:index": "./run-benchmark.sh $HOME/osgrep-benchmarks --index",
|
|
26
|
-
"benchmark:agent": "npx tsx src/bench/benchmark-agent.ts",
|
|
27
|
-
"benchmark:chart": "npx tsx src/bench/generate-benchmark-chart.ts",
|
|
28
|
-
"format": "biome check --write .",
|
|
29
|
-
"format:check": "biome check .",
|
|
30
|
-
"lint": "biome lint .",
|
|
31
|
-
"typecheck": "tsc --noEmit",
|
|
32
|
-
"prepublishOnly": "pnpm build"
|
|
15
|
+
"gmax": "dist/index.js"
|
|
33
16
|
},
|
|
34
17
|
"keywords": [
|
|
35
|
-
"
|
|
18
|
+
"grepmax",
|
|
36
19
|
"grep",
|
|
37
20
|
"glob",
|
|
38
21
|
"search"
|
|
@@ -76,5 +59,21 @@
|
|
|
76
59
|
"ts-node": "^10.9.2",
|
|
77
60
|
"typescript": "^5.9.3",
|
|
78
61
|
"vitest": "^1.6.1"
|
|
62
|
+
},
|
|
63
|
+
"scripts": {
|
|
64
|
+
"prebuild": "mkdir -p dist",
|
|
65
|
+
"build": "tsc",
|
|
66
|
+
"postbuild": "chmod +x dist/index.js",
|
|
67
|
+
"dev": "npx tsc && node dist/index.js",
|
|
68
|
+
"test": "vitest run",
|
|
69
|
+
"test:watch": "vitest",
|
|
70
|
+
"benchmark": "./run-benchmark.sh",
|
|
71
|
+
"benchmark:index": "./run-benchmark.sh $HOME/gmax-benchmarks --index",
|
|
72
|
+
"benchmark:agent": "npx tsx src/bench/benchmark-agent.ts",
|
|
73
|
+
"benchmark:chart": "npx tsx src/bench/generate-benchmark-chart.ts",
|
|
74
|
+
"format": "biome check --write .",
|
|
75
|
+
"format:check": "biome check .",
|
|
76
|
+
"lint": "biome lint .",
|
|
77
|
+
"typecheck": "tsc --noEmit"
|
|
79
78
|
}
|
|
80
|
-
}
|
|
79
|
+
}
|