grepmax 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of grepmax might be problematic. Click here for more details.
- package/LICENSE +202 -0
- package/NOTICE +33 -0
- package/README.md +375 -0
- package/dist/commands/claude-code.js +60 -0
- package/dist/commands/codex.js +98 -0
- package/dist/commands/doctor.js +92 -0
- package/dist/commands/droid.js +189 -0
- package/dist/commands/index.js +125 -0
- package/dist/commands/list.js +120 -0
- package/dist/commands/mcp.js +572 -0
- package/dist/commands/opencode.js +199 -0
- package/dist/commands/search.js +539 -0
- package/dist/commands/serve.js +512 -0
- package/dist/commands/setup.js +162 -0
- package/dist/commands/skeleton.js +288 -0
- package/dist/commands/symbols.js +129 -0
- package/dist/commands/trace.js +50 -0
- package/dist/commands/verify.js +174 -0
- package/dist/config.js +120 -0
- package/dist/eval.js +618 -0
- package/dist/index.js +82 -0
- package/dist/lib/core/languages.js +237 -0
- package/dist/lib/graph/graph-builder.js +105 -0
- package/dist/lib/index/chunker.js +663 -0
- package/dist/lib/index/grammar-loader.js +110 -0
- package/dist/lib/index/ignore-patterns.js +63 -0
- package/dist/lib/index/index-config.js +86 -0
- package/dist/lib/index/sync-helpers.js +97 -0
- package/dist/lib/index/syncer.js +396 -0
- package/dist/lib/index/walker.js +164 -0
- package/dist/lib/index/watcher.js +245 -0
- package/dist/lib/output/formatter.js +161 -0
- package/dist/lib/output/json-formatter.js +6 -0
- package/dist/lib/search/intent.js +23 -0
- package/dist/lib/search/searcher.js +475 -0
- package/dist/lib/setup/model-loader.js +107 -0
- package/dist/lib/setup/setup-helpers.js +106 -0
- package/dist/lib/skeleton/body-fields.js +175 -0
- package/dist/lib/skeleton/index.js +24 -0
- package/dist/lib/skeleton/retriever.js +36 -0
- package/dist/lib/skeleton/skeletonizer.js +483 -0
- package/dist/lib/skeleton/summary-formatter.js +90 -0
- package/dist/lib/store/meta-cache.js +143 -0
- package/dist/lib/store/types.js +2 -0
- package/dist/lib/store/vector-db.js +340 -0
- package/dist/lib/utils/cleanup.js +33 -0
- package/dist/lib/utils/exit.js +38 -0
- package/dist/lib/utils/file-utils.js +131 -0
- package/dist/lib/utils/filter-builder.js +17 -0
- package/dist/lib/utils/formatter.js +230 -0
- package/dist/lib/utils/git.js +83 -0
- package/dist/lib/utils/lock.js +157 -0
- package/dist/lib/utils/project-root.js +107 -0
- package/dist/lib/utils/server-registry.js +97 -0
- package/dist/lib/workers/colbert-math.js +107 -0
- package/dist/lib/workers/colbert-tokenizer.js +113 -0
- package/dist/lib/workers/download-worker.js +169 -0
- package/dist/lib/workers/embeddings/colbert.js +213 -0
- package/dist/lib/workers/embeddings/granite.js +180 -0
- package/dist/lib/workers/embeddings/mlx-client.js +144 -0
- package/dist/lib/workers/orchestrator.js +350 -0
- package/dist/lib/workers/pool.js +373 -0
- package/dist/lib/workers/process-child.js +92 -0
- package/dist/lib/workers/worker.js +31 -0
- package/package.json +80 -0
- package/plugins/osgrep/.claude-plugin/plugin.json +20 -0
- package/plugins/osgrep/hooks/start.js +92 -0
- package/plugins/osgrep/hooks/stop.js +3 -0
- package/plugins/osgrep/hooks.json +26 -0
- package/plugins/osgrep/skills/osgrep/SKILL.md +82 -0
|
@@ -0,0 +1,512 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.serve = void 0;
|
|
46
|
+
const node_child_process_1 = require("node:child_process");
|
|
47
|
+
const fs = __importStar(require("node:fs"));
|
|
48
|
+
const http = __importStar(require("node:http"));
|
|
49
|
+
const path = __importStar(require("node:path"));
|
|
50
|
+
const commander_1 = require("commander");
|
|
51
|
+
const config_1 = require("../config");
|
|
52
|
+
const grammar_loader_1 = require("../lib/index/grammar-loader");
|
|
53
|
+
const index_config_1 = require("../lib/index/index-config");
|
|
54
|
+
const sync_helpers_1 = require("../lib/index/sync-helpers");
|
|
55
|
+
const syncer_1 = require("../lib/index/syncer");
|
|
56
|
+
const watcher_1 = require("../lib/index/watcher");
|
|
57
|
+
const searcher_1 = require("../lib/search/searcher");
|
|
58
|
+
const setup_helpers_1 = require("../lib/setup/setup-helpers");
|
|
59
|
+
const meta_cache_1 = require("../lib/store/meta-cache");
|
|
60
|
+
const vector_db_1 = require("../lib/store/vector-db");
|
|
61
|
+
const exit_1 = require("../lib/utils/exit");
|
|
62
|
+
const project_root_1 = require("../lib/utils/project-root");
|
|
63
|
+
const server_registry_1 = require("../lib/utils/server-registry");
|
|
64
|
+
function isMlxServerUp() {
|
|
65
|
+
const port = parseInt(process.env.MLX_EMBED_PORT || "8100", 10);
|
|
66
|
+
return new Promise((resolve) => {
|
|
67
|
+
const req = http.get({ hostname: "127.0.0.1", port, path: "/health", timeout: 2000 }, (res) => {
|
|
68
|
+
res.resume();
|
|
69
|
+
resolve(res.statusCode === 200);
|
|
70
|
+
});
|
|
71
|
+
req.on("error", () => resolve(false));
|
|
72
|
+
req.on("timeout", () => {
|
|
73
|
+
req.destroy();
|
|
74
|
+
resolve(false);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
function startMlxServer(mlxModel) {
|
|
79
|
+
// Look for mlx-embed-server relative to the osgrep package
|
|
80
|
+
const candidates = [
|
|
81
|
+
path.resolve(__dirname, "../../mlx-embed-server"),
|
|
82
|
+
path.resolve(__dirname, "../mlx-embed-server"),
|
|
83
|
+
];
|
|
84
|
+
const serverDir = candidates.find((d) => fs.existsSync(path.join(d, "server.py")));
|
|
85
|
+
if (!serverDir)
|
|
86
|
+
return null;
|
|
87
|
+
const logPath = "/tmp/mlx-embed-server.log";
|
|
88
|
+
const out = fs.openSync(logPath, "a");
|
|
89
|
+
const env = Object.assign({}, process.env);
|
|
90
|
+
if (mlxModel) {
|
|
91
|
+
env.MLX_EMBED_MODEL = mlxModel;
|
|
92
|
+
}
|
|
93
|
+
const child = (0, node_child_process_1.spawn)("uv", ["run", "python", "server.py"], {
|
|
94
|
+
cwd: serverDir,
|
|
95
|
+
detached: true,
|
|
96
|
+
stdio: ["ignore", out, out],
|
|
97
|
+
env,
|
|
98
|
+
});
|
|
99
|
+
child.unref();
|
|
100
|
+
return child;
|
|
101
|
+
}
|
|
102
|
+
exports.serve = new commander_1.Command("serve")
|
|
103
|
+
.description("Run osgrep as a background server with live indexing")
|
|
104
|
+
.option("-p, --port <port>", "Port to listen on", process.env.OSGREP_PORT || "4444")
|
|
105
|
+
.option("-b, --background", "Run in background", false)
|
|
106
|
+
.option("--cpu", "Use CPU-only embeddings (skip MLX GPU server)", false)
|
|
107
|
+
.option("--no-idle-timeout", "Disable the 30-minute idle shutdown", false)
|
|
108
|
+
.action((_args, cmd) => __awaiter(void 0, void 0, void 0, function* () {
|
|
109
|
+
var _a, _b;
|
|
110
|
+
const options = cmd.optsWithGlobals();
|
|
111
|
+
let port = parseInt(options.port, 10);
|
|
112
|
+
const startPort = port;
|
|
113
|
+
const projectRoot = (_a = (0, project_root_1.findProjectRoot)(process.cwd())) !== null && _a !== void 0 ? _a : process.cwd();
|
|
114
|
+
// Check if already running
|
|
115
|
+
const existing = (0, server_registry_1.getServerForProject)(projectRoot);
|
|
116
|
+
if (existing && (0, server_registry_1.isProcessRunning)(existing.pid)) {
|
|
117
|
+
console.log(`Server already running for ${projectRoot} (PID: ${existing.pid}, Port: ${existing.port})`);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
if (options.background) {
|
|
121
|
+
const args = process.argv
|
|
122
|
+
.slice(2)
|
|
123
|
+
.filter((arg) => arg !== "-b" && arg !== "--background");
|
|
124
|
+
const logDir = path.join(config_1.PATHS.globalRoot, "logs");
|
|
125
|
+
fs.mkdirSync(logDir, { recursive: true });
|
|
126
|
+
const safeName = path
|
|
127
|
+
.basename(projectRoot)
|
|
128
|
+
.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
129
|
+
const logFile = path.join(logDir, `server-${safeName}.log`);
|
|
130
|
+
const out = fs.openSync(logFile, "a");
|
|
131
|
+
const err = fs.openSync(logFile, "a");
|
|
132
|
+
const child = (0, node_child_process_1.spawn)(process.argv[0], [process.argv[1], ...args], {
|
|
133
|
+
detached: true,
|
|
134
|
+
stdio: ["ignore", out, err],
|
|
135
|
+
cwd: process.cwd(),
|
|
136
|
+
env: Object.assign(Object.assign({}, process.env), { OSGREP_BACKGROUND: "true" }),
|
|
137
|
+
});
|
|
138
|
+
child.unref();
|
|
139
|
+
console.log(`Started background server (PID: ${child.pid})`);
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
const paths = (0, project_root_1.ensureProjectPaths)(projectRoot);
|
|
143
|
+
const projectName = path.basename(projectRoot);
|
|
144
|
+
// Propagate project root to worker processes
|
|
145
|
+
process.env.OSGREP_PROJECT_ROOT = projectRoot;
|
|
146
|
+
// Determine embed mode: --cpu flag overrides, then config, then default
|
|
147
|
+
// Default to GPU on Apple Silicon, CPU everywhere else
|
|
148
|
+
const isAppleSilicon = process.arch === "arm64" && process.platform === "darwin";
|
|
149
|
+
const indexConfig = (0, index_config_1.readIndexConfig)(paths.configPath);
|
|
150
|
+
const useGpu = options.cpu
|
|
151
|
+
? false
|
|
152
|
+
: ((_b = indexConfig === null || indexConfig === void 0 ? void 0 : indexConfig.embedMode) !== null && _b !== void 0 ? _b : (isAppleSilicon ? "gpu" : "cpu")) === "gpu";
|
|
153
|
+
const mlxModel = indexConfig === null || indexConfig === void 0 ? void 0 : indexConfig.mlxModel;
|
|
154
|
+
// MLX GPU embed server — started when GPU mode is active.
|
|
155
|
+
let mlxChild = null;
|
|
156
|
+
if (!useGpu) {
|
|
157
|
+
console.log(`[serve:${projectName}] CPU-only mode`);
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
const mlxUp = yield isMlxServerUp();
|
|
161
|
+
if (mlxUp) {
|
|
162
|
+
console.log(`[serve:${projectName}] MLX GPU server already running`);
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
mlxChild = startMlxServer(mlxModel);
|
|
166
|
+
if (mlxChild) {
|
|
167
|
+
console.log(`[serve] Starting MLX GPU embed server (PID: ${mlxChild.pid})${mlxModel ? ` [${mlxModel}]` : ""}`);
|
|
168
|
+
let ready = false;
|
|
169
|
+
for (let i = 0; i < 30; i++) {
|
|
170
|
+
yield new Promise((r) => setTimeout(r, 1000));
|
|
171
|
+
if (yield isMlxServerUp()) {
|
|
172
|
+
console.log(`[serve:${projectName}] MLX GPU server ready`);
|
|
173
|
+
ready = true;
|
|
174
|
+
break;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
if (!ready) {
|
|
178
|
+
console.error(`[serve:${projectName}] MLX GPU server failed to start. Run with --cpu to use CPU embeddings.`);
|
|
179
|
+
process.exitCode = 1;
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
else {
|
|
184
|
+
console.error(`[serve:${projectName}] MLX server not found. Run with --cpu to use CPU embeddings.`);
|
|
185
|
+
process.exitCode = 1;
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
try {
|
|
191
|
+
yield (0, setup_helpers_1.ensureSetup)();
|
|
192
|
+
yield (0, grammar_loader_1.ensureGrammars)(console.log, { silent: true });
|
|
193
|
+
// Initial sync is self-contained (creates+closes its own VectorDB+MetaCache).
|
|
194
|
+
if (!process.env.OSGREP_BACKGROUND) {
|
|
195
|
+
const { spinner, onProgress } = (0, sync_helpers_1.createIndexingSpinner)(projectRoot, "Indexing before starting server...");
|
|
196
|
+
try {
|
|
197
|
+
yield (0, syncer_1.initialSync)({ projectRoot, onProgress });
|
|
198
|
+
spinner.succeed("Initial index ready. Starting server...");
|
|
199
|
+
}
|
|
200
|
+
catch (e) {
|
|
201
|
+
spinner.fail("Indexing failed");
|
|
202
|
+
throw e;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
else {
|
|
206
|
+
yield (0, syncer_1.initialSync)({ projectRoot });
|
|
207
|
+
}
|
|
208
|
+
// Open long-lived resources for serving + watching.
|
|
209
|
+
const vectorDb = new vector_db_1.VectorDB(paths.lancedbDir);
|
|
210
|
+
const metaCache = new meta_cache_1.MetaCache(paths.lmdbPath);
|
|
211
|
+
const searcher = new searcher_1.Searcher(vectorDb);
|
|
212
|
+
// Start live file watcher
|
|
213
|
+
let fileWatcher = (0, watcher_1.startWatcher)({
|
|
214
|
+
projectRoot,
|
|
215
|
+
vectorDb,
|
|
216
|
+
metaCache,
|
|
217
|
+
osgrepDir: paths.osgrepDir,
|
|
218
|
+
onReindex: (files, durationMs) => {
|
|
219
|
+
console.log(`[watch:${projectName}] Reindexed ${files} file${files !== 1 ? "s" : ""} (${(durationMs / 1000).toFixed(1)}s)`);
|
|
220
|
+
},
|
|
221
|
+
});
|
|
222
|
+
console.log(`[serve:${projectName}] File watcher active`);
|
|
223
|
+
// Idle timeout: shut down if no searches for 30 minutes
|
|
224
|
+
// Disabled when started by MCP server (--no-idle-timeout)
|
|
225
|
+
let lastActivity = Date.now();
|
|
226
|
+
if (options.idleTimeout) {
|
|
227
|
+
const IDLE_TIMEOUT_MS = 30 * 60 * 1000;
|
|
228
|
+
const idleCheck = setInterval(() => {
|
|
229
|
+
if (Date.now() - lastActivity > IDLE_TIMEOUT_MS) {
|
|
230
|
+
console.log(`[serve:${projectName}] Idle timeout reached, shutting down.`);
|
|
231
|
+
clearInterval(idleCheck);
|
|
232
|
+
process.kill(process.pid, "SIGTERM");
|
|
233
|
+
}
|
|
234
|
+
}, 60000);
|
|
235
|
+
idleCheck.unref();
|
|
236
|
+
}
|
|
237
|
+
const server = http.createServer((req, res) => __awaiter(void 0, void 0, void 0, function* () {
|
|
238
|
+
var _a, _b, _c, _d, _e;
|
|
239
|
+
try {
|
|
240
|
+
if (req.method === "GET" && req.url === "/health") {
|
|
241
|
+
lastActivity = Date.now();
|
|
242
|
+
res.statusCode = 200;
|
|
243
|
+
res.setHeader("Content-Type", "application/json");
|
|
244
|
+
res.end(JSON.stringify({ status: "ok" }));
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
if (req.method === "GET" && req.url === "/stats") {
|
|
248
|
+
try {
|
|
249
|
+
const dbStats = yield vectorDb.getStats();
|
|
250
|
+
const cfg = (0, index_config_1.readIndexConfig)(paths.configPath);
|
|
251
|
+
const stats = {
|
|
252
|
+
files: dbStats.chunks > 0
|
|
253
|
+
? yield vectorDb.getDistinctFileCount()
|
|
254
|
+
: 0,
|
|
255
|
+
chunks: dbStats.chunks,
|
|
256
|
+
totalBytes: dbStats.totalBytes,
|
|
257
|
+
vectorDim: (_a = cfg === null || cfg === void 0 ? void 0 : cfg.vectorDim) !== null && _a !== void 0 ? _a : null,
|
|
258
|
+
embedMode: (_b = cfg === null || cfg === void 0 ? void 0 : cfg.embedMode) !== null && _b !== void 0 ? _b : (isAppleSilicon ? "gpu" : "cpu"),
|
|
259
|
+
model: (_c = cfg === null || cfg === void 0 ? void 0 : cfg.embedModel) !== null && _c !== void 0 ? _c : null,
|
|
260
|
+
mlxModel: (_d = cfg === null || cfg === void 0 ? void 0 : cfg.mlxModel) !== null && _d !== void 0 ? _d : null,
|
|
261
|
+
indexedAt: (_e = cfg === null || cfg === void 0 ? void 0 : cfg.indexedAt) !== null && _e !== void 0 ? _e : null,
|
|
262
|
+
watching: fileWatcher !== null,
|
|
263
|
+
};
|
|
264
|
+
res.statusCode = 200;
|
|
265
|
+
res.setHeader("Content-Type", "application/json");
|
|
266
|
+
res.end(JSON.stringify(stats));
|
|
267
|
+
}
|
|
268
|
+
catch (err) {
|
|
269
|
+
res.statusCode = 500;
|
|
270
|
+
res.setHeader("Content-Type", "application/json");
|
|
271
|
+
res.end(JSON.stringify({
|
|
272
|
+
error: (err === null || err === void 0 ? void 0 : err.message) || "stats_failed",
|
|
273
|
+
}));
|
|
274
|
+
}
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
if (req.method === "POST" && req.url === "/search") {
|
|
278
|
+
lastActivity = Date.now();
|
|
279
|
+
const chunks = [];
|
|
280
|
+
let totalSize = 0;
|
|
281
|
+
let aborted = false;
|
|
282
|
+
req.on("data", (chunk) => {
|
|
283
|
+
if (aborted)
|
|
284
|
+
return;
|
|
285
|
+
totalSize += chunk.length;
|
|
286
|
+
if (totalSize > 1000000) {
|
|
287
|
+
aborted = true;
|
|
288
|
+
res.statusCode = 413;
|
|
289
|
+
res.setHeader("Content-Type", "application/json");
|
|
290
|
+
res.end(JSON.stringify({ error: "payload_too_large" }));
|
|
291
|
+
req.destroy();
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
chunks.push(chunk);
|
|
295
|
+
});
|
|
296
|
+
req.on("end", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
297
|
+
if (aborted)
|
|
298
|
+
return;
|
|
299
|
+
try {
|
|
300
|
+
const body = chunks.length
|
|
301
|
+
? JSON.parse(Buffer.concat(chunks).toString("utf-8"))
|
|
302
|
+
: {};
|
|
303
|
+
const query = typeof body.query === "string" ? body.query : "";
|
|
304
|
+
const limit = typeof body.limit === "number" ? body.limit : 10;
|
|
305
|
+
let searchPath = "";
|
|
306
|
+
if (typeof body.path === "string") {
|
|
307
|
+
const resolvedPath = path.resolve(projectRoot, body.path);
|
|
308
|
+
const rootPrefix = projectRoot.endsWith(path.sep)
|
|
309
|
+
? projectRoot
|
|
310
|
+
: `${projectRoot}${path.sep}`;
|
|
311
|
+
// Normalize paths for consistency (Windows/Linux)
|
|
312
|
+
const normalizedRootPrefix = path.normalize(rootPrefix);
|
|
313
|
+
const normalizedResolvedPath = path.normalize(resolvedPath);
|
|
314
|
+
if (normalizedResolvedPath !== projectRoot &&
|
|
315
|
+
!normalizedResolvedPath.startsWith(normalizedRootPrefix)) {
|
|
316
|
+
res.statusCode = 400;
|
|
317
|
+
res.setHeader("Content-Type", "application/json");
|
|
318
|
+
res.end(JSON.stringify({ error: "invalid_path" }));
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
searchPath = path.relative(projectRoot, resolvedPath);
|
|
322
|
+
}
|
|
323
|
+
// Add AbortController for cancellation
|
|
324
|
+
const ac = new AbortController();
|
|
325
|
+
req.on("close", () => {
|
|
326
|
+
if (req.complete)
|
|
327
|
+
return;
|
|
328
|
+
ac.abort();
|
|
329
|
+
});
|
|
330
|
+
res.on("close", () => {
|
|
331
|
+
if (res.writableFinished)
|
|
332
|
+
return;
|
|
333
|
+
ac.abort();
|
|
334
|
+
});
|
|
335
|
+
const result = yield searcher.search(query, limit, { rerank: true }, undefined, searchPath, undefined, // intent
|
|
336
|
+
ac.signal);
|
|
337
|
+
if (ac.signal.aborted) {
|
|
338
|
+
// Request was cancelled, don't write response if possible
|
|
339
|
+
// (Though usually 'close' means the socket is gone anyway)
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
res.statusCode = 200;
|
|
343
|
+
res.setHeader("Content-Type", "application/json");
|
|
344
|
+
res.end(JSON.stringify({ results: result.data }));
|
|
345
|
+
}
|
|
346
|
+
catch (err) {
|
|
347
|
+
if (err instanceof Error && err.name === "AbortError") {
|
|
348
|
+
// Request cancelled
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
res.statusCode = 500;
|
|
352
|
+
res.setHeader("Content-Type", "application/json");
|
|
353
|
+
res.end(JSON.stringify({
|
|
354
|
+
error: (err === null || err === void 0 ? void 0 : err.message) || "search_failed",
|
|
355
|
+
}));
|
|
356
|
+
}
|
|
357
|
+
}));
|
|
358
|
+
req.on("error", (err) => {
|
|
359
|
+
console.error(`[serve:${projectName}] request error:`, err);
|
|
360
|
+
aborted = true;
|
|
361
|
+
});
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
res.statusCode = 404;
|
|
365
|
+
res.end();
|
|
366
|
+
}
|
|
367
|
+
catch (err) {
|
|
368
|
+
console.error(`[serve:${projectName}] request handler error:`, err);
|
|
369
|
+
if (!res.headersSent) {
|
|
370
|
+
res.statusCode = 500;
|
|
371
|
+
res.setHeader("Content-Type", "application/json");
|
|
372
|
+
res.end(JSON.stringify({ error: "internal_error" }));
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
}));
|
|
376
|
+
server.on("error", (e) => {
|
|
377
|
+
if (e.code === "EADDRINUSE") {
|
|
378
|
+
const nextPort = port + 1;
|
|
379
|
+
if (nextPort < startPort + 10) {
|
|
380
|
+
console.log(`Port ${port} in use, retrying with ${nextPort}...`);
|
|
381
|
+
port = nextPort;
|
|
382
|
+
server.close(() => {
|
|
383
|
+
server.listen(port);
|
|
384
|
+
});
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
console.error(`Could not find an open port between ${startPort} and ${startPort + 9}`);
|
|
388
|
+
}
|
|
389
|
+
console.error(`[serve:${projectName}] server error:`, e);
|
|
390
|
+
// Ensure we exit if server fails to start
|
|
391
|
+
process.exit(1);
|
|
392
|
+
});
|
|
393
|
+
server.listen(port, () => {
|
|
394
|
+
const address = server.address();
|
|
395
|
+
const actualPort = typeof address === "object" && address ? address.port : port;
|
|
396
|
+
if (!process.env.OSGREP_BACKGROUND) {
|
|
397
|
+
console.log(`osgrep server listening on http://localhost:${actualPort} (${projectRoot})`);
|
|
398
|
+
}
|
|
399
|
+
(0, server_registry_1.registerServer)({
|
|
400
|
+
pid: process.pid,
|
|
401
|
+
port: actualPort,
|
|
402
|
+
projectRoot,
|
|
403
|
+
startTime: Date.now(),
|
|
404
|
+
});
|
|
405
|
+
});
|
|
406
|
+
const shutdown = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
407
|
+
(0, server_registry_1.unregisterServer)(process.pid);
|
|
408
|
+
// Stop file watcher first
|
|
409
|
+
if (fileWatcher) {
|
|
410
|
+
try {
|
|
411
|
+
yield fileWatcher.close();
|
|
412
|
+
}
|
|
413
|
+
catch (_a) { }
|
|
414
|
+
fileWatcher = null;
|
|
415
|
+
}
|
|
416
|
+
// Stop MLX server if we started it
|
|
417
|
+
if (mlxChild === null || mlxChild === void 0 ? void 0 : mlxChild.pid) {
|
|
418
|
+
try {
|
|
419
|
+
process.kill(mlxChild.pid, "SIGTERM");
|
|
420
|
+
}
|
|
421
|
+
catch (_b) { }
|
|
422
|
+
}
|
|
423
|
+
// Properly await server close
|
|
424
|
+
yield new Promise((resolve, reject) => {
|
|
425
|
+
server.close((err) => {
|
|
426
|
+
if (err) {
|
|
427
|
+
console.error("Error closing server:", err);
|
|
428
|
+
reject(err);
|
|
429
|
+
}
|
|
430
|
+
else {
|
|
431
|
+
resolve();
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
// Timeout fallback in case close hangs
|
|
435
|
+
setTimeout(resolve, 5000);
|
|
436
|
+
});
|
|
437
|
+
// Clean close of owned resources
|
|
438
|
+
try {
|
|
439
|
+
metaCache.close();
|
|
440
|
+
}
|
|
441
|
+
catch (e) {
|
|
442
|
+
console.error("Error closing meta cache:", e);
|
|
443
|
+
}
|
|
444
|
+
try {
|
|
445
|
+
yield vectorDb.close();
|
|
446
|
+
}
|
|
447
|
+
catch (e) {
|
|
448
|
+
console.error("Error closing vector DB:", e);
|
|
449
|
+
}
|
|
450
|
+
yield (0, exit_1.gracefulExit)();
|
|
451
|
+
});
|
|
452
|
+
process.on("SIGINT", shutdown);
|
|
453
|
+
process.on("SIGTERM", shutdown);
|
|
454
|
+
}
|
|
455
|
+
catch (error) {
|
|
456
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
457
|
+
console.error("Serve failed:", message);
|
|
458
|
+
process.exitCode = 1;
|
|
459
|
+
yield (0, exit_1.gracefulExit)(1);
|
|
460
|
+
}
|
|
461
|
+
}));
|
|
462
|
+
exports.serve
|
|
463
|
+
.command("status")
|
|
464
|
+
.description("Show status of background servers")
|
|
465
|
+
.action(() => {
|
|
466
|
+
const servers = (0, server_registry_1.listServers)();
|
|
467
|
+
if (servers.length === 0) {
|
|
468
|
+
console.log("No running servers found.");
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
console.log("Running servers:");
|
|
472
|
+
servers.forEach((s) => {
|
|
473
|
+
console.log(`- PID: ${s.pid} | Port: ${s.port} | Root: ${s.projectRoot}`);
|
|
474
|
+
});
|
|
475
|
+
});
|
|
476
|
+
exports.serve
|
|
477
|
+
.command("stop")
|
|
478
|
+
.description("Stop background servers")
|
|
479
|
+
.option("--all", "Stop all servers", false)
|
|
480
|
+
.action((options) => {
|
|
481
|
+
var _a;
|
|
482
|
+
if (options.all) {
|
|
483
|
+
const servers = (0, server_registry_1.listServers)();
|
|
484
|
+
let count = 0;
|
|
485
|
+
servers.forEach((s) => {
|
|
486
|
+
try {
|
|
487
|
+
process.kill(s.pid, "SIGTERM");
|
|
488
|
+
count++;
|
|
489
|
+
}
|
|
490
|
+
catch (e) {
|
|
491
|
+
console.error(`Failed to stop PID ${s.pid}:`, e);
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
console.log(`Stopped ${count} servers.`);
|
|
495
|
+
}
|
|
496
|
+
else {
|
|
497
|
+
const projectRoot = (_a = (0, project_root_1.findProjectRoot)(process.cwd())) !== null && _a !== void 0 ? _a : process.cwd();
|
|
498
|
+
const server = (0, server_registry_1.getServerForProject)(projectRoot);
|
|
499
|
+
if (server) {
|
|
500
|
+
try {
|
|
501
|
+
process.kill(server.pid, "SIGTERM");
|
|
502
|
+
console.log(`Stopped server for ${projectRoot} (PID: ${server.pid})`);
|
|
503
|
+
}
|
|
504
|
+
catch (e) {
|
|
505
|
+
console.error(`Failed to stop PID ${server.pid}:`, e);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
else {
|
|
509
|
+
console.log(`No server found for ${projectRoot}`);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
});
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.setup = void 0;
|
|
46
|
+
const fs = __importStar(require("node:fs"));
|
|
47
|
+
const path = __importStar(require("node:path"));
|
|
48
|
+
const p = __importStar(require("@clack/prompts"));
|
|
49
|
+
const commander_1 = require("commander");
|
|
50
|
+
const config_1 = require("../config");
|
|
51
|
+
const grammar_loader_1 = require("../lib/index/grammar-loader");
|
|
52
|
+
const index_config_1 = require("../lib/index/index-config");
|
|
53
|
+
const setup_helpers_1 = require("../lib/setup/setup-helpers");
|
|
54
|
+
const exit_1 = require("../lib/utils/exit");
|
|
55
|
+
const project_root_1 = require("../lib/utils/project-root");
|
|
56
|
+
const MLX_MODELS = [
|
|
57
|
+
{
|
|
58
|
+
value: "ibm-granite/granite-embedding-small-english-r2",
|
|
59
|
+
label: "Granite Small (general purpose, 384-dim)",
|
|
60
|
+
},
|
|
61
|
+
];
|
|
62
|
+
exports.setup = new commander_1.Command("setup")
|
|
63
|
+
.description("Interactive setup: download models, choose embedding mode")
|
|
64
|
+
.action(() => __awaiter(void 0, void 0, void 0, function* () {
|
|
65
|
+
var _a, _b;
|
|
66
|
+
p.intro("osgrep setup");
|
|
67
|
+
// Step 1: Download ONNX models + grammars (existing behavior)
|
|
68
|
+
try {
|
|
69
|
+
yield (0, setup_helpers_1.ensureSetup)();
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
p.cancel("Setup failed");
|
|
73
|
+
console.error(error);
|
|
74
|
+
process.exit(1);
|
|
75
|
+
}
|
|
76
|
+
// Download grammars
|
|
77
|
+
const grammarSpinner = p.spinner();
|
|
78
|
+
grammarSpinner.start("Checking Tree-sitter grammars...");
|
|
79
|
+
yield (0, grammar_loader_1.ensureGrammars)(undefined, { silent: true });
|
|
80
|
+
grammarSpinner.stop("Grammars ready");
|
|
81
|
+
// Step 2: Show model status
|
|
82
|
+
const modelIds = [config_1.MODEL_IDS.embed, config_1.MODEL_IDS.colbert];
|
|
83
|
+
const modelStatuses = modelIds.map((id) => {
|
|
84
|
+
const modelPath = path.join(config_1.PATHS.models, ...id.split("/"));
|
|
85
|
+
return { id, exists: fs.existsSync(modelPath) };
|
|
86
|
+
});
|
|
87
|
+
modelStatuses.forEach(({ id, exists }) => {
|
|
88
|
+
p.log.info(`${exists ? "✓" : "✗"} ${id}`);
|
|
89
|
+
});
|
|
90
|
+
// Check skiplist
|
|
91
|
+
const colbertPath = path.join(config_1.PATHS.models, ...config_1.MODEL_IDS.colbert.split("/"));
|
|
92
|
+
const skiplistPath = path.join(colbertPath, "skiplist.json");
|
|
93
|
+
if (!fs.existsSync(skiplistPath)) {
|
|
94
|
+
try {
|
|
95
|
+
const url = `https://huggingface.co/${config_1.MODEL_IDS.colbert}/resolve/main/skiplist.json`;
|
|
96
|
+
const response = yield fetch(url);
|
|
97
|
+
if (response.ok) {
|
|
98
|
+
const buffer = yield response.arrayBuffer();
|
|
99
|
+
fs.writeFileSync(skiplistPath, Buffer.from(buffer));
|
|
100
|
+
p.log.success("Skiplist downloaded");
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
catch (_c) {
|
|
104
|
+
p.log.warn("Skiplist download failed (will use fallback)");
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// Step 3: Interactive embed mode selection
|
|
108
|
+
const paths = (0, project_root_1.ensureProjectPaths)(process.cwd());
|
|
109
|
+
const existingConfig = (0, index_config_1.readIndexConfig)(paths.configPath);
|
|
110
|
+
const embedMode = yield p.select({
|
|
111
|
+
message: "Embedding mode",
|
|
112
|
+
options: [
|
|
113
|
+
{
|
|
114
|
+
value: "cpu",
|
|
115
|
+
label: "CPU only",
|
|
116
|
+
hint: "ONNX — works everywhere",
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
value: "gpu",
|
|
120
|
+
label: "GPU (MLX)",
|
|
121
|
+
hint: "Apple Silicon only, faster indexing + search",
|
|
122
|
+
},
|
|
123
|
+
],
|
|
124
|
+
initialValue: (_a = existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.embedMode) !== null && _a !== void 0 ? _a : (process.arch === "arm64" && process.platform === "darwin"
|
|
125
|
+
? "gpu"
|
|
126
|
+
: "cpu"),
|
|
127
|
+
});
|
|
128
|
+
if (p.isCancel(embedMode)) {
|
|
129
|
+
p.cancel("Setup cancelled");
|
|
130
|
+
yield (0, exit_1.gracefulExit)();
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
let mlxModel;
|
|
134
|
+
if (embedMode === "gpu") {
|
|
135
|
+
const modelChoice = yield p.select({
|
|
136
|
+
message: "MLX embedding model",
|
|
137
|
+
options: MLX_MODELS.map((m) => ({
|
|
138
|
+
value: m.value,
|
|
139
|
+
label: m.label,
|
|
140
|
+
})),
|
|
141
|
+
initialValue: (_b = existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.mlxModel) !== null && _b !== void 0 ? _b : MLX_MODELS[0].value,
|
|
142
|
+
});
|
|
143
|
+
if (p.isCancel(modelChoice)) {
|
|
144
|
+
p.cancel("Setup cancelled");
|
|
145
|
+
yield (0, exit_1.gracefulExit)();
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
mlxModel = modelChoice;
|
|
149
|
+
}
|
|
150
|
+
// Step 4: Write config
|
|
151
|
+
(0, index_config_1.writeSetupConfig)(paths.configPath, { embedMode, mlxModel });
|
|
152
|
+
// Step 5: Warn about reindex if mode/model changed
|
|
153
|
+
if ((existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.indexedAt) &&
|
|
154
|
+
(existingConfig.embedMode !== embedMode ||
|
|
155
|
+
existingConfig.mlxModel !== mlxModel)) {
|
|
156
|
+
p.log.warn("Embedding mode changed. Run `osgrep serve` to reindex with the new settings.");
|
|
157
|
+
}
|
|
158
|
+
p.outro(embedMode === "gpu"
|
|
159
|
+
? `Ready — GPU mode with ${mlxModel}`
|
|
160
|
+
: "Ready — CPU mode");
|
|
161
|
+
yield (0, exit_1.gracefulExit)();
|
|
162
|
+
}));
|