open-context-engine 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/README.zh-CN.md +4 -2
- package/bin/opencontextengine.mjs +2 -2
- package/docs/QUICKSTART.md +13 -5
- package/package.json +4 -3
- package/scripts/retrieval-server.py +9 -1
- package/src/mcp.mjs +1 -1
- package/src/retrieval/languages/go.py +7 -5
- package/src/retrieval/languages/go_types.go +2 -0
- package/src/retrieval/languages/typescript.mjs +4 -2
- package/src/retrieval/languages/typescript.py +1 -1
- package/src/retrieval/live.py +2 -7
- package/src/retrieval/writer_lock.py +25 -0
- package/src/runtime.mjs +9 -2
- package/src/service.mjs +7 -5
- package/src/setup.mjs +6 -5
- package/src/workspaces.mjs +1 -1
package/README.md
CHANGED
|
@@ -35,7 +35,9 @@ Measured with the optional batch rerank API. 40 source-derived tasks, each asked
|
|
|
35
35
|
|
|
36
36
|
## Quick start
|
|
37
37
|
|
|
38
|
-
Requires **macOS or
|
|
38
|
+
Requires **macOS, Linux, or Windows**, Node.js 22.14+, Python 3.10+, Git, and configured embedding/reranking services. Go repositories also need Go 1.22+.
|
|
39
|
+
|
|
40
|
+
Native Windows support is available in version **0.1.3** and later. Run the installation commands in PowerShell.
|
|
39
41
|
|
|
40
42
|
Install from npm, then expand your client's guide. Model settings are shared across clients on the same machine.
|
|
41
43
|
|
|
@@ -148,7 +150,7 @@ Your agent supplies the current project's absolute path as `directory_path`. To
|
|
|
148
150
|
|
|
149
151
|
## Explore
|
|
150
152
|
|
|
151
|
-
[Benchmark report](docs/BENCHMARKS.md) · [Raw evaluations](docs/eval/results) · [Retrieval engine](src/retrieval)
|
|
153
|
+
[Benchmark report](https://github.com/AnnaSuSu/OpenContextEngine/blob/main/docs/BENCHMARKS.md) · [Raw evaluations](https://github.com/AnnaSuSu/OpenContextEngine/tree/main/docs/eval/results) · [Retrieval engine](https://github.com/AnnaSuSu/OpenContextEngine/tree/main/src/retrieval)
|
|
152
154
|
|
|
153
155
|
## License
|
|
154
156
|
|
package/README.zh-CN.md
CHANGED
|
@@ -35,7 +35,9 @@ OpenContextEngine 是一个**可自行部署、面向 AI 编程助手的代码
|
|
|
35
35
|
|
|
36
36
|
## 快速开始
|
|
37
37
|
|
|
38
|
-
需要 **macOS 或
|
|
38
|
+
需要 **macOS、Linux 或 Windows**、Node.js 22.14+、Python 3.10+、Git,以及已配置好的向量和重排服务。分析 Go 仓库还需要 Go 1.22+。
|
|
39
|
+
|
|
40
|
+
**0.1.3** 起支持原生 Windows,安装命令可在 PowerShell 中运行。
|
|
39
41
|
|
|
40
42
|
通过 npm 安装,展开你所用客户端的教程即可。同一台机器、同一用户下的多个客户端可以共用模型配置。下方链接的详细技术文档目前为英文。
|
|
41
43
|
|
|
@@ -148,7 +150,7 @@ Claude 会通过 `directory_path` 传入项目的绝对路径。首次请求会
|
|
|
148
150
|
|
|
149
151
|
## 进一步了解
|
|
150
152
|
|
|
151
|
-
[评测报告](docs/BENCHMARKS.md) · [原始评测数据](docs/eval/results) · [检索引擎源码](src/retrieval)
|
|
153
|
+
[评测报告](https://github.com/AnnaSuSu/OpenContextEngine/blob/main/docs/BENCHMARKS.md) · [原始评测数据](https://github.com/AnnaSuSu/OpenContextEngine/tree/main/docs/eval/results) · [检索引擎源码](https://github.com/AnnaSuSu/OpenContextEngine/tree/main/src/retrieval)
|
|
152
154
|
|
|
153
155
|
## 许可证
|
|
154
156
|
|
|
@@ -10,7 +10,7 @@ import { serviceConfig } from '../src/service.mjs';
|
|
|
10
10
|
const help = `OpenContextEngine — repository context for AI coding agents
|
|
11
11
|
|
|
12
12
|
Usage:
|
|
13
|
-
open-context-engine setup [--python
|
|
13
|
+
open-context-engine setup [--python <interpreter-path>] [--non-interactive]
|
|
14
14
|
open-context-engine mcp [--root /project] [--state /outside/index]
|
|
15
15
|
open-context-engine mcp --connect
|
|
16
16
|
open-context-engine mcp-config
|
|
@@ -19,7 +19,7 @@ Usage:
|
|
|
19
19
|
|
|
20
20
|
Setup installs isolated Python dependencies, saves shared model settings, and
|
|
21
21
|
prints MCP configuration. Without --root, your agent supplies directory_path.
|
|
22
|
-
Requires macOS/Linux, Node.js 22.14+, Python 3.10+, and Git.
|
|
22
|
+
Requires macOS/Linux/Windows, Node.js 22.14+, Python 3.10+, and Git.
|
|
23
23
|
`;
|
|
24
24
|
function mcpConfig() {
|
|
25
25
|
const env = process.env.OCE_CONFIG_HOME ? {OCE_CONFIG_HOME:configDirectory()} : undefined;
|
package/docs/QUICKSTART.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Connect OpenContextEngine to your agent
|
|
2
2
|
|
|
3
|
-
OpenContextEngine runs a local repository index and calls configured model services for embeddings and reranking. Its MCP interface uses stdio.
|
|
3
|
+
OpenContextEngine runs a local repository index and calls configured model services for embeddings and reranking. Its MCP interface uses stdio. Supported hosts: **macOS, Linux, and Windows**. Native Windows support requires version **0.1.3** or later.
|
|
4
4
|
|
|
5
5
|
## 1. Install
|
|
6
6
|
|
|
@@ -13,7 +13,7 @@ npm install -g open-context-engine
|
|
|
13
13
|
open-context-engine setup
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
-
`setup` asks for your embedding and reranking endpoints, keys, model names, and embedding dimensions. It creates a
|
|
16
|
+
`setup` asks for your embedding and reranking endpoints, keys, model names, and embedding dimensions. It creates a dedicated Python virtual environment, installs NumPy and tiktoken, and preloads tokenizer data. No model weights are installed. API key input is not echoed. Use `--python /absolute/path/to/python3` to select a base interpreter.
|
|
17
17
|
|
|
18
18
|
You can also run the same setup from source:
|
|
19
19
|
|
|
@@ -24,13 +24,21 @@ npm ci
|
|
|
24
24
|
node bin/opencontextengine.mjs setup
|
|
25
25
|
```
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
On Windows, run the npm installation commands in PowerShell with `node`, `npm`, `python`, and `git` on `PATH`; WSL is not required. Setup defaults to `python` and uses `Scripts/python.exe` inside its virtual environment. To select a particular interpreter, run:
|
|
28
|
+
|
|
29
|
+
```powershell
|
|
30
|
+
open-context-engine setup --python "C:\Program Files\Python312\python.exe"
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Use native absolute project paths such as `C:\Users\you\project` in MCP calls. Generated MCP JSON escapes backslashes automatically.
|
|
34
|
+
|
|
35
|
+
For internal builds, maintainers can create an archive with `npm pack` and install it with `npm install -g /path/to/open-context-engine-0.1.3.tgz`. See the [release checklist](https://github.com/AnnaSuSu/OpenContextEngine/blob/main/docs/RELEASING.md).
|
|
28
36
|
|
|
29
37
|
The CLI and package are named `open-context-engine`. The previous `opencontextengine` command remains an alias. Existing configuration and cache directories keep their paths, so saved keys and indexes are reused.
|
|
30
38
|
|
|
31
39
|
## 2. Shared model configuration
|
|
32
40
|
|
|
33
|
-
Setup saves
|
|
41
|
+
Setup saves `.config/opencontextengine/config.json` under your home directory. Files use owner-only permissions on macOS/Linux; Windows uses the directory's inherited access permissions. All MCP clients running under the same user share these settings. Python environments live in the adjacent `runtimes/` directory. Use `OCE_CONFIG_HOME` to select a separate configuration directory; setup includes that override in its generated MCP configuration.
|
|
34
42
|
|
|
35
43
|
Configuration precedence is **process environment → saved user settings → source checkout `.env` defaults**. The `.env` of the project being searched is never loaded. Existing source installations using `.env` and `OCE_PYTHON` continue to work.
|
|
36
44
|
|
|
@@ -185,7 +193,7 @@ First-time setup requires network access to npm/PyPI and tokenizer data. If Pyth
|
|
|
185
193
|
|
|
186
194
|
## Installation verification
|
|
187
195
|
|
|
188
|
-
1. Install from npm (or an internal tarball) on macOS or
|
|
196
|
+
1. Install from npm (or an internal tarball) on macOS, Linux, or Windows and run setup with your model endpoints.
|
|
189
197
|
2. Paste the generated MCP configuration into your client, restart it, and search a small project.
|
|
190
198
|
3. Save an edit, add a file, and delete a file; verify search returns current source.
|
|
191
199
|
4. Switch to another project and back; verify the results belong to the requested project.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "open-context-engine",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.14"
|
|
@@ -33,7 +33,8 @@
|
|
|
33
33
|
},
|
|
34
34
|
"os": [
|
|
35
35
|
"darwin",
|
|
36
|
-
"linux"
|
|
36
|
+
"linux",
|
|
37
|
+
"win32"
|
|
37
38
|
],
|
|
38
39
|
"files": [
|
|
39
40
|
"README.zh-CN.md",
|
|
@@ -79,6 +80,6 @@
|
|
|
79
80
|
"access": "public",
|
|
80
81
|
"registry": "https://registry.npmjs.org/"
|
|
81
82
|
},
|
|
82
|
-
"readme": "<div align=\"center\">\n <picture>\n <source media=\"(prefers-color-scheme: dark)\" srcset=\"assets/brand/logo-lockup-dark.svg\">\n <img src=\"assets/brand/logo-lockup.svg\" alt=\"OpenContextEngine\" width=\"660\">\n </picture>\n <p><strong>Precise code context for AI coding agents.</strong></p>\n <p>Find related code across files. Follow its connections. Give your agent the evidence it needs.</p>\n <p>\n <a href=\"docs/BENCHMARKS.md#seven-method-comparison\"><img src=\"https://img.shields.io/badge/dev_evidence_coverage-94.79%25-23875b?style=flat-square\" alt=\"Development evidence coverage: 94.79%\"></a>\n <a href=\"docs/BENCHMARKS.md#seven-method-comparison\"><img src=\"https://img.shields.io/badge/median_retrieval-1.73_s-23875b?style=flat-square\" alt=\"Median retrieval: 1.73 seconds\"></a>\n <a href=\"docs/BENCHMARKS.md#engineering-validation\"><img src=\"https://img.shields.io/badge/verified_tests-107-23875b?style=flat-square\" alt=\"107 verified tests\"></a>\n <a href=\"docs/QUICKSTART.md\"><img src=\"https://img.shields.io/badge/MCP-stdio-193c34?style=flat-square\" alt=\"MCP over stdio\"></a>\n </p>\n <p><a href=\"#quick-start\">Quick start</a> · <a href=\"docs/BENCHMARKS.md\">Benchmarks</a> · <a href=\"docs/QUICKSTART.md\">MCP setup</a> · <strong>English</strong> | <a href=\"README.zh-CN.md\">简体中文</a></p>\n</div>\n\nOpenContextEngine is a **self-hostable code context engine for AI coding agents**. Connect it to your agent through MCP to help it explore an unfamiliar codebase, locate implementations, and find the related code needed for a fix or feature.\n\nIt indexes your working directory and follows saved changes. Given a natural-language task, it combines semantic and keyword search, code relationships, and reranking to return relevant source snippets with file paths and line numbers, within a fixed context budget.\n\n## Why OpenContextEngine\n\n- **Search beyond exact words.** Describe a behavior; retrieve its implementation and connected code across files.\n- **Understand code structure.** Python, TypeScript, JavaScript, and Go adapters, plus text fallback for other languages, configuration, and scripts.\n- **Stay current as you edit.** Saved changes, file deletions, and branch switches sync automatically. Unchanged embeddings are reused; incomplete updates never replace a complete index.\n- **Work across projects through MCP.** Your agent supplies the project path; indexes start on demand and are reused. `search_code` retrieves evidence; `index_status` reports synchronization.\n\n## Measured results\n\n\n\n**94.79% required evidence coverage · 1.73 s median retrieval · 69/80 queries with complete evidence.** Seven engines, four repositories, the same 4,000-token output budget. OpenContextEngine retained the most required evidence in this internal development evaluation.\n\nMeasured with the optional batch rerank API. 40 source-derived tasks, each asked in Chinese and English. Coverage measures source evidence, not coding-agent success. Timings reflect native retrieval for open tools and SDK client calls for ACE. [Full comparison, configurations, and per-query results →](docs/eval/METHOD_COMPARISON.md)\n\n## Quick start\n\nRequires **macOS or
|
|
83
|
+
"readme": "<div align=\"center\">\n <picture>\n <source media=\"(prefers-color-scheme: dark)\" srcset=\"assets/brand/logo-lockup-dark.svg\">\n <img src=\"assets/brand/logo-lockup.svg\" alt=\"OpenContextEngine\" width=\"660\">\n </picture>\n <p><strong>Precise code context for AI coding agents.</strong></p>\n <p>Find related code across files. Follow its connections. Give your agent the evidence it needs.</p>\n <p>\n <a href=\"docs/BENCHMARKS.md#seven-method-comparison\"><img src=\"https://img.shields.io/badge/dev_evidence_coverage-94.79%25-23875b?style=flat-square\" alt=\"Development evidence coverage: 94.79%\"></a>\n <a href=\"docs/BENCHMARKS.md#seven-method-comparison\"><img src=\"https://img.shields.io/badge/median_retrieval-1.73_s-23875b?style=flat-square\" alt=\"Median retrieval: 1.73 seconds\"></a>\n <a href=\"docs/BENCHMARKS.md#engineering-validation\"><img src=\"https://img.shields.io/badge/verified_tests-107-23875b?style=flat-square\" alt=\"107 verified tests\"></a>\n <a href=\"docs/QUICKSTART.md\"><img src=\"https://img.shields.io/badge/MCP-stdio-193c34?style=flat-square\" alt=\"MCP over stdio\"></a>\n </p>\n <p><a href=\"#quick-start\">Quick start</a> · <a href=\"docs/BENCHMARKS.md\">Benchmarks</a> · <a href=\"docs/QUICKSTART.md\">MCP setup</a> · <strong>English</strong> | <a href=\"README.zh-CN.md\">简体中文</a></p>\n</div>\n\nOpenContextEngine is a **self-hostable code context engine for AI coding agents**. Connect it to your agent through MCP to help it explore an unfamiliar codebase, locate implementations, and find the related code needed for a fix or feature.\n\nIt indexes your working directory and follows saved changes. Given a natural-language task, it combines semantic and keyword search, code relationships, and reranking to return relevant source snippets with file paths and line numbers, within a fixed context budget.\n\n## Why OpenContextEngine\n\n- **Search beyond exact words.** Describe a behavior; retrieve its implementation and connected code across files.\n- **Understand code structure.** Python, TypeScript, JavaScript, and Go adapters, plus text fallback for other languages, configuration, and scripts.\n- **Stay current as you edit.** Saved changes, file deletions, and branch switches sync automatically. Unchanged embeddings are reused; incomplete updates never replace a complete index.\n- **Work across projects through MCP.** Your agent supplies the project path; indexes start on demand and are reused. `search_code` retrieves evidence; `index_status` reports synchronization.\n\n## Measured results\n\n\n\n**94.79% required evidence coverage · 1.73 s median retrieval · 69/80 queries with complete evidence.** Seven engines, four repositories, the same 4,000-token output budget. OpenContextEngine retained the most required evidence in this internal development evaluation.\n\nMeasured with the optional batch rerank API. 40 source-derived tasks, each asked in Chinese and English. Coverage measures source evidence, not coding-agent success. Timings reflect native retrieval for open tools and SDK client calls for ACE. [Full comparison, configurations, and per-query results →](docs/eval/METHOD_COMPARISON.md)\n\n## Quick start\n\nRequires **macOS, Linux, or Windows**, Node.js 22.14+, Python 3.10+, Git, and configured embedding/reranking services. Go repositories also need Go 1.22+.\n\nNative Windows support is available in version **0.1.3** and later. Run the installation commands in PowerShell.\n\nInstall from npm, then expand your client's guide. Model settings are shared across clients on the same machine.\n\n<details open>\n<summary><strong>Codex — install, connect, and search</strong></summary>\n\n**1. Install the CLI**\n\nWith Codex CLI already installed, run:\n\n```sh\nnpm install -g open-context-engine\n```\n\n**2. Configure your models**\n\n```sh\nopen-context-engine setup\n```\n\nEnter your embedding and reranking base URLs, API keys, model names, and embedding dimensions. Setup installs isolated Python dependencies and saves your settings. The default reranker uses the ordinary `/rerank` API. [Endpoint examples →](docs/QUICKSTART.md#2-shared-model-configuration)\n\n**3. Add the MCP server**\n\n```sh\ncodex mcp add open-context-engine -- open-context-engine mcp\ncodex mcp get open-context-engine\n```\n\nThe second command checks the saved configuration. These commands assume `open-context-engine` is on the client's `PATH`. For the desktop app or source installations, use the [absolute-path configuration](docs/QUICKSTART.md#client-setup-notes). Restart an already-running Codex client after adding the server.\n\n**4. Search your project**\n\n```sh\ncd /path/to/your-project\ncodex\n```\n\nAsk:\n\n> Use open-context-engine's search_code tool to explain this project's main functionality. Include the entry points and relevant file paths and line numbers.\n\nCodex supplies the project's absolute path as `directory_path`. The first request starts indexing; if it is still building, ask Codex to check `index_status` and retry when ready. Later searches reuse the index, and saved changes update automatically.\n\n</details>\n\n<details>\n<summary><strong>Claude Code — install, connect, and search</strong></summary>\n\n**1. Install the CLI**\n\nWith Claude Code already installed, run:\n\n```sh\nnpm install -g open-context-engine\n```\n\n**2. Configure your models**\n\n```sh\nopen-context-engine setup\n```\n\nEnter your embedding and reranking base URLs, API keys, model names, and embedding dimensions. Setup installs isolated Python dependencies and saves your settings. If you already completed setup for Codex, reuse those settings and skip this step. [Endpoint examples →](docs/QUICKSTART.md#2-shared-model-configuration)\n\n**3. Add the MCP server**\n\n```sh\nclaude mcp add --transport stdio --scope user open-context-engine -- open-context-engine mcp\n```\n\nUser scope makes the server available across your projects. For a shared project configuration, run the command from that project and replace `--scope user` with `--scope project`. These commands assume `open-context-engine` is on the client's `PATH`; see [client setup notes](docs/QUICKSTART.md#client-setup-notes) for absolute paths. Restart an already-running Claude Code session after adding the server.\n\n**4. Search your project**\n\n```sh\ncd /path/to/your-project\nclaude\n```\n\nRun `/mcp` to check the connection, then ask:\n\n> Use open-context-engine's search_code tool to explain this project's main functionality. Include the entry points and relevant file paths and line numbers.\n\nClaude supplies the project's absolute path as `directory_path`. The first request starts indexing; if it is still building, ask Claude to check `index_status` and retry when ready. Later searches reuse the index, and saved changes update automatically.\n\n</details>\n\n<details>\n<summary>Other MCP clients</summary>\n\n[Install and run setup](docs/QUICKSTART.md#1-install), then use the configuration printed by `open-context-engine mcp-config` in your client's supported format. For clients that accept `mcpServers` JSON and can find the installed command on `PATH`:\n\n```json\n{\n \"mcpServers\": {\n \"open-context-engine\": {\n \"command\": \"open-context-engine\",\n \"args\": [\"mcp\"]\n }\n }\n}\n```\n\nYour agent supplies the current project's absolute path as `directory_path`. To pin one project, add `\"--root\", \"/absolute/path/to/your-repository\"` to `args`.\n\n</details>\n\n[Model configuration, troubleshooting, and update behavior →](docs/QUICKSTART.md)\n\n## Explore\n\n[Benchmark report](https://github.com/AnnaSuSu/OpenContextEngine/blob/main/docs/BENCHMARKS.md) · [Raw evaluations](https://github.com/AnnaSuSu/OpenContextEngine/tree/main/docs/eval/results) · [Retrieval engine](https://github.com/AnnaSuSu/OpenContextEngine/tree/main/src/retrieval)\n\n## License\n\n[MIT](LICENSE) © 2026 AnnaSuSu.\n\n## Acknowledgments\n\nThanks to the [LINUX DO](https://linux.do/) community.\n",
|
|
83
84
|
"readmeFilename": "README.md"
|
|
84
85
|
}
|
|
@@ -5,6 +5,7 @@ import json
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
import re
|
|
7
7
|
import secrets
|
|
8
|
+
from socketserver import TCPServer
|
|
8
9
|
import sys
|
|
9
10
|
import threading
|
|
10
11
|
import time
|
|
@@ -17,6 +18,13 @@ from routed import RoutedEngine, VERSION
|
|
|
17
18
|
from live import LiveIndex, IndexUnavailable
|
|
18
19
|
|
|
19
20
|
|
|
21
|
+
class LoopbackHTTPServer(ThreadingHTTPServer):
|
|
22
|
+
def server_bind(self):
|
|
23
|
+
# This worker binds a numeric loopback address; reverse DNS is unnecessary.
|
|
24
|
+
TCPServer.server_bind(self)
|
|
25
|
+
self.server_name, self.server_port = self.server_address
|
|
26
|
+
|
|
27
|
+
|
|
20
28
|
def plan_query(query):
|
|
21
29
|
parts = [part.strip() for part in re.split(r'[::;;,]|,\s+(?=how|which|why|where|what)|\s+and\s+(?=how|which|why|where|what)', query, flags=re.I) if len(part.strip()) > 7]
|
|
22
30
|
facets = parts if 1 < len(parts) <= 4 else [query]
|
|
@@ -119,7 +127,7 @@ def serve(config):
|
|
|
119
127
|
finally:
|
|
120
128
|
lock.release()
|
|
121
129
|
|
|
122
|
-
server =
|
|
130
|
+
server = LoopbackHTTPServer(('127.0.0.1',config.get('port',23505)),Handler)
|
|
123
131
|
server.daemon_threads = True
|
|
124
132
|
print(json.dumps({'listening':f'http://127.0.0.1:{server.server_port}',
|
|
125
133
|
'health':{'status':'running','mode':'live'} if live else health}),flush=True)
|
package/src/mcp.mjs
CHANGED
|
@@ -15,7 +15,7 @@ export function createMcpServer(config, {resolveConfig, automatic = false} = {})
|
|
|
15
15
|
}
|
|
16
16
|
const pathSchema = z.string().min(1).describe('Absolute path to the project directory. Required in automatic workspace mode.');
|
|
17
17
|
const directoryPath = automatic ? pathSchema : pathSchema.optional();
|
|
18
|
-
const server = new McpServer({name:'open-context-engine',version:'0.1.
|
|
18
|
+
const server = new McpServer({name:'open-context-engine',version:'0.1.3'}, {
|
|
19
19
|
instructions:workspaceInstructions + 'Search for source evidence. Results include source paths and line numbers. '
|
|
20
20
|
+ 'Search waits for saved file changes to be indexed. If an update is pending or fails, inspect index_status and retry after it completes. '
|
|
21
21
|
+ 'Read target files again before editing, because code may change after a search.',
|
|
@@ -8,6 +8,7 @@ import re
|
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
import shutil
|
|
10
10
|
import subprocess
|
|
11
|
+
import sys
|
|
11
12
|
import tempfile
|
|
12
13
|
|
|
13
14
|
from .schema import physical_lines
|
|
@@ -54,23 +55,24 @@ def parse(sources, options):
|
|
|
54
55
|
if cache.is_symlink() or (os.name == 'posix' and
|
|
55
56
|
(cache.stat().st_uid != os.getuid() or cache.stat().st_mode & 0o077)):
|
|
56
57
|
raise RuntimeError('Go parser cache must be private to the current user')
|
|
57
|
-
|
|
58
|
+
suffix = '.exe' if sys.platform == 'win32' else ''
|
|
59
|
+
executable = cache/(identity+suffix)
|
|
58
60
|
if executable.is_symlink():
|
|
59
61
|
raise RuntimeError('Invalid Go parser cache entry')
|
|
60
62
|
if not executable.exists():
|
|
61
63
|
with tempfile.TemporaryDirectory(dir=cache) as directory:
|
|
62
|
-
target = Path(directory)/'parser'
|
|
64
|
+
target = Path(directory)/('parser'+suffix)
|
|
63
65
|
env = {**os.environ, 'GOENV': 'off', 'GOWORK': 'off', 'GOTOOLCHAIN': 'local',
|
|
64
66
|
'GOPROXY': 'off', 'GO111MODULE': 'off', 'CGO_ENABLED': '0', 'GOFLAGS': '',
|
|
65
67
|
'GOCACHE': str(cache/'build')}
|
|
66
68
|
built = subprocess.run([binary, 'build', '-o', str(target), str(source), str(semantic)],
|
|
67
|
-
env=env, capture_output=True,
|
|
69
|
+
env=env, capture_output=True, encoding='utf-8', timeout=120)
|
|
68
70
|
if built.returncode:
|
|
69
71
|
raise RuntimeError('Go parser build failed: '+built.stderr[:2000])
|
|
70
72
|
os.replace(target, executable)
|
|
71
73
|
result = subprocess.run([str(executable)], input=json.dumps({'options': options, 'files': [
|
|
72
74
|
{'path': source.path, 'text': source.text} for source in sources]}),
|
|
73
|
-
|
|
75
|
+
encoding='utf-8', capture_output=True, timeout=120)
|
|
74
76
|
if result.returncode:
|
|
75
77
|
raise ValueError('Go adapter failed: '+result.stderr.strip()[:2000])
|
|
76
78
|
return json.loads(result.stdout)
|
|
@@ -84,7 +86,7 @@ def extract(sources, max_lines=65, options=None):
|
|
|
84
86
|
for file in parsed['files']:
|
|
85
87
|
path = file['path']
|
|
86
88
|
lines = physical_lines(source_by_path[path].text)
|
|
87
|
-
module =
|
|
89
|
+
module = Path(path).parent.as_posix()+'::'+file['package']
|
|
88
90
|
root = {'name': '', 'kind': 'module', 'start': 1, 'end': len(lines), 'decl': 0}
|
|
89
91
|
owners = [root]*len(lines)
|
|
90
92
|
for entry in file['entries']:
|
|
@@ -73,6 +73,8 @@ func resolveTypes(fset *token.FileSet, trees map[string]*ast.File, texts map[str
|
|
|
73
73
|
context.GOOS = goos
|
|
74
74
|
context.GOARCH = goarch
|
|
75
75
|
context.CgoEnabled = false
|
|
76
|
+
// Snapshot paths are slash-separated regardless of the helper's host OS.
|
|
77
|
+
context.JoinPath = path.Join
|
|
76
78
|
context.OpenFile = func(name string) (io.ReadCloser, error) {
|
|
77
79
|
if text, ok := texts[path.Clean(name)]; ok {
|
|
78
80
|
return io.NopCloser(strings.NewReader(text)), nil
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/** Parse and bind only the frozen source set. Never emit, execute, or load repo plugins. */
|
|
2
2
|
import ts from 'typescript';
|
|
3
|
-
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { readFileSync, existsSync, realpathSync } from 'node:fs';
|
|
4
4
|
import path from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
5
6
|
|
|
6
7
|
const COMPILER_VERSION = '5.9.3';
|
|
7
8
|
if (ts.version !== COMPILER_VERSION) throw new Error(`Expected TypeScript ${COMPILER_VERSION}; run npm ci`);
|
|
@@ -223,7 +224,8 @@ export function extract(files, maxLines = 65, settings = {}) {
|
|
|
223
224
|
return { compilerVersion: ts.version, units };
|
|
224
225
|
}
|
|
225
226
|
|
|
226
|
-
if (process.argv[1] &&
|
|
227
|
+
if (process.argv[1] && existsSync(process.argv[1])
|
|
228
|
+
&& realpathSync.native(fileURLToPath(import.meta.url)) === realpathSync.native(process.argv[1])) {
|
|
227
229
|
try {
|
|
228
230
|
const input = JSON.parse(readFileSync(0, 'utf8'));
|
|
229
231
|
process.stdout.write(JSON.stringify(extract(input.files, input.maxLines, input.options)));
|
|
@@ -11,7 +11,7 @@ def extract(sources, max_lines=65, options=None):
|
|
|
11
11
|
'maxLines': max_lines, 'options': options or {}}
|
|
12
12
|
try:
|
|
13
13
|
result = subprocess.run(['node', str(Path(__file__).with_suffix('.mjs'))],
|
|
14
|
-
input=json.dumps(payload),
|
|
14
|
+
input=json.dumps(payload), encoding='utf-8', capture_output=True,
|
|
15
15
|
timeout=120, check=False)
|
|
16
16
|
except FileNotFoundError as error:
|
|
17
17
|
raise RuntimeError('TypeScript indexing requires Node.js and npm ci') from error
|
package/src/retrieval/live.py
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
from collections import Counter
|
|
3
3
|
from contextlib import closing
|
|
4
4
|
from dataclasses import dataclass
|
|
5
|
-
import fcntl
|
|
6
5
|
import hashlib
|
|
7
6
|
import json
|
|
8
7
|
import os
|
|
@@ -19,6 +18,7 @@ from engine import document, post
|
|
|
19
18
|
from languages import adapter_manifest, source_units
|
|
20
19
|
from languages.files import discover_snapshot
|
|
21
20
|
from routed import RoutedEngine
|
|
21
|
+
from writer_lock import acquire_writer_lock
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def digest(value):
|
|
@@ -72,12 +72,7 @@ class LiveIndex:
|
|
|
72
72
|
self.phase = 'starting'
|
|
73
73
|
self.parse_cache = {}
|
|
74
74
|
self.state.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
75
|
-
self.lock_file = (self.state/'writer.lock')
|
|
76
|
-
try:
|
|
77
|
-
fcntl.flock(self.lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
78
|
-
except OSError:
|
|
79
|
-
self.lock_file.close()
|
|
80
|
-
raise ValueError('This index directory already has a running writer') from None
|
|
75
|
+
self.lock_file = acquire_writer_lock(self.state/'writer.lock')
|
|
81
76
|
self.thread = threading.Thread(target=self._run, name='repository-index', daemon=True)
|
|
82
77
|
|
|
83
78
|
def scan(self):
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Nonblocking process lock, released when its file handle is closed."""
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
if sys.platform == 'win32':
|
|
5
|
+
import msvcrt
|
|
6
|
+
|
|
7
|
+
def lock(file):
|
|
8
|
+
# Every writer locks the same byte, including when the file is empty.
|
|
9
|
+
file.seek(0)
|
|
10
|
+
msvcrt.locking(file.fileno(), msvcrt.LK_NBLCK, 1)
|
|
11
|
+
else:
|
|
12
|
+
import fcntl
|
|
13
|
+
|
|
14
|
+
def lock(file):
|
|
15
|
+
fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def acquire_writer_lock(path):
|
|
19
|
+
file = path.open('a+b')
|
|
20
|
+
try:
|
|
21
|
+
lock(file)
|
|
22
|
+
except OSError:
|
|
23
|
+
file.close()
|
|
24
|
+
raise ValueError('This index directory already has a running writer') from None
|
|
25
|
+
return file
|
package/src/runtime.mjs
CHANGED
|
@@ -6,6 +6,12 @@ import { createHash } from 'node:crypto';
|
|
|
6
6
|
import { configDirectory, projectRoot } from './config.mjs';
|
|
7
7
|
|
|
8
8
|
const exec = promisify(execFile);
|
|
9
|
+
export function defaultPython(platform = process.platform) {
|
|
10
|
+
return platform === 'win32' ? 'python' : 'python3';
|
|
11
|
+
}
|
|
12
|
+
export function venvPython(directory, platform = process.platform) {
|
|
13
|
+
return platform === 'win32' ? join(directory,'Scripts','python.exe') : join(directory,'bin','python');
|
|
14
|
+
}
|
|
9
15
|
export async function run(command, args, options = {}) {
|
|
10
16
|
try {return await exec(command,args,{timeout:600000,maxBuffer:4*1024*1024,...options});}
|
|
11
17
|
catch (error) {
|
|
@@ -21,7 +27,8 @@ export async function verifyPython(python, environment = process.env, execute =
|
|
|
21
27
|
return (await execute(python,['-c',probe],{env:environment,timeout:120000})).stdout.trim();
|
|
22
28
|
}
|
|
23
29
|
|
|
24
|
-
export async function ensureRuntime(environment, {
|
|
30
|
+
export async function ensureRuntime(environment, {platform = process.platform,
|
|
31
|
+
python = defaultPython(platform), execute = run, log = () => {}} = {}) {
|
|
25
32
|
const runtimes = join(configDirectory(environment),'runtimes');
|
|
26
33
|
const requirements = await readFile(join(projectRoot,'requirements.txt'));
|
|
27
34
|
const fingerprint = createHash('sha256').update(requirements).digest('hex');
|
|
@@ -43,7 +50,7 @@ export async function ensureRuntime(environment, {python = 'python3', execute =
|
|
|
43
50
|
await mkdir(runtimes,{recursive:true,mode:0o700});
|
|
44
51
|
// Virtual environments cannot be relocated. Each installation is built in its final directory.
|
|
45
52
|
const directory = await mkdtemp(join(runtimes,'python-'));
|
|
46
|
-
const executable =
|
|
53
|
+
const executable = venvPython(directory,platform);
|
|
47
54
|
const env = {...environment,TIKTOKEN_CACHE_DIR:join(directory,'tokenizer')};
|
|
48
55
|
try {
|
|
49
56
|
log('Creating an isolated Python environment...');
|
package/src/service.mjs
CHANGED
|
@@ -5,21 +5,22 @@ import { homedir } from 'node:os';
|
|
|
5
5
|
import { resolve, dirname, delimiter } from 'node:path';
|
|
6
6
|
import { createInterface } from 'node:readline';
|
|
7
7
|
import { projectRoot, loadEnvironment } from './config.mjs';
|
|
8
|
+
import { defaultPython, venvPython } from './runtime.mjs';
|
|
8
9
|
import { embeddingTransportConfig, remoteRerankerConfig, rerankerExecutionTransport } from './eval/remote-models.mjs';
|
|
9
10
|
|
|
10
11
|
export function serviceConfig({root, state, port = 0} = {}, environment = process.env) {
|
|
11
12
|
const env = loadEnvironment(environment);
|
|
12
13
|
const embedding = embeddingTransportConfig(env);
|
|
13
14
|
const reranker = remoteRerankerConfig(env), runtime = rerankerExecutionTransport(env);
|
|
14
|
-
const repository = root ? realpathSync(resolve(root)) : undefined;
|
|
15
|
+
const repository = root ? realpathSync.native(resolve(root)) : undefined;
|
|
15
16
|
const workspaceId = repository && createHash('sha256').update(repository).digest('hex').slice(0, 24);
|
|
16
|
-
const candidates = ['.venv
|
|
17
|
-
'.pilot-state/language-adapters-venv
|
|
17
|
+
const candidates = ['.venv', '.pilot-state/baselines/cocoindex-venv',
|
|
18
|
+
'.pilot-state/language-adapters-venv'].map(path => venvPython(resolve(projectRoot, path)));
|
|
18
19
|
const currentState = repository && resolve(homedir(), '.cache/opencontextengine', workspaceId);
|
|
19
20
|
const previousState = repository && resolve(homedir(), '.cache/reponerve', workspaceId);
|
|
20
21
|
const defaultState = repository && (existsSync(currentState) ? currentState : existsSync(previousState) ? previousState : currentState);
|
|
21
22
|
return {
|
|
22
|
-
python: env.OCE_PYTHON || candidates.find(existsSync) ||
|
|
23
|
+
python: env.OCE_PYTHON || candidates.find(existsSync) || defaultPython(),
|
|
23
24
|
workerEnv: {...(env.OCE_GO_BINARY ? {OCE_GO_BINARY:env.OCE_GO_BINARY} : {}),
|
|
24
25
|
...(env.TIKTOKEN_CACHE_DIR ? {TIKTOKEN_CACHE_DIR:env.TIKTOKEN_CACHE_DIR} : {})},
|
|
25
26
|
config: {root: repository, state: state ? resolve(state) : repository
|
|
@@ -41,7 +42,8 @@ export function startService(settings, {log = line => process.stderr.write(line
|
|
|
41
42
|
const {python, config} = settings;
|
|
42
43
|
const child = spawn(python, [resolve(projectRoot, 'scripts/retrieval-server.py')], {
|
|
43
44
|
cwd: projectRoot, env: {...process.env, ...settings.workerEnv,
|
|
44
|
-
PATH:dirname(process.execPath)+delimiter+(process.env.PATH || ''),
|
|
45
|
+
PATH:dirname(process.execPath)+delimiter+(process.env.PATH || ''),
|
|
46
|
+
PYTHONUTF8:'1', PYTHONIOENCODING:'utf-8', OPENBLAS_NUM_THREADS:'2', OMP_NUM_THREADS:'2'},
|
|
45
47
|
stdio:['pipe', 'pipe', 'pipe'],
|
|
46
48
|
});
|
|
47
49
|
const lines = createInterface({input: child.stdout});
|
package/src/setup.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import { createInterface } from 'node:readline/promises';
|
|
|
2
2
|
import { Writable } from 'node:stream';
|
|
3
3
|
import { configDirectory, loadEnvironment, saveUserConfig } from './config.mjs';
|
|
4
4
|
import { remoteRerankerConfig, embeddingTransportConfig, rerankerExecutionTransport } from './eval/remote-models.mjs';
|
|
5
|
-
import { ensureRuntime, run } from './runtime.mjs';
|
|
5
|
+
import { ensureRuntime, run, defaultPython } from './runtime.mjs';
|
|
6
6
|
|
|
7
7
|
export function validateModels(env) {
|
|
8
8
|
embeddingTransportConfig(env);
|
|
@@ -37,8 +37,9 @@ export function terminalPrompt(input = process.stdin, output = process.stderr) {
|
|
|
37
37
|
}
|
|
38
38
|
|
|
39
39
|
export async function setup({environment = process.env, nonInteractive = false, python,
|
|
40
|
-
prompt, install = ensureRuntime, execute = run,
|
|
41
|
-
|
|
40
|
+
platform = process.platform, prompt, install = ensureRuntime, execute = run,
|
|
41
|
+
log = message => process.stderr.write(message+'\n')} = {}) {
|
|
42
|
+
if (!['darwin','linux','win32'].includes(platform)) throw new Error('OpenContextEngine supports macOS, Linux and Windows');
|
|
42
43
|
let env = {EMBEDDING_MODEL:'Qwen3-Embedding-4B',RERANK_MODEL:'Qwen3-Reranker-4B',
|
|
43
44
|
OCE_EMBEDDING_DIMENSIONS:'1024',...loadEnvironment(environment)};
|
|
44
45
|
log(`Configuration: ${configDirectory(environment)}`);
|
|
@@ -57,10 +58,10 @@ export async function setup({environment = process.env, nonInteractive = false,
|
|
|
57
58
|
validateModels(env);
|
|
58
59
|
await execute('git',['--version'],{timeout:10000});
|
|
59
60
|
if (python) delete env.OCE_PYTHON;
|
|
60
|
-
const runtime = await install(env,{python:python ||
|
|
61
|
+
const runtime = await install(env,{platform,python:python || defaultPython(platform),execute,log});
|
|
61
62
|
env = {...env,...runtime};
|
|
62
63
|
const path = saveUserConfig(env,environment);
|
|
63
|
-
log(`Saved configuration to ${path}
|
|
64
|
+
log(`Saved configuration to ${path}.`);
|
|
64
65
|
log('Setup complete. Go projects additionally require Go 1.22+ on PATH or OCE_GO_BINARY.');
|
|
65
66
|
return {path,env};
|
|
66
67
|
}
|
package/src/workspaces.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { createHash } from 'node:crypto';
|
|
|
4
4
|
import { serviceConfig, startService } from './service.mjs';
|
|
5
5
|
|
|
6
6
|
function directory(path) {
|
|
7
|
-
const canonical = realpathSync(path);
|
|
7
|
+
const canonical = realpathSync.native(path);
|
|
8
8
|
if (!statSync(canonical).isDirectory()) throw new Error('directory_path must point to a directory');
|
|
9
9
|
return canonical;
|
|
10
10
|
}
|