@octocodeai/octocode-engine 16.6.2 → 17.0.1

package/dist/security/withSecurityValidation.js

@@ -13,13 +13,24 @@ export function configureSecurity(deps) {
 function createErrorResult(text) {
     return { content: [{ type: 'text', text }], isError: true };
 }
-function withToolTimeout(toolName, promise, signal, timeoutMs) {
+// Combine an external caller-provided AbortSignal with an internal one so that
+// aborting either cancels the tool. Returns the single source unchanged when
+// only one (or none) is present, preserving existing behavior.
+function mergeAbortSignals(external, internal) {
+    if (!external)
+        return internal;
+    if (!internal)
+        return external;
+    return AbortSignal.any([external, internal]);
+}
+function withToolTimeout(toolName, promise, signal, timeoutMs, onTimeout) {
     const timeout = getTimeoutMs(timeoutMs);
     if (signal?.aborted) {
         return Promise.resolve(createErrorResult(`Tool '${toolName}' was cancelled before execution.`));
     }
     return new Promise(resolve => {
         const timer = setTimeout(() => {
+            onTimeout?.();
             resolve(createErrorResult(`Tool '${toolName}' timed out after ${timeout / 1000}s. Try reducing query complexity or scope.`));
         }, timeout);
         const onAbort = () => {
@@ -50,7 +61,7 @@ function withToolTimeout(toolName, promise, signal, timeoutMs) {
     });
 }
 async function runSecure(opts) {
-    const { toolName, handler, args, authInfo, sessionId, signal, timeoutMs } = opts;
+    const { toolName, handler, args, authInfo, sessionId, signal, timeoutMs, onTimeout, abortSignal, } = opts;
     try {
         const sanitizer = getSanitizer();
         const validation = sanitizer.validateInputParameters(args);
@@ -58,7 +69,8 @@ async function runSecure(opts) {
             return createErrorResult(`Security validation failed: ${validation.warnings.join('; ')}`);
         }
         const sanitizedParams = validation.sanitizedParams;
-        const rawResult = await withToolTimeout(toolName, handler(sanitizedParams, authInfo, sessionId), signal, timeoutMs);
+        const mergedSignal = mergeAbortSignals(signal, abortSignal);
+        const rawResult = await withToolTimeout(toolName, handler(sanitizedParams, authInfo, sessionId), mergedSignal, timeoutMs, onTimeout);
         return rawResult;
     }
     catch (error) {

package/docs/LSP_SERVER_LIFECYCLE.md

@@ -0,0 +1,258 @@
+# LSP in Octocode — Lifecycle, Provisioning, and the No-Fallback Contract
+
+Companion: `docs/context/LSP_GUIDE.md` (protocol primer + platformized resolution ladder).
+
+---
+
+## Two layers, two different jobs
+
+| Layer | What it is | Speed | Answers |
+|---|---|---|---|
+| **Tree-sitter / OXC** | parser compiled into the engine | sub-ms/file | *"what does this code look like?"* — outlines, shapes, calls, imports (`structural/`, `signatures/`, `grammar.rs`) |
+| **LSP** | a real language server, spawned over stdio | cold 1–120s, warm <100ms | *"what does this symbol mean?"* — cross-file definition identity, all references, call/type graph (`lsp/client.rs`, `manager.ts`, tools-core `semantic_content/execution.ts`) |
+
+These are **not** interchangeable. Tree-sitter cannot resolve a symbol across files, infer a type, or follow an import — that needs a language server.
+
+## The no-fallback contract (the rule that matters)
+
+When a semantic operation needs a language server and **no server is available**, octocode **throws** — it does *not* fabricate a syntactic or same-file approximation. A faked answer is worse than an honest failure, because the calling agent would trust it.
+
+- The thrown error is the standard typed envelope: `status:"error"`, `errorCode:"lspServerUnavailable"`. In bulk it lands under `errors[]`.
+- The message names the language, says no server is available, gives the install hint, and **directs the agent to `localSearchCode` (text/structural search) + `localGetFileContent`** instead.
+- octocode never returns a same-file-only `references` result, or a tree-sitter guess, dressed up as a semantic answer.
+
+**Throws when no server:** `definition`, `references`, `hover`, `callers`, `callees`, `callHierarchy`, `typeDefinition`, `implementation`, `workspaceSymbol`, `supertypes`, `subtypes`, `diagnostic`.
+
+**Never throws (genuine tree-sitter features, server-free):** `documentSymbols` (native OXC for JS/TS, Markdown heading outline, or LSP when present) and structural/AST search via `localSearchCode`. These are real syntactic capabilities, not LSP stand-ins. `documentSymbols` only throws for a non-JS/TS language with no server *and* no outline.
+
+> A server that *is* running but lacks a capability, or returns zero results, still yields an honest *empty* (`unsupportedOperation` / `noReferences` / …) — that is an accurate answer ("none"), not a missing-server failure.
+
+## Provisioning — three classes (how a server becomes available)
+
+octocode maximizes the chance a real server answers via the resolution ladder (override → PATH → bundled → ecosystem discovery → managed cache; see `LSP_GUIDE.md` §13). Use `npx octocode lsp-server list` to see all servers and their current status, `npx octocode lsp-server status <file>` to check resolution for a specific file, and `npx octocode lsp-server install <name>` to trigger an auto-download. Servers fall into:
+
+- **Bundled (npm dep, offline)** — pure-JS servers launched with the current Node; zero install. TS/JS, Python (pyright), Shell (bash-language-server), PHP (intelephense), YAML, JSON/HTML/CSS.
+- **Auto-download (managed cache)** — portable single-binary servers fetched from a pinned release into `~/.octocode/lsp/<server>/<tag>/` (prompt-by-default, SHA-verified). rust-analyzer (all platforms), clangd (no linux-arm64 asset). Set `OCTOCODE_LSP_AUTO_INSTALL=auto` to skip the prompt or `=off` to disable downloads entirely.
+- **Detect-and-instruct (host toolchain)** — need a runtime octocode won't auto-install: gopls (Go), jdtls (JDK 21+), sourcekit-lsp (Xcode/CLI tools on macOS), csharp-ls (.NET SDK). The status/hint tells you how to install; semantic ops throw until you do.
+
+## Supported language servers
+
+Scope is the **main languages**. Niche/long-tail servers were intentionally removed from
+LSP routing (their tree-sitter grammars remain for structural/AST search). A file type not
+listed below has no server config: semantic ops return `lspServerUnavailable` and the agent
+falls back to text search.
+
+| Language | Extensions | Server | Provisioning |
+|---|---|---|---|
+| TypeScript / JS (+ TSX/JSX) | `.ts .mts .cts .tsx .js .mjs .cjs .jsx` | typescript-language-server (`tsgo`/override aware) | **bundled** |
+| Python | `.py .pyi` | pyright (`pylsp` via override) | **bundled** |
+| Shell | `.sh` | bash-language-server | **bundled** |
+| PHP | `.php` | intelephense | **bundled** |
+| YAML | `.yaml .yml` | yaml-language-server | **bundled** |
+| JSON | `.json .jsonc` | vscode-json-language-server | **bundled** |
+| HTML | `.html .htm` | vscode-html-language-server | **bundled** |
+| CSS / SCSS / LESS | `.css .scss .less` | vscode-css-language-server | **bundled** |
+| Rust | `.rs` | rust-analyzer | **auto-download** |
+| C / C++ | `.c .h .cpp .cc .cxx .hpp` | clangd | **auto-download** (no linux-arm64 asset) |
+| Go | `.go` | gopls | **detect-and-instruct** (needs Go toolchain) |
+| Java | `.java` | jdtls | **detect-and-instruct** (needs JDK 21+) |
+| Swift | `.swift` | sourcekit-lsp | **detect-and-instruct** (needs Xcode or `xcode-select --install`) |
+| C# | `.cs` | csharp-ls | **detect-and-instruct** (needs .NET SDK + `dotnet tool install -g csharp-ls`) |
+| SQL | `.sql` | sqls | **PATH / override only** |
+
+Any built-in server can be overridden with `OCTOCODE_<LANG>_SERVER_PATH` or `.octocode/lsp-servers.json`.
+PATH/override-only servers resolve only if already on `PATH` / in an ecosystem dir; otherwise
+semantic ops throw with an install hint.
+
+**Removed from LSP routing** (use text/structural search instead): TOML, Ruby, Kotlin,
+Elixir, Terraform, Lua, Proto, OCaml, Zig, Julia, Erlang, R, GDScript. Their tree-sitter
+grammars stay available for `localSearchCode` structural/AST queries.
+
+### Custom / bring-your-own LSP (any language)
+
+A language with **no built-in spec** (e.g. Scala, Kotlin, Ruby) gets full semantic support by
+registering a server in a JSON config — no rebuild, no code change. This is also how you swap a
+built-in server for a different one. Resolution reads, in order (`config.rs::user_config_paths`):
+
+1. `$OCTOCODE_LSP_CONFIG` (explicit file path)
+2. `<workspace>/.octocode/lsp-servers.json` (per-project, checked in or local)
+3. `~/.octocode/lsp-servers.json` (per-user, all projects)
+
+The file maps a **file extension** to a launch spec. A custom entry takes precedence over the
+built-in spec for that extension:
+
+```jsonc
+// .octocode/lsp-servers.json — register Scala (metals)
+{
+  "languageServers": {
+    ".scala": { "command": "metals", "args": ["stdio"], "languageId": "scala" },
+    ".sc":    { "command": "metals", "args": ["stdio"], "languageId": "scala" }
+  }
+}
+```
+
+| Field | Required | Meaning |
+|---|---|---|
+| `command` | yes | Executable name (resolved on `PATH`) or absolute path. Shell wrappers are rejected. |
+| `languageId` | yes | LSP `languageId` sent on `textDocument/didOpen` (e.g. `scala`, `ruby`). |
+| `args` | no | Launch args (default `[]`). |
+| `initializationOptions` | no | Passed verbatim in the LSP `initialize` request. |
+
+With the config present, semantic ops (`definition`, `references`, `hover`, call hierarchy, …)
+work for that language exactly like a built-in one. **Without it, the extension stays unsupported:
+the engine resolves no server and the no-fallback contract applies** — semantic ops throw
+`lspServerUnavailable` and the agent falls back to `localSearchCode` + `localGetFileContent`.
+Both halves of this contract are asserted by the benchmark (`benchmark/lsp/check-lsp.mjs`,
+"Custom LSP — bring-your-own server (Scala / metals)"), and verified **live against a real
+server** by `benchmark/lsp/check-custom-lsp.mjs` (`yarn lsp:custom`) — which registers
+`bash-language-server` (a language with no built-in spec) and runs real `documentSymbols` /
+`references` / `hover` through it.
+
+### Markup & docs: what's LSP vs minify
+
+- **HTML / CSS / SCSS / LESS / JSON / YAML are LSP** — served by the bundled
+  `vscode-*-language-server` / `yaml-language-server` (markup/data, offline-ready). They're
+  not "code" languages but they do have real language servers.
+- **Markdown / MDX are NOT LSP.** They are handled by the **minifier** using heading-section
+  heuristics (ATX `#`/`##` and setext headings → `minify/strategies/markdown.rs`; `md`,
+  `markdown`, `mdx` all map to the markdown strategy in `minify/config.rs`). There is no
+  markdown language server in octocode — `documentSymbols` on a `.md` file uses the native
+  heading-outline path, and structure/compression comes from the minifier, not a server.
+
+## Full format support matrix
+
+Per-extension capabilities across all four axes — minify strategy, structural AST,
+signature outline, and LSP server — machine-generated from the shipped napi binary.
+The LSP column here is the per-extension view of the [Supported language servers](#supported-language-servers)
+table above.
+
+<!-- BEGIN GENERATED: support-matrix (yarn matrix:check --write) — do not edit between these markers -->
+
+_Generated by `yarn matrix:check --write` (benchmark/check-matrix.mjs) — every cell probed live against the shipped napi binary. Do not edit between the markers; run `yarn matrix:check` to re-verify._
+
+> **Note:** The matrix queries the native layer (`config.rs`) only. Two servers injected at the TS layer (`config.ts`) are not reflected: **bash-language-server** for `.sh` (languageId `shellscript`) and **intelephense** for `.php` (the native spec emits `intelephense` as command — both bundled and active). Their LSP cells show `—` in the matrix below but the servers are bundled and resolve correctly (`npx octocode lsp-server status <file>` will show `resolved: bundled`).
+
+**151 extensions** known to the engine — 61 with structural AST, 47 with a signature outline, 32 with an LSP server, 90 minify-only.
+
+### Rich formats — AST + signature + LSP
+
+Extensions with a wired tree-sitter grammar (and, where configured, a language server). The minify column is the configured strategy.
+
+| Extension | Minify | Structural AST | Signature outline | LSP (server → language-id) |
+|-----------|--------|:--------------:|-------------------|----------------------------|
+| `.bash` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.c` | `conservative` | ✅ | ✅ tree-sitter | `clangd` → `c` |
+| `.cc` | `conservative` | ✅ | ✅ tree-sitter | `clangd` → `cpp` |
+| `.cjs` | `terser` | ✅ | ✅ tree-sitter | `typescript-language-server` → `javascript` |
+| `.cpp` | `conservative` | ✅ | ✅ tree-sitter | `clangd` → `cpp` |
+| `.cs` | `conservative` | ✅ | ✅ tree-sitter | `csharp-ls` → `csharp` |
+| `.css` | `aggressive` | ✅ | — | `vscode-css-language-server` → `css` |
+| `.cts` | `conservative` | ✅ | ✅ tree-sitter | `typescript-language-server` → `typescript` |
+| `.cxx` | `conservative` | ✅ | ✅ tree-sitter | `clangd` → `cpp` |
+| `.erl` | `aggressive` | ✅ | ✅ tree-sitter | — |
+| `.ex` | `aggressive` | ✅ | ✅ tree-sitter | — |
+| `.exs` | `aggressive` | ✅ | ✅ tree-sitter | — |
+| `.gemspec` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.go` | `conservative` | ✅ | ✅ tree-sitter | `gopls` → `go` |
+| `.h` | `conservative` | ✅ | ✅ tree-sitter | `clangd` → `c` |
+| `.hcl` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.hh` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.hpp` | `conservative` | ✅ | ✅ tree-sitter | `clangd` → `cpp` |
+| `.hrl` | `aggressive` | ✅ | ✅ tree-sitter | — |
+| `.htm` | `aggressive` | ✅ | — | `vscode-html-language-server` → `html` |
+| `.html` | `aggressive` | ✅ | — | `vscode-html-language-server` → `html` |
+| `.hxx` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.java` | `conservative` | ✅ | ✅ tree-sitter | `jdtls` → `java` |
+| `.jl` | `conservative` | ✅ | — | — |
+| `.js` | `terser` | ✅ | ✅ tree-sitter | `typescript-language-server` → `javascript` |
+| `.json` | `json` | ✅ | — | `vscode-json-language-server` → `json` |
+| `.jsonc` | `json` | ✅ | — | `vscode-json-language-server` → `json` |
+| `.jsx` | `terser` | ✅ | ✅ tree-sitter | `typescript-language-server` → `javascriptreact` |
+| `.kt` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.kts` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.less` | `aggressive` | ✅ | — | `vscode-css-language-server` → `less` |
+| `.lua` | `aggressive` | ✅ | ✅ tree-sitter | — |
+| `.mjs` | `terser` | ✅ | ✅ tree-sitter | `typescript-language-server` → `javascript` |
+| `.ml` | `conservative` | ✅ | — | — |
+| `.mli` | `conservative` | ✅ | — | — |
+| `.mts` | `conservative` | ✅ | ✅ tree-sitter | `typescript-language-server` → `typescript` |
+| `.php` | `conservative` | ✅ | ✅ tree-sitter | `intelephense` → `php` |
+| `.proto` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.py` | `conservative` | ✅ | ✅ tree-sitter | `pylsp` → `python` |
+| `.pyi` | `conservative` | ✅ | ✅ tree-sitter | `pylsp` → `python` |
+| `.r` | `aggressive` | ✅ | ✅ tree-sitter | — |
+| `.rake` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.rb` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.rs` | `conservative` | ✅ | ✅ tree-sitter | `rust-analyzer` → `rust` |
+| `.ru` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.sbt` | — | ✅ | ✅ tree-sitter | — |
+| `.sc` | — | ✅ | ✅ tree-sitter | — |
+| `.scala` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.scss` | `aggressive` | ✅ | — | `vscode-css-language-server` → `scss` |
+| `.sh` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.sql` | `conservative` | ✅ | — | `sqls` → `sql` |
+| `.swift` | `conservative` | ✅ | ✅ tree-sitter | `sourcekit-lsp` → `swift` |
+| `.tf` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.tfvars` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.toml` | `conservative` | ✅ | — | — |
+| `.ts` | `conservative` | ✅ | ✅ tree-sitter | `typescript-language-server` → `typescript` |
+| `.tsx` | `conservative` | ✅ | ✅ tree-sitter | `typescript-language-server` → `typescriptreact` |
+| `.yaml` | `conservative` | ✅ | — | `yaml-language-server` → `yaml` |
+| `.yml` | `conservative` | ✅ | — | `yaml-language-server` → `yaml` |
+| `.zig` | `conservative` | ✅ | ✅ tree-sitter | — |
+| `.zsh` | `conservative` | ✅ | ✅ tree-sitter | — |
+
+Notes:
+- **Signature outline is tree-sitter only** — markup/style/config grammars (HTML/CSS/SCSS/LESS/Scala/JSON/YAML/TOML) parse for structural `rule` queries but have no function body, so no skeleton. There is **no** regex/heuristic fallback.
+- **`.jsx`** resolves the LSP server as `javascriptreact` (to enable JSX) even though its tree-sitter grammar registry id is `javascript` (shared JS grammar). `.tsx` has its own grammar, so both ids are `typescriptreact`.
+- **`.hh` / `.hxx`** have the C++ grammar + signatures but **no clangd server config** (only `.cpp/.cc/.cxx/.hpp` are mapped).
+- **C/C++**: structural `rule` queries (e.g. `kind: call_expression`) work fully; a bare call-shaped `pattern` can hit tree-sitter's declaration-vs-call ambiguity — prefer a `rule` with `kind`. JS/TS also have a native (oxc) symbol/in-file-reference path that needs **no server installed**.
+
+### Minify-only formats
+
+Native comment/whitespace stripping; no AST/LSP. (90 extensions, grouped by strategy.)
+
+**`aggressive`** (18): `clj` `cljs` `ejs` `erb` `handlebars` `hbs` `jinja` `jinja2` `mustache` `pl` `pm` `svelte` `svg` `twig` `vue` `xml` `xsl` `xslt`
+
+**`conservative`** (66): `adb` `ads` `asm` `awk` `bzl` `cfg` `cmake` `coffee` `conf` `config` `csv` `dart` `dockerignore` `elm` `env` `f` `f03` `f08` `f90` `f95` `fish` `for` `fs` `fsx` `gitignore` `gql` `gradle` `graphql` `groovy` `haml` `hs` `ini` `jade` `kotlin` `lhs` `lisp` `lsp` `mm` `nasm` `nim` `nix` `pas` `perl` `plsql` `pp` `properties` `ps1` `psd1` `psm1` `pug` `rkt` `rst` `rust` `sass` `scm` `slim` `star` `styl` `tsql` `v` `vb` `vbs` `vhd` `vhdl` `wast` `wat`
+
+**`general`** (2): `log` `txt`
+
+**`json`** (1): `json5`
+
+**`markdown`** (3): `markdown` `md` `mdx`
+
+### Verify
+
+```bash
+yarn matrix:check     # this matrix, live
+yarn ast:check        # structural search + signatures on real samples
+yarn lsp:check        # language-id + server resolution + native semantics
+yarn lsp:live         # spawn a real server, exercise every LSP operation type
+yarn minify:check     # minifier over every configured format
+yarn benchmark        # all of the above
+```
+
+<!-- END GENERATED: support-matrix -->
+
+## Lifecycle — pool, cold start, indexing
+
+- **Pool** (`lspClientPool.ts`): one warm `LSPClient` per (server × workspace), 60s idle timeout (`OCTOCODE_LSP_POOL_IDLE_MS`). A long-lived MCP session reuses warm servers across tool calls; one-shot CLI invocations don't share a pool.
+- **Cold start / indexing**: a server reads the project and builds its model before answering correctly. Costs vary — typescript-language-server <1s, gopls 3–15s, rust-analyzer 5–60s (multiple `$/progress` waves), jdtls 30–120s.
+- **Readiness** (`manager.ts` + `json_rpc.rs`): for servers that emit `$/progress` (go, rust, java, csharp, swift) the pool factory calls `waitForReady` with a per-language cap before the first query; servers without `$/progress` (TS/JS, Python, clangd, data formats) skip the wait to avoid a fixed 2s settle penalty.
+- **Spawn gate**: every resolved command passes `validateLSPServerPath` (rejects shell wrappers / nonexistent / non-executable) in `LSPClient.start()` before the process is spawned.
+- **Discovery caching** (`serverDiscovery.ts`): ecosystem-dir lookup results are memoised per `(command, workspaceRoot)` for the process lifetime. Ecosystem dirs are pre-filtered to existing ones once, cutting stat calls from ~15-per-server to ~5. Call `clearDiscoveryCache()` (or restart) after installing a server mid-session.
+
+## Open / future (non-blocking)
+
+Progress streaming to the caller (`lsp.indexingStatus`), an opt-in `waitForIndexingMs`, tree-sitter symbol extraction for compiled-language `documentSymbols`, and a SCIP precomputed index — all deferred; none change the no-fallback contract above.
+
+> **Known issue — cold `references` under-reports.** On a one-shot CLI invocation,
+> `references` (and other project-wide ops) can return incomplete results
+> *labelled* `complete=true`, because `typescript-language-server` emits no
+> `$/progress` and the project isn't indexed yet when queried. This is the one
+> behaviour that can make an agent wrong rather than just slow — analysis and the
+> proposed fix (honest `complete=false` + cross-check hint, with an opt-in
+> `waitForIndexingMs`) are in
+> [`LSP_REFERENCES_INDEXING_RFC.md`](https://github.com/bgauryy/octocode/blob/main/docs/context/LSP_REFERENCES_INDEXING_RFC.md). Meanwhile,
+> prefer `--op callers` or `localSearchCode` to confirm "who uses this".

package/index.d.ts

@@ -4,7 +4,8 @@ export declare class NativeLspClient {
   constructor(config: JsLanguageServerConfig)
   start(): Promise<void>
   stop(): Promise<void>
-  waitForReady(timeoutMs?: number | undefined | null): Promise<void>
+  /** Readiness after post-`initialized` indexing: `progressIdle` = a `$/progress` cycle drained to idle; `settledFallback` = no progress seen, only the settle window elapsed; `timeout` = progress still active at the deadline. */
+  waitForReady(timeoutMs?: number | undefined | null): Promise<'progressIdle' | 'settledFallback' | 'timeout'>
   hasCapability(capability: string): boolean
   /** Server-selected LSP `positionEncoding` (utf-16 unless the server is non-conformant); null if omitted/not started. */
   positionEncoding(): string | null
@@ -44,94 +45,6 @@ export declare function applyContentViewMinification(content: string, filePath:
  */
 export declare function applyMinification(content: string, filePath: string): string
 
-/**
- * Structural inspection of a binary (executable / object / archive) file.
- *
- * `format`/`arch`/`bits`/`endianness`/`stripped` are always populated (the
- * fields the old `identify` mode produced). The list fields are populated only
- * when the file is a recognized executable format `goblin` could parse; for an
- * unrecognized file they stay empty and `notes` explains why (e.g. "use
- * mode=list/decompress for archives").
- *
- * Every list is capped (see `crate::binary::inspect::LIST_CAP`); the `*_count`
- * fields carry the true totals, and `truncated` is set when any list was cut.
- */
-export interface BinaryInspectInfo {
-  /** elf | macho | macho-fat | pe | coff | archive | wasm | unknown */
-  format: string
-  /** Human-readable one-line summary (drop-in for `file -b`). */
-  description: string
-  /** Space-separated hex of the leading bytes (drop-in for `xxd -p -l 32`). */
-  magicHex: string
-  arch?: string
-  bits?: number
-  /** "little" | "big" */
-  endianness?: string
-  stripped?: boolean
-  /** Hex entry-point address, when the format has one. */
-  entry?: string
-  symbols: Array<string>
-  imports: Array<string>
-  exports: Array<string>
-  sections: Array<string>
-  /** Dynamic dependencies / needed shared libraries. */
-  libraries: Array<string>
-  symbolCount: number
-  importCount: number
-  exportCount: number
-  /** True when any list was capped. */
-  truncated: boolean
-  /** Advisory notes (unrecognized format, parse degradation, size truncation). */
-  notes: Array<string>
-}
-
-/** Result of a strings extraction pass over a binary buffer. */
-export interface BinaryStrings {
-  /**
-   * Printable runs, longest-first. ASCII **and** UTF-16 (LE/BE) — the win
-   * over GNU `strings -a`, which misses wide strings. Each entry is prefixed
-   * with its hex byte offset when offsets were requested.
-   */
-  strings: Array<string>
-  /** Total runs found before any display capping. */
-  totalFound: number
-  /**
-   * True when more of the file remains to scan beyond this window — follow
-   * `next_scan_offset`. Lossless continuation cursor, **not** a data-loss
-   * flag (the old fixed-cap meaning): every byte is reachable by paging.
-   */
-  truncated: boolean
-  /**
-   * Absolute byte offset to start the next scan window, or `None` at EOF.
-   * Rewound to a safe break so no string is split across windows.
-   */
-  nextScanOffset?: number
-}
-
-/**
- * Native strings extraction. Recovers printable ASCII **and** UTF-16 (LE/BE)
- * runs of at least `min_length` from the scan window of `path` beginning at
- * `scan_offset`, longest-first, optionally hex offset-prefixed. Replaces the
- * `strings` shell-out and additionally surfaces the wide strings GNU
- * `strings -a` misses.
- *
- * Lossless pagination: the returned `nextScanOffset` (when set) is the absolute
- * byte offset of the next window, rewound to a safe break so no string is split
- * across windows. Pass `scanOffset = 0` for the first window.
- */
-export declare function extractBinaryStringsNative(path: string, minLength: number, includeOffsets: boolean, scanOffset: number): BinaryStrings
-
-/**
- * Native binary inspection (format lane). Parses `path` as an executable /
- * object / archive and returns its identity plus — for recognized executable
- * formats — symbols, imports, exports, sections and dynamic dependencies.
- *
- * Replaces the `file` + `xxd` shell-outs. Never throws on malformed input: a
- * parse failure degrades to magic-byte identity with an explanatory note. The
- * only `Err` cases are unreadable / oversized files.
- */
-export declare function inspectBinaryNative(path: string): BinaryInspectInfo
-
 /** Extract a byte-range substring from `content`. */
 export declare function byteSliceContent(content: string, byteStart: number, byteEnd: number): string
 
@@ -442,6 +355,19 @@ export interface FilterPatchOptions {
   contextLines?: number
 }
 
+/**
+ * Myers line diff (`oldText` → `newText`). Returns a full edit script of
+ * `{ opType, line }` ops (`same` | `add` | `remove`). Prefer this over an
+ * O(N·M) LCS for agent edit previews on mid/large files.
+ */
+export declare function computeLineDiff(oldText: string, newText: string): Array<LineDiffOp>
+
+export interface LineDiffOp {
+  /** `"same"` | `"add"` | `"remove"` */
+  opType: string
+  line: string
+}
+
 /** Convert a `file://` URI string back to an absolute filesystem path. */
 export declare function fromUri(uri: string): string
 
@@ -489,8 +415,10 @@ export declare const SUPPORTED_STRUCTURAL_EXTENSIONS: readonly string[]
  *
  * Char offsets match JavaScript `string.substring()` — pass them directly to
  * JavaScript string slicing without conversion.
+ *
+ * Runs on libuv's worker pool (tree-sitter parse) — returns a Promise.
  */
-export declare function getSemanticBoundaryOffsets(content: string, filePath: string): Array<number>
+export declare function getSemanticBoundaryOffsets(content: string, filePath: string): Promise<Array<number>>
 
 /**
  * Returns all extensions that have signature extraction support
@@ -943,8 +871,13 @@ export interface StructuralQueryExplanation {
 
 export interface StructuralDetailedMatch extends StructuralMatch {
   id: string
+  /** tree-sitter `kind` of the matched node (e.g. `call_expression`). */
   nodeKind?: string
-  confidence: 'exact-ast' | 'partial-ast' | 'fallback-text' | string
+  /**
+   * The octo matcher is a precise AST matcher, so every match is an exact
+   * tree-sitter node match — there is no partial/fallback tier.
+   */
+  confidence: 'exact-ast'
 }
 
 export interface StructuralSearchDetailedResult {
@@ -973,8 +906,10 @@ export interface StructuralSearchDetailedResult {
  * (exactly one). Returns node ranges (1-based lines, ready as `lineHint`s)
  * plus captured metavariables. Throws on unsupported extension, invalid
  * pattern/rule, or both/neither query supplied.
+ *
+ * Runs on libuv's worker pool (tree-sitter parse) — returns a Promise.
  */
-export declare function structuralSearch(content: string, filePath: string, pattern?: string | undefined | null, rule?: string | undefined | null): Array<StructuralMatch>
+export declare function structuralSearch(content: string, filePath: string, pattern?: string | undefined | null, rule?: string | undefined | null): Promise<Array<StructuralMatch>>
 
 /**
  * Detailed structural search. Unsupported extensions and invalid queries return
@@ -987,7 +922,11 @@ export interface StructuralSearchFileResult {
   matches: Array<StructuralMatch>
 }
 
-export declare function structuralSearchFiles(options: StructuralSearchFilesOptions): StructuralSearchFilesResult
+/**
+ * Runs on libuv's worker pool — the directory walk + per-file parse is
+ * CPU/IO-bound and can span thousands of files. Returns a Promise.
+ */
+export declare function structuralSearchFiles(options: StructuralSearchFilesOptions): Promise<StructuralSearchFilesResult>
 
 export declare function structuralSearchFilesDetailed(options: StructuralSearchFilesOptions): StructuralSearchFilesDetailedResult
 
@@ -996,7 +935,25 @@ export interface StructuralSearchFilesOptions {
   pattern?: string
   rule?: string
   include?: Array<string>
+  /**
+   * File-path globs to skip (gitignore-style, e.g. `"*.min.js"`, `"src/gen/**"`).
+   * Mirrors `localSearchCode.exclude` so OQL `scope.exclude` is honored on the
+   * structural lane — previously silently dropped (typed-contract violation).
+   */
+  exclude?: Array<string>
   excludeDir?: Array<string>
+  /**
+   * Include hidden (dot) files. `None` preserves the default walker behavior
+   * (hidden ignored); `Some(true)` forces them in.
+   */
+  hidden?: boolean
+  /**
+   * Bypass `.gitignore`/`.ignore` rules. `None` preserves defaults; `Some(true)`
+   * searches files normally hidden by ignore files (mirrors `localSearchCode.noIgnore`).
+   */
+  noIgnore?: boolean
+  /** Maximum directory descent depth (0 = just the root). `None` = unbounded. */
+  maxDepth?: number
   maxFiles?: number
   maxFileBytes?: number
 }
@@ -1006,6 +963,13 @@ export interface StructuralSearchFilesResult {
   totalMatches: number
   parsedFiles: number
   skippedByPreFilter: number
+  /**
+   * Candidate files whose extension has no grammar — not evaluated, hence
+   * not proof of absence. Mirrors the detailed result's counter so the two
+   * shapes agree and the warning text can't collapse unevaluated into
+   * anchor-absent.
+   */
+  skippedUnsupported: number
   skippedUnreadable: number
   skippedLarge: number
   warnings: Array<string>

package/index.js

@@ -42,6 +42,7 @@ export const structuralSearchFiles = nativeBinding.structuralSearchFiles
 export const structuralSearchFilesDetailed = nativeBinding.structuralSearchFilesDetailed
 export const getSupportedStructuralExtensions = nativeBinding.getSupportedStructuralExtensions
 export const getSemanticBoundaryOffsets = nativeBinding.getSemanticBoundaryOffsets
+
 export const getSupportedSignatureExtensions = nativeBinding.getSupportedSignatureExtensions
 export const jsonToYamlString = nativeBinding.jsonToYamlString
 export const getMINIFY_CONFIG = nativeBinding.getMINIFY_CONFIG
@@ -59,9 +60,8 @@ export const byteSliceContent = nativeBinding.byteSliceContent
 export const sliceContent = nativeBinding.sliceContent
 export const extractMatchingLines = nativeBinding.extractMatchingLines
 export const filterPatch = nativeBinding.filterPatch
+export const computeLineDiff = nativeBinding.computeLineDiff
 export const PatchLineType = nativeBinding.PatchLineType
-export const inspectBinaryNative = nativeBinding.inspectBinaryNative
-export const extractBinaryStringsNative = nativeBinding.extractBinaryStringsNative
 export const NativeLspClient = nativeBinding.NativeLspClient
 export const resolvePosition = nativeBinding.resolvePosition
 export const resolvePositionFromContent = nativeBinding.resolvePositionFromContent

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@octocodeai/octocode-engine",
-  "version": "16.6.2",
+  "version": "17.0.1",
   "description": "Rust native Octocode engine for context compression, structural search, and LSP semantic navigation",
   "type": "module",
   "main": "index.cjs",
@@ -12,7 +12,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/bgauryy/octocode-mcp.git",
+    "url": "https://github.com/bgauryy/octocode.git",
     "directory": "packages/octocode-engine"
   },
   "exports": {
@@ -256,18 +256,19 @@
   },
   "files": [
     "dist",
+    "docs/**",
     "index.cjs",
     "index.js",
     "index.d.ts",
     "README.md"
   ],
   "optionalDependencies": {
-    "@octocodeai/octocode-engine-darwin-arm64": "16.6.2",
-    "@octocodeai/octocode-engine-darwin-x64": "16.6.2",
-    "@octocodeai/octocode-engine-linux-arm64-gnu": "16.6.2",
-    "@octocodeai/octocode-engine-linux-x64-gnu": "16.6.2",
-    "@octocodeai/octocode-engine-linux-x64-musl": "16.6.2",
-    "@octocodeai/octocode-engine-win32-x64-msvc": "16.6.2"
+    "@octocodeai/octocode-engine-darwin-arm64": "17.0.1",
+    "@octocodeai/octocode-engine-darwin-x64": "17.0.1",
+    "@octocodeai/octocode-engine-linux-arm64-gnu": "17.0.1",
+    "@octocodeai/octocode-engine-linux-x64-gnu": "17.0.1",
+    "@octocodeai/octocode-engine-linux-x64-musl": "17.0.1",
+    "@octocodeai/octocode-engine-win32-x64-msvc": "17.0.1"
   },
   "engines": {
     "node": ">=20.0.0"
@@ -302,8 +303,10 @@
     "platforms:check": "node scripts/check-platform-binaries.cjs",
     "version:check": "node scripts/check-version-consistency.cjs",
     "version:sync": "node scripts/sync-versions.cjs",
+    "manifest:gen": "node scripts/gen-server-manifest.mjs",
+    "verify:manifest": "node scripts/gen-server-manifest.mjs && git diff --exit-code src/lsp/serverManifestData.ts || (echo 'ERROR: src/lsp/serverManifestData.ts is out of sync with serverManifest.json — run yarn manifest:gen and commit the result' && exit 1)",
     "loader:check": "node scripts/check-loader-entries.cjs",
-    "prepublish:verify": "yarn version:check && yarn loader:check && yarn pack:check && yarn platforms:check",
+    "prepublish:verify": "yarn version:check && yarn loader:check && yarn verify:manifest && yarn pack:check && yarn platforms:check",
     "binary-size:check": "node scripts/check-binary-size.cjs",
     "ast:check": "yarn workspace @octocodeai/octocode-benchmark ast:check",
     "lsp:check": "yarn workspace @octocodeai/octocode-benchmark lsp:check",
@@ -316,7 +319,7 @@
     "check:rust": "cargo check --all-targets --all-features",
     "fmt:rust": "cargo fmt --all",
     "fmt:rust:check": "cargo fmt --all -- --check",
-    "test:rust": "cargo test --all-targets --all-features",
+    "test:rust": "cargo test --lib --tests --all-features && cargo build --benches --all-features",
     "test:node": "vitest run",
     "test:node:coverage": "vitest run --coverage",
     "test:watch": "vitest watch",
@@ -332,12 +335,13 @@
     "audit:rust": "cargo audit -D unmaintained -D unsound -D yanked",
     "audit": "yarn audit:rust",
     "verify:rust": "yarn check:rust && yarn fmt:rust:check && yarn lint:rust && yarn test:rust && yarn deps:rust && yarn udeps:rust && yarn audit:rust",
-    "verify": "yarn verify:patterns && yarn typecheck && yarn verify:rust && yarn test && yarn benchmark && yarn pack:check && yarn version:check && yarn loader:check && yarn binary-size:check",
+    "verify": "yarn verify:patterns && yarn verify:manifest && yarn typecheck && yarn verify:rust && yarn test && yarn benchmark && yarn pack:check && yarn version:check && yarn loader:check && yarn binary-size:check",
     "readme:sync": "node ../../scripts/sync-package-readmes.mjs .",
     "prepack": "yarn readme:sync",
     "prepublishOnly": "yarn version:sync"
   },
   "dependencies": {
+    "@octocodeai/config": "17.0.0",
     "bash-language-server": "^5.6.0",
     "intelephense": "^1.14.1",
     "pyright": "^1.1.411",