@elyracode/semantic-index 0.9.3

package/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+## [0.9.3] - 2026-05-29
+
+### Added
+- Initial release: `semantic_index_build` and `semantic_search` tools for local semantic code search over an OpenAI-compatible embeddings endpoint (OpenAI, Ollama, LM Studio)
+- `elyra-semantic-index` skill

package/README.md

@@ -0,0 +1,48 @@
+# @elyracode/semantic-index
+
+Local semantic code search for Elyra. Embeds your codebase so the agent can find relevant code by meaning, not just keywords.
+
+## Install
+
+```
+elyra install npm:@elyracode/semantic-index
+```
+
+## Configuration
+
+Configure the embeddings endpoint via environment variables. The defaults use the OpenAI API, but you can point at any OpenAI-compatible endpoint -- including fully local ones.
+
+| Env var | Description | Default |
+|---------|-------------|---------|
+| `ELYRA_EMBED_BASE_URL` | OpenAI-compatible embeddings base URL. For fully local/private indexing, point at Ollama (`http://localhost:11434/v1`) or LM Studio. | `https://api.openai.com/v1` |
+| `ELYRA_EMBED_MODEL` | Embedding model. For Ollama use `nomic-embed-text`. | `text-embedding-3-small` |
+| `ELYRA_EMBED_API_KEY` | API key for the embeddings endpoint. Falls back to `OPENAI_API_KEY`. Not needed for local Ollama. | -- |
+
+## Tools
+
+| Tool | Description |
+|------|-------------|
+| `semantic_index_build` | Index the project's tracked source files. Run this once, and re-run after big changes. |
+| `semantic_search` | Find code relevant to a natural-language query. |
+
+## How it works
+
+The index is stored locally at `.elyra/semantic-index.json` and never leaves your machine -- the only network traffic is the embedding requests to your configured endpoint.
+
+For full data sovereignty, use a local embeddings endpoint (such as Ollama) so nothing leaves your machine at all.
+
+## Usage
+
+Once installed, ask Elyra to find code by meaning:
+
+```
+> Where is rate limiting handled?
+> How does the app verify webhook signatures?
+> Find the code that retries failed jobs
+```
+
+The agent builds the index with `semantic_index_build`, then uses `semantic_search` to locate relevant code.
+
+## Skill
+
+This package includes the `elyra-semantic-index` skill, which guides the agent on when and how to use semantic search effectively.

package/extensions/index.ts

@@ -0,0 +1,301 @@
+import { execSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import type { ExtensionAPI } from "@elyracode/coding-agent";
+import { Type } from "typebox";
+
+// ── Configuration ───────────────────────────────────────────────────────────
+
+interface EmbedConfig {
+	baseUrl: string;
+	model: string;
+	apiKey: string | undefined;
+}
+
+function getEmbedConfig(): EmbedConfig {
+	const baseUrl = (process.env.ELYRA_EMBED_BASE_URL || "https://api.openai.com/v1").replace(/\/+$/, "");
+	const model = process.env.ELYRA_EMBED_MODEL || "text-embedding-3-small";
+	const apiKey = process.env.ELYRA_EMBED_API_KEY || process.env.OPENAI_API_KEY;
+	return { baseUrl, model, apiKey };
+}
+
+// ── Embeddings client ───────────────────────────────────────────────────────
+
+/** Request embeddings for a batch of inputs from an OpenAI-compatible endpoint. */
+async function embed(config: EmbedConfig, inputs: string[]): Promise<number[][]> {
+	const headers: Record<string, string> = { "Content-Type": "application/json" };
+	if (config.apiKey) headers.Authorization = `Bearer ${config.apiKey}`;
+
+	const response = await fetch(`${config.baseUrl}/embeddings`, {
+		method: "POST",
+		headers,
+		body: JSON.stringify({ model: config.model, input: inputs }),
+		signal: AbortSignal.timeout(60_000),
+	});
+	if (!response.ok) {
+		const text = await response.text();
+		throw new Error(`Embeddings request failed (${response.status}): ${text.slice(0, 300)}`);
+	}
+	const data = (await response.json()) as { data?: Array<{ embedding: number[] }> };
+	if (!data.data || data.data.length !== inputs.length) {
+		throw new Error("Embeddings response did not match the number of inputs");
+	}
+	return data.data.map((d) => d.embedding);
+}
+
+// ── File discovery & chunking ───────────────────────────────────────────────
+
+const SOURCE_EXTENSIONS = new Set([
+	"ts", "tsx", "js", "jsx", "mjs", "cjs", "php", "py", "rb", "go", "rs", "java", "kt", "swift",
+	"c", "h", "cpp", "hpp", "cc", "cs", "scala", "ex", "exs", "vue", "svelte", "css", "scss",
+	"sql", "sh", "bash", "md", "json", "yaml", "yml", "toml",
+]);
+
+const MAX_FILE_BYTES = 256 * 1024;
+const CHUNK_LINES = 40;
+const CHUNK_OVERLAP = 8;
+
+interface Chunk {
+	file: string;
+	startLine: number;
+	endLine: number;
+	text: string;
+}
+
+/** List tracked source files via git (respects .gitignore, skips node_modules). */
+function listSourceFiles(cwd: string): string[] {
+	let output: string;
+	try {
+		output = execSync("git ls-files", { cwd, encoding: "utf-8", maxBuffer: 32 * 1024 * 1024 });
+	} catch {
+		return [];
+	}
+	return output
+		.split("\n")
+		.map((f) => f.trim())
+		.filter(Boolean)
+		.filter((f) => {
+			const ext = f.split(".").pop()?.toLowerCase();
+			return ext !== undefined && SOURCE_EXTENSIONS.has(ext);
+		});
+}
+
+/** Split a file into overlapping line-windows. */
+function chunkFile(relPath: string, content: string): Chunk[] {
+	const lines = content.split("\n");
+	const chunks: Chunk[] = [];
+	const step = CHUNK_LINES - CHUNK_OVERLAP;
+	for (let start = 0; start < lines.length; start += step) {
+		const end = Math.min(start + CHUNK_LINES, lines.length);
+		const text = lines.slice(start, end).join("\n").trim();
+		if (text.length > 0) {
+			chunks.push({ file: relPath, startLine: start + 1, endLine: end, text });
+		}
+		if (end >= lines.length) break;
+	}
+	return chunks;
+}
+
+// ── Index storage ───────────────────────────────────────────────────────────
+
+interface IndexedChunk extends Chunk {
+	vector: number[];
+}
+
+interface IndexFile {
+	version: 1;
+	model: string;
+	createdAt: string;
+	chunks: IndexedChunk[];
+}
+
+function indexPath(cwd: string): string {
+	return join(cwd, ".elyra", "semantic-index.json");
+}
+
+function saveIndex(cwd: string, index: IndexFile): void {
+	const path = indexPath(cwd);
+	mkdirSync(dirname(path), { recursive: true });
+	const tmp = `${path}.tmp`;
+	writeFileSync(tmp, JSON.stringify(index));
+	renameSync(tmp, path); // atomic replace
+}
+
+function loadIndex(cwd: string): IndexFile | null {
+	const path = indexPath(cwd);
+	if (!existsSync(path)) return null;
+	try {
+		const parsed = JSON.parse(readFileSync(path, "utf-8")) as IndexFile;
+		if (parsed.version === 1 && Array.isArray(parsed.chunks)) return parsed;
+	} catch {
+		// corrupt index
+	}
+	return null;
+}
+
+// ── Similarity ──────────────────────────────────────────────────────────────
+
+function cosineSimilarity(a: number[], b: number[]): number {
+	let dot = 0;
+	let normA = 0;
+	let normB = 0;
+	const len = Math.min(a.length, b.length);
+	for (let i = 0; i < len; i++) {
+		dot += a[i] * b[i];
+		normA += a[i] * a[i];
+		normB += b[i] * b[i];
+	}
+	if (normA === 0 || normB === 0) return 0;
+	return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+// ── Result helpers ──────────────────────────────────────────────────────────
+
+function ok(text: string) {
+	return { content: [{ type: "text" as const, text }] };
+}
+
+function err(text: string) {
+	return { content: [{ type: "text" as const, text }], isError: true as const };
+}
+
+// ── Extension ───────────────────────────────────────────────────────────────
+
+export default function (elyra: ExtensionAPI): void {
+	let cwd = "";
+	elyra.on("session_start", async (_event, ctx) => {
+		cwd = ctx.cwd;
+	});
+	const getCwd = () => cwd || process.cwd();
+
+	// ── semantic_index_build ─────────────────────────────────────────────
+
+	elyra.registerTool({
+		name: "semantic_index_build",
+		label: "Build Semantic Index",
+		description:
+			"Index the project's tracked source files for semantic search. Reads each file, splits it into chunks, embeds them via the configured embeddings endpoint, and stores the index locally at .elyra/semantic-index.json. Run this once per project, and again after large changes. Requires an embeddings endpoint (ELYRA_EMBED_BASE_URL / ELYRA_EMBED_MODEL / ELYRA_EMBED_API_KEY or OPENAI_API_KEY).",
+		parameters: Type.Object({
+			max_files: Type.Optional(
+				Type.Integer({ description: "Cap the number of files indexed (useful for very large repos). Default: no cap." }),
+			),
+		}),
+		promptSnippet: "Build a semantic index of the codebase",
+		async execute(_id, params) {
+			const config = getEmbedConfig();
+			const cwdNow = getCwd();
+
+			let files = listSourceFiles(cwdNow);
+			if (files.length === 0) {
+				return err("No tracked source files found (is this a git repository?).");
+			}
+			if (params.max_files && params.max_files > 0) {
+				files = files.slice(0, params.max_files);
+			}
+
+			// Build chunks from readable, reasonably-sized files.
+			const chunks: Chunk[] = [];
+			for (const rel of files) {
+				const abs = join(cwdNow, rel);
+				try {
+					if (statSync(abs).size > MAX_FILE_BYTES) continue;
+					const content = readFileSync(abs, "utf-8");
+					chunks.push(...chunkFile(rel, content));
+				} catch {
+					// skip unreadable files
+				}
+			}
+
+			if (chunks.length === 0) {
+				return err("No content to index after chunking.");
+			}
+
+			// Embed in batches.
+			const BATCH = 64;
+			const indexed: IndexedChunk[] = [];
+			try {
+				for (let i = 0; i < chunks.length; i += BATCH) {
+					const batch = chunks.slice(i, i + BATCH);
+					const vectors = await embed(config, batch.map((c) => c.text));
+					for (let j = 0; j < batch.length; j++) {
+						indexed.push({ ...batch[j], vector: vectors[j] });
+					}
+				}
+			} catch (error) {
+				const msg = error instanceof Error ? error.message : String(error);
+				return err(`Failed to embed chunks: ${msg}`);
+			}
+
+			const index: IndexFile = {
+				version: 1,
+				model: config.model,
+				createdAt: new Date().toISOString(),
+				chunks: indexed,
+			};
+			try {
+				saveIndex(cwdNow, index);
+			} catch (error) {
+				const msg = error instanceof Error ? error.message : String(error);
+				return err(`Failed to save index: ${msg}`);
+			}
+
+			return ok(
+				`Indexed ${indexed.length} chunks from ${files.length} files using ${config.model}. Saved to .elyra/semantic-index.json. Use semantic_search to query it.`,
+			);
+		},
+	});
+
+	// ── semantic_search ──────────────────────────────────────────────────
+
+	elyra.registerTool({
+		name: "semantic_search",
+		label: "Semantic Search",
+		description:
+			"Find code relevant to a natural-language query by meaning, not just keywords. Returns the most semantically similar code chunks with their file path and line range. Requires a built index (run semantic_index_build first). Use this to locate where a concept is implemented, or to gather relevant context before making changes.",
+		parameters: Type.Object({
+			query: Type.String({ description: "Natural-language description of what you're looking for" }),
+			limit: Type.Optional(Type.Integer({ description: "Number of results to return. Default 8, max 25." })),
+		}),
+		promptSnippet: "Search the codebase by meaning",
+		async execute(_id, params) {
+			const cwdNow = getCwd();
+			const index = loadIndex(cwdNow);
+			if (!index) {
+				return err("No semantic index found. Run semantic_index_build first.");
+			}
+
+			const config = getEmbedConfig();
+			if (index.model !== config.model) {
+				return err(
+					`Index was built with model "${index.model}" but the current model is "${config.model}". Rebuild with semantic_index_build.`,
+				);
+			}
+
+			let queryVector: number[];
+			try {
+				[queryVector] = await embed(config, [params.query]);
+			} catch (error) {
+				const msg = error instanceof Error ? error.message : String(error);
+				return err(`Failed to embed query: ${msg}`);
+			}
+
+			const limit = Math.min(Math.max(params.limit ?? 8, 1), 25);
+			const scored = index.chunks
+				.map((chunk) => ({ chunk, score: cosineSimilarity(queryVector, chunk.vector) }))
+				.sort((a, b) => b.score - a.score)
+				.slice(0, limit);
+
+			if (scored.length === 0) {
+				return ok("No matching chunks.");
+			}
+
+			const lines: string[] = [`Top ${scored.length} matches for: ${params.query}\n`];
+			for (const { chunk, score } of scored) {
+				lines.push(`── ${chunk.file}:${chunk.startLine}-${chunk.endLine}  (score ${score.toFixed(3)})`);
+				lines.push(chunk.text);
+				lines.push("");
+			}
+			return ok(lines.join("\n"));
+		},
+	});
+}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@elyracode/semantic-index",
+  "version": "0.9.3",
+  "description": "Local semantic code search for Elyra -- embed your codebase and find relevant code by meaning, not just keywords",
+  "type": "module",
+  "keywords": [
+    "elyra-package",
+    "semantic-search",
+    "embeddings",
+    "code-search",
+    "rag"
+  ],
+  "license": "MIT",
+  "author": "Knut W. Horne",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kwhorne/elyra.git",
+    "directory": "packages/semantic-index"
+  },
+  "elyra": {
+    "extensions": [
+      "./extensions/index.ts"
+    ],
+    "skills": [
+      "./skills"
+    ]
+  },
+  "peerDependencies": {
+    "@elyracode/coding-agent": "*",
+    "typebox": "*"
+  },
+  "scripts": {
+    "clean": "echo 'nothing to clean'",
+    "build": "echo 'nothing to build'",
+    "check": "echo 'nothing to check'"
+  }
+}

package/skills/elyra-semantic-index/SKILL.md

@@ -0,0 +1,32 @@
+---
+name: elyra-semantic-index
+description: Find code by meaning, not just keywords. Use when looking for where something is implemented, how a concept is handled across the codebase, or finding relevant code before making changes -- especially in large or unfamiliar codebases.
+---
+
+# Semantic Code Search
+
+## When to Use
+
+Use semantic search when:
+- You need to find where something is implemented but don't know the exact symbol or file name
+- You want to understand how a concept (auth, retries, caching, validation) is handled across the codebase
+- You're gathering relevant code before making a change
+- You're working in a large or unfamiliar codebase where grep by keyword isn't enough
+
+## Available Tools
+
+| Tool | Use when |
+|------|----------|
+| `semantic_index_build` | Build or refresh the index of the project's tracked source files |
+| `semantic_search` | Find code relevant to a natural-language query |
+
+## Workflow
+
+1. Build the index once per project with `semantic_index_build`. Re-run it after large changes to keep results accurate.
+2. Use `semantic_search` with a natural-language query to find conceptually relevant code.
+3. Combine approaches: use `grep` for exact symbol or string matches, and `semantic_search` for conceptual matches when you don't know the exact terms.
+
+## Notes
+
+- If `semantic_search` reports that no index exists, run `semantic_index_build` first, then retry the query.
+- Requires an embeddings endpoint configured via environment variables (`ELYRA_EMBED_BASE_URL`, `ELYRA_EMBED_MODEL`, `ELYRA_EMBED_API_KEY`). A local endpoint such as Ollama keeps everything on your machine.