open-context-engine 0.1.0

package/src/retrieval/cascade.py

@@ -0,0 +1,142 @@
+"""One neural ranking stage after cheap, parallel facet/structure retrieval.
+
+Query-independent code structure is prepared once. Query facets interact with
+stored code vectors numerically; they do not trigger separate neural reranks.
+Selection considers both individual spans and complete small function bundles.
+"""
+from collections import defaultdict
+import math
+import time
+
+import numpy as np
+
+from engine import Engine, document, post
+
+VERSION = 'structural-cascade-v2'
+POLICY = {'candidateLimit': 96, 'facetSeeds': 4, 'neighborsPerSeed': 10,
+          'facetPool': 24, 'maxFunctionBudgetFraction': .45,
+          'embeddingScoreTemperature': .06, 'functionCompletionFactor': 1.25}
+
+
+class CascadeEngine(Engine):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.functions = defaultdict(list)
+        for u in self.units:
+            if u['kind'] == 'function':
+                self.functions[u['symbol']].append(u['id'])
+        for group in self.functions.values():
+            group.sort(key=lambda i: self.units[i]['start'])
+
+    def recall(self, queries, dense, facets):
+        pools, fused = [], defaultdict(float)
+        local_scores = []
+        for col, query in enumerate(queries):
+            lexical = self.lexical(query + (' ' + ' '.join(facets[col-1]['terms']) if col else ''))
+            local = defaultdict(float)
+            for scores in (dense[:, col], lexical):
+                for rank, uid in enumerate(np.argsort(-scores)[:60]):
+                    if scores[uid] > 0:
+                        local[int(uid)] += 1 / (30 + rank)
+            pool = sorted(local, key=lambda i: (-local[i], i))[:POLICY['facetPool']]
+            pools.append(pool)
+            local_scores.append(local)
+            for uid, score in local.items():
+                # Max preserves a facet-specific specialist; mean rewards breadth.
+                fused[uid] = max(fused[uid], score)
+        for uid in list(fused):
+            fused[uid] += .35 * sum(s.get(uid, 0) for s in local_scores) / len(queries)
+        seeds = set(uid for pool in pools for uid in pool[:POLICY['facetSeeds']])
+        expanded = set()
+        graph_prior = defaultdict(float)
+        for uid in seeds:
+            neighbors = set(self.units[uid]['edges'] + self.incoming[uid])
+            ranked = sorted(neighbors, key=lambda i: (-fused[i], i))[:POLICY['neighborsPerSeed']]
+            for neighbor in ranked:
+                expanded.add(neighbor)
+                graph_prior[neighbor] = max(graph_prior[neighbor], fused[uid] * .5)
+        candidates = set().union(*(set(pool) for pool in pools), expanded)
+        ordered = sorted(candidates, key=lambda i: (-(fused[i] + graph_prior[i]), i))
+        # Retain direct hits from every facet before applying the shared ceiling.
+        protected = list(dict.fromkeys(uid for pool in pools for uid in pool[:12]))
+        retained = list(dict.fromkeys(protected + ordered))[:POLICY['candidateLimit']]
+        return retained, expanded, len(candidates)
+
+    def pack(self, retained, scores, affinity, budget):
+        position = {uid: i for i, uid in enumerate(retained)}
+        actions = [(uid,) for uid in retained]
+        for group in self.functions.values():
+            if len(group) > 1 and all(uid in position for uid in group) and sum(self.costs[uid] for uid in group) <= budget * POLICY['maxFunctionBudgetFraction']:
+                actions.append(tuple(group))
+        relevance = np.asarray([scores[uid] for uid in retained])
+        values = relevance[:, None] * (.35 + .65 * affinity)
+        covered = np.zeros(affinity.shape[1])
+        selected, selected_set, trace, spent = [], set(), [], 0
+        while actions:
+            choices = []
+            for action in actions:
+                fresh = [uid for uid in action if uid not in selected_set]
+                cost = sum(self.costs[uid] for uid in fresh)
+                if not fresh or spent + cost > budget:
+                    continue
+                indices = [position[uid] for uid in fresh]
+                benefit = values[indices].max(axis=0)
+                base = float(relevance[indices].max())
+                gain = .7 * float(np.mean(benefit / (1 + covered))) + .3 * base
+                if len(action) > 1:
+                    gain *= POLICY['functionCompletionFactor']
+                gain /= (max(120, cost) / 300) ** .35
+                choices.append((gain, action, fresh, cost, benefit))
+            if not choices:
+                break
+            gain, action, fresh, cost, benefit = max(choices, key=lambda c: (c[0], -c[2][0]))
+            if gain < .015:
+                break
+            actions.remove(action)
+            selected.extend(fresh)
+            selected_set.update(fresh)
+            spent += cost
+            covered += benefit
+            trace.append({'ids': fresh, 'functionBundle': len(action) > 1,
+                          'tokens': cost, 'gain': gain, 'scores': [scores[i] for i in fresh]})
+        raw = '\n'.join(self.render(self.units[uid]) for uid in selected)
+        return raw, trace
+
+    def search(self, plan, budget=4000):
+        start = time.monotonic()
+        facets = plan['facets']
+        queries = [plan['intent']] + [f['question'] for f in facets]
+        # Encode repeated intent/facet text once, without caching across requests.
+        unique_queries = list(dict.fromkeys(queries))
+        result = post(self.embed_url + '/embeddings', {'model': 'Qwen3-Embedding-4B',
+            'input': ['Instruct: Retrieve source code implementing the requested behavior.\nQuery: ' + q for q in unique_queries]}, self.embedding_key)
+        unique_vectors = [r['embedding'] for r in sorted(result['data'], key=lambda r: r['index'])]
+        vectors = np.asarray([unique_vectors[unique_queries.index(q)] for q in queries], dtype=np.float32)
+        embedded_at = time.monotonic()
+        dense = self.vectors @ vectors.T
+        retained, expanded, candidate_count = self.recall(queries, dense, facets)
+        recalled_at = time.monotonic()
+        reranked = post(self.reranker['baseUrl'] + '/rerank', {'model': self.reranker['model'],
+            'query': plan['intent'], 'documents': [document(self.units[uid], 5000) for uid in retained]}, self.reranker['apiKey'])
+        rows = reranked['results']
+        if len(rows) != len(retained) or {r['index'] for r in rows} != set(range(len(retained))) or any(not math.isfinite(r['relevance_score']) or not 0 <= r['relevance_score'] <= 1 for r in rows):
+            raise ValueError('Invalid rerank mapping or scores')
+        scores = {retained[r['index']]: r['relevance_score'] for r in rows}
+        ranked_at = time.monotonic()
+        # Soft maximum relative to each facet's best candidate avoids assuming
+        # cosine similarities are calibrated cross-language probabilities.
+        facet_dense = dense[retained, 1:]
+        affinity = np.exp(np.minimum(0, facet_dense - facet_dense.max(axis=0)) / POLICY['embeddingScoreTemperature'])
+        raw, selected = self.pack(retained, scores, affinity, budget)
+        finished = time.monotonic()
+        return raw, {'version': VERSION, 'elapsedMs': round((finished-start)*1000),
+            'tokens': len(self.encoding.encode(raw)), 'candidateCount': candidate_count,
+            'rerankedCount': len(retained), 'expandedCount': len(expanded),
+            'modelRequests': {'embedding': 1, 'rerank': 1}, 'queryCache': False,
+            'timingMs': {'embedding': round((embedded_at-start)*1000), 'recall': round((recalled_at-embedded_at)*1000),
+                'rerank': round((ranked_at-recalled_at)*1000), 'rerankModel': reranked.get('meta', {}).get('elapsed_ms'),
+                'selection': round((finished-ranked_at)*1000)},
+            'rerankInputTokens': reranked.get('usage', {}).get('input_tokens'),
+            'candidates': [{'id': uid, 'score': scores[uid], 'path': self.units[uid]['path'],
+                'start': self.units[uid]['start'], 'end': self.units[uid]['end']} for uid in retained],
+            'selected': selected, 'plan': plan, 'policy': POLICY}

package/src/retrieval/engine.py

@@ -0,0 +1,195 @@
+"""Language-neutral retrieval over snapshot-bound source units. Models are called over HTTP, never loaded.
+
+AST spans, conservative static links, lexical/dense fusion, facet reranking and
+whole-span packing are repository independent. No evaluation labels are read.
+"""
+from collections import Counter, defaultdict
+import hashlib
+import json
+import math
+import re
+import time
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+import numpy as np
+import tiktoken
+
+from languages import source_units, adapter_manifest
+
+VERSION = 'structural-units-v2'
+
+
+def terms(text):
+    text = re.sub(r'([a-z])([A-Z])', r'\1 \2', text)
+    return re.findall(r'[a-z][a-z0-9]+', text.lower().replace('_', ' '))
+
+
+def post(url, payload, key='local-only', timeout=120):
+    request = Request(url, json.dumps(payload).encode(), {
+        'Content-Type': 'application/json', 'Authorization': 'Bearer ' + key})
+    with urlopen(request, timeout=timeout) as response:
+        return json.load(response)
+
+
+def document(u, limit=2200):
+    return ('Path: ' + u['path'] + '\nSymbol: ' + u['name'] + '\n' + u['text'])[:limit]
+
+
+class Engine:
+    def __init__(self, units, vectors, embed_url, reranker, embedding_key='local-only', embedding_model='Qwen3-Embedding-4B'):
+        self.units, self.vectors = units, vectors
+        self.embed_url, self.reranker = embed_url, reranker
+        self.embedding_key = embedding_key
+        self.embedding_model = embedding_model
+        self.encoding = tiktoken.get_encoding('cl100k_base')
+        self.postings = defaultdict(list)
+        lengths = []
+        for u in units:
+            counts = Counter(terms((u['path'] + ' ' + u['name'] + ' ') * 3 + u['text']))
+            lengths.append(sum(counts.values()))
+            for term, frequency in counts.items():
+                self.postings[term].append((u['id'], frequency))
+        self.lengths = np.asarray(lengths)
+        self.average_length = max(1, self.lengths.mean())
+        self.incoming = defaultdict(list)
+        for u in units:
+            for target in u['edges']:
+                self.incoming[target].append(u['id'])
+        self.costs = [len(self.encoding.encode(self.render(u))) + 2 for u in units]
+
+    @staticmethod
+    def render(u):
+        # Text units preserve Unicode separators inside a physical source line.
+        lines = u['text'].split('\n') if u.get('language') in {'text', 'javascript', 'typescript', 'go'} else u['text'].splitlines()
+        return f"Path: {u['path']}\n" + '\n'.join(f'{i}\t{line}' for i, line in
+            enumerate(lines, u['start'])) + '\n'
+
+    def lexical(self, query):
+        scores = np.zeros(len(self.units))
+        for term in set(terms(query)):
+            matches = self.postings.get(term, [])
+            if not matches:
+                continue
+            ids, tf = np.asarray(matches).T
+            idf = math.log(1 + (len(self.units) - len(ids) + .5) / (len(ids) + .5))
+            scores[ids] += idf * tf * 2.2 / (tf + 1.2 * (.25 + .75 * self.lengths[ids] / self.average_length))
+        return scores
+
+    def rerank(self, query, ids):
+        if not ids:
+            return {}
+        data = post(self.reranker['baseUrl'] + '/rerank', {'model': self.reranker['model'],
+            'query': query, 'documents': [document(self.units[i], 5000) for i in ids]}, self.reranker['apiKey'])
+        if len(data['results']) != len(ids):
+            raise ValueError('Incomplete rerank response')
+        return {ids[r['index']]: r['relevance_score'] for r in data['results']}
+
+    def search(self, plan, budget=4000):
+        start = time.monotonic()
+        facets = plan['facets']
+        queries = [plan['intent']] + [f['question'] for f in facets]
+        embedded = post(self.embed_url + '/embeddings', {'model': self.embedding_model,
+            'input': ['Instruct: Retrieve source code implementing the requested behavior.\nQuery: ' + q for q in queries]}, self.embedding_key)
+        qvectors = np.asarray([r['embedding'] for r in sorted(embedded['data'], key=lambda r: r['index'])], dtype=np.float32)
+        dense = self.vectors @ qvectors.T
+        pools, fused = [], defaultdict(float)
+        for col, query in enumerate(queries):
+            lexical = self.lexical(query + (' ' + ' '.join(facets[col-1]['terms']) if col else ''))
+            local = defaultdict(float)
+            for scores in (dense[:, col], lexical):
+                for rank, uid in enumerate(np.argsort(-scores)[:40]):
+                    if scores[uid] > 0:
+                        local[int(uid)] += 1 / (30 + rank)
+            pool = sorted(local, key=local.get, reverse=True)[:28]
+            pools.append(pool)
+            for uid, score in local.items():
+                fused[uid] += score
+        facet_scores = []
+        for query, pool in zip(queries, pools):
+            facet_scores.append(self.rerank(query, pool))
+        seeds = set()
+        for scores in facet_scores:
+            seeds.update(sorted(scores, key=scores.get, reverse=True)[:3])
+        candidates = set().union(*(set(s) for s in facet_scores))
+        expanded = set()
+        for uid in seeds:
+            # Only statically resolved links. Dynamic receivers are not guessed.
+            neighbors = self.units[uid]['edges'] + self.incoming[uid]
+            expanded.update(sorted(set(neighbors), key=lambda x: fused[x], reverse=True)[:10])
+        candidates.update(expanded)
+        ranked = sorted(candidates, key=lambda uid: max(s.get(uid, 0) for s in facet_scores) + min(.15, fused[uid]), reverse=True)
+        retained = ranked[:80]
+        # Ensure graph-only discoveries get a chance in the joint rerank.
+        retained = list(dict.fromkeys(retained[:64] + sorted(expanded, key=lambda x: fused[x], reverse=True)[:16]))
+        overall = self.rerank(plan['intent'], retained)
+        # Score newly discovered graph spans against each facet, too.
+        for col, facet in enumerate(facets, 1):
+            missing = [uid for uid in retained if uid not in facet_scores[col]]
+            facet_scores[col].update(self.rerank(facet['question'], missing))
+        covered = np.zeros(len(facets))
+        selected, spent, trace = [], 0, []
+        available = set(retained)
+        while available:
+            choices = []
+            for uid in available:
+                if spent + self.costs[uid] > budget:
+                    continue
+                values = np.asarray([s.get(uid, 0) for s in facet_scores[1:]])
+                # Diminishing coverage gain keeps later stages from being crowded out.
+                gain = float(np.sum(values / (1 + covered))) / len(facets)
+                gain = .7 * gain + .3 * overall.get(uid, 0)
+                gain /= (max(120, self.costs[uid]) / 300) ** .35
+                choices.append((gain, uid, values))
+            if not choices:
+                break
+            gain, uid, values = max(choices, key=lambda item: (item[0], -item[1]))
+            if gain < .015:
+                break
+            available.remove(uid)
+            selected.append(uid)
+            spent += self.costs[uid]
+            covered += values
+            trace.append({'id': uid, 'overall': overall.get(uid, 0), 'facets': values.tolist(),
+                          'tokens': self.costs[uid], 'gain': gain, 'graphExpanded': uid in expanded})
+        raw = '\n'.join(self.render(self.units[uid]) for uid in selected)
+        return raw, {'elapsedMs': round((time.monotonic()-start)*1000), 'tokens': len(self.encoding.encode(raw)),
+            'candidateCount': len(candidates), 'rerankedCount': len(retained), 'expandedCount': len(expanded),
+            'selected': trace, 'plan': plan}
+
+
+def build_index(root, snapshot, state, embed_url):
+    state = Path(state)
+    state.mkdir(parents=True, exist_ok=True)
+    adapters = adapter_manifest(snapshot['files'], snapshot.get('languageOptions'))
+    identity = hashlib.sha256((VERSION + json.dumps(snapshot, sort_keys=True)
+                              + json.dumps(adapters, sort_keys=True)).encode()).hexdigest()
+    metadata = state / 'metadata.json'
+    if metadata.exists() and json.loads(metadata.read_text())['identity'] == identity:
+        return json.loads((state / 'units.json').read_text()), np.load(state / 'vectors.npy'), {
+            **json.loads(metadata.read_text()), 'cacheHit': True}
+    start = time.monotonic()
+    selection = {}
+    units = source_units(root, snapshot['files'], language_options=snapshot.get('languageOptions'), report=selection)
+    if not units:
+        raise ValueError('No indexable nonempty source text in snapshot')
+    vectors = []
+    for offset in range(0, len(units), 64):
+        result = post(embed_url + '/embeddings', {'model': 'Qwen3-Embedding-4B',
+            'input': [document(u) for u in units[offset:offset+64]]}, timeout=300)
+        rows = sorted(result['data'], key=lambda r: r['index'])
+        vectors.extend(r['embedding'] for r in rows)
+        if offset % 512 == 0:
+            print(json.dumps({'stage': 'indexing', 'done': len(vectors), 'total': len(units)}), flush=True)
+    matrix = np.asarray(vectors, dtype=np.float32)
+    if matrix.shape != (len(units), 1024) or not np.isfinite(matrix).all():
+        raise ValueError('Invalid index vectors')
+    (state / 'units.json').write_text(json.dumps(units))
+    np.save(state / 'vectors.npy', matrix)
+    summary = {'identity': identity, 'version': VERSION, 'units': len(units),
+        'files': len(snapshot['files']), 'nonemptyFiles': len({u['path'] for u in units}),
+        'selection': selection,
+        'languageAdapters': adapters, 'languageUnits': dict(Counter(u['language'] for u in units)),
+        'resolvedEdges': sum(len(u['edges']) for u in units), 'indexingMs': round((time.monotonic()-start)*1000), 'cacheHit': False}
+    metadata.write_text(json.dumps(summary, indent=2))
+    return units, matrix, summary

package/src/retrieval/entities.py

@@ -0,0 +1,187 @@
+"""Retrieve and assemble source entities, retaining their constituent spans.
+
+Each entity has multiple stored vectors (one per source span). Cheap maximum
+interaction with query facets locates entities; a single neural ranking stage
+scores their representative code. Static links then supply bounded evidence
+completion without another model call. No task-specific symbols or labels.
+"""
+from collections import defaultdict
+import math
+import time
+import numpy as np
+
+from engine import Engine, post
+
+VERSION = 'entity-cascade-v3'
+POLICY = {'candidateEntities': 80, 'perFacetEntities': 32, 'anchorEntities': 8,
+          'graphHops': 2, 'maxAssembledEntities': 128, 'maxAtomicBudgetFraction': .65,
+          'facetTemperature': .06, 'documentChars': 5000}
+
+
+class EntityEngine(Engine):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        groups = defaultdict(list)
+        for u in self.units:
+            groups[(u['symbol'], u['kind'])].append(u['id'])
+        self.entities = []
+        self.unit_entity = np.zeros(len(self.units), dtype=np.int32)
+        for (symbol, kind), ids in groups.items():
+            eid = len(self.entities)
+            ids.sort(key=lambda i: self.units[i]['start'])
+            self.entities.append({'symbol': symbol, 'kind': kind, 'ids': ids,
+                                  'cost': sum(self.costs[uid] for uid in ids)})
+            self.unit_entity[ids] = eid
+        self.links = [dict() for _ in self.entities]
+        for u in self.units:
+            source = int(self.unit_entity[u['id']])
+            for target_unit in u['edges']:
+                target = int(self.unit_entity[target_unit])
+                if source == target:
+                    continue
+                other = self.units[target_unit]
+                # Parent context is weaker than a statically resolved callee.
+                forward = .45 if u['owner'] == other['symbol'] else .85
+                reverse = .65 if u['owner'] == other['symbol'] else .55
+                self.links[source][target] = max(self.links[source].get(target, 0), forward)
+                self.links[target][source] = max(self.links[target].get(source, 0), reverse)
+
+    def aggregate(self, unit_scores):
+        shape = (len(self.entities),) + unit_scores.shape[1:]
+        scores = np.full(shape, -np.inf, dtype=np.float32)
+        np.maximum.at(scores, self.unit_entity, unit_scores)
+        return scores
+
+    def candidate_entities(self, queries, facets, dense):
+        fused, pools = defaultdict(float), []
+        for col, query in enumerate(queries):
+            lex = self.aggregate(self.lexical(query + (' ' + ' '.join(facets[col-1]['terms']) if col else '')))
+            local = defaultdict(float)
+            for scores in [dense[:, col], lex]:
+                for rank, eid in enumerate(np.argsort(-scores)[:80]):
+                    if scores[eid] > 0:
+                        local[int(eid)] += 1 / (30 + rank)
+            ordered = sorted(local, key=lambda i: (-local[i], i))[:POLICY['perFacetEntities']]
+            pools.append(ordered)
+            for eid, score in local.items():
+                fused[eid] = max(fused[eid], score)
+        protected = list(dict.fromkeys(eid for pool in pools for eid in pool[:12]))
+        available = set(eid for pool in pools for eid in pool)
+        ordered = sorted(available, key=lambda i: (-fused[i], i))
+        return list(dict.fromkeys(protected + ordered))[:POLICY['candidateEntities']]
+
+    def representation(self, eid, unit_dense):
+        entity = self.entities[eid]
+        ids = entity['ids']
+        # Long functions are represented by their strongest matching code spans,
+        # not always their prefix. Every selected span remains source anchored.
+        ranked = sorted(ids, key=lambda i: (-float(unit_dense[i].max()), i))
+        header = self.units[ids[0]]
+        parts = [f"Path: {header['path']}\nSymbol: {header['name']}\n"]
+        remaining = POLICY['documentChars'] - len(parts[0])
+        for uid in ranked:
+            u = self.units[uid]
+            value = f"Lines {u['start']}-{u['end']}:\n{u['text']}\n"
+            parts.append(value[:remaining])
+            remaining -= min(len(value), remaining)
+            if remaining <= 0:
+                break
+        return ''.join(parts)
+
+    def complete(self, candidates, scores, dense):
+        expanded_scores = dict(scores)
+        provenance = {}
+        anchors = sorted(candidates, key=lambda i: (-scores[i], i))[:POLICY['anchorEntities']]
+        frontier = {eid: scores[eid] for eid in anchors if scores[eid] >= .15}
+        visited = set(frontier)
+        for hop in range(POLICY['graphHops']):
+            next_frontier = {}
+            for source, confidence in frontier.items():
+                neighbors = self.links[source]
+                # Rank by semantic compatibility before following broad class links.
+                ordered = sorted(neighbors, key=lambda i: (-float(dense[i].max()), i))[:16]
+                for target in ordered:
+                    compatibility = float(np.exp(min(0, float(dense[target].max()-dense[source].max())) / .12))
+                    proposal = confidence * neighbors[target] * (.4 + .6 * compatibility)
+                    if proposal > expanded_scores.get(target, 0):
+                        expanded_scores[target] = proposal
+                        provenance[target] = {'anchor': source, 'hop': hop+1, 'kind': 'static-relationship', 'inferredScore': proposal}
+                    if target not in visited and proposal > .05:
+                        next_frontier[target] = max(next_frontier.get(target, 0), proposal)
+            visited.update(next_frontier)
+            frontier = next_frontier
+        selected = sorted(expanded_scores, key=lambda i: (-expanded_scores[i], i))[:POLICY['maxAssembledEntities']]
+        return selected, expanded_scores, provenance
+
+    def pack_entities(self, entities, scores, dense, unit_dense, budget):
+        affinity = np.exp(np.minimum(0, dense[:, 1:]-dense[:, 1:].max(axis=0)) / POLICY['facetTemperature'])
+        covered = np.zeros(affinity.shape[1])
+        actions = []
+        for eid in entities:
+            entity = self.entities[eid]
+            if entity['cost'] <= budget * POLICY['maxAtomicBudgetFraction']:
+                actions.append((eid, tuple(entity['ids'])))
+            else:
+                # Oversized entities remain accessible at their original AST spans.
+                actions.extend((eid, (uid,)) for uid in entity['ids'])
+        spent, selected, trace = 0, [], []
+        while actions:
+            best = None
+            for eid, ids in actions:
+                cost = sum(self.costs[i] for i in ids)
+                if spent+cost > budget:
+                    continue
+                facet = affinity[eid]
+                if len(ids) == 1 and len(self.entities[eid]['ids']) > 1:
+                    facet = np.exp(np.minimum(0, unit_dense[ids[0], 1:]-dense[:, 1:].max(axis=0)) / POLICY['facetTemperature'])
+                value = scores[eid] * (.35 + .65 * facet)
+                gain = .7 * float(np.mean(value/(1+covered))) + .3 * scores[eid]
+                gain /= (max(120, cost)/300) ** .35
+                choice = (gain, eid, ids, cost, value)
+                if best is None or gain > best[0]:
+                    best = choice
+            if best is None or best[0] < .015:
+                break
+            gain, eid, ids, cost, value = best
+            actions.remove((eid, ids))
+            selected.extend(ids)
+            spent += cost
+            covered += value
+            trace.append({'entity': eid, 'ids': list(ids), 'symbol': self.entities[eid]['symbol'],
+                'tokens': cost, 'score': scores[eid], 'completeEntity': len(ids)==len(self.entities[eid]['ids'])})
+        return '\n'.join(self.render(self.units[i]) for i in selected), trace
+
+    def search(self, plan, budget=4000):
+        start = time.monotonic()
+        facets = plan['facets']
+        queries = [plan['intent']] + [f['question'] for f in facets]
+        unique = list(dict.fromkeys(queries))
+        result = post(self.embed_url+'/embeddings', {'model':'Qwen3-Embedding-4B',
+            'input':['Instruct: Retrieve source code implementing the requested behavior.\nQuery: '+q for q in unique]}, self.embedding_key)
+        vectors = [r['embedding'] for r in sorted(result['data'],key=lambda r:r['index'])]
+        embedded = time.monotonic()
+        unit_dense = self.vectors @ np.asarray([vectors[unique.index(q)] for q in queries],dtype=np.float32).T
+        dense = self.aggregate(unit_dense)
+        candidates = self.candidate_entities(queries,facets,dense)
+        documents = [self.representation(eid,unit_dense) for eid in candidates]
+        recalled = time.monotonic()
+        response = post(self.reranker['baseUrl']+'/rerank',{'model':self.reranker['model'],
+            'query':plan['intent'],'documents':documents},self.reranker['apiKey'])
+        rows = response['results']
+        if len(rows)!=len(candidates) or {r['index'] for r in rows}!=set(range(len(candidates))) or any(not math.isfinite(r['relevance_score']) or not 0<=r['relevance_score']<=1 for r in rows):
+            raise ValueError('Invalid rerank response')
+        scores = {candidates[r['index']]:r['relevance_score'] for r in rows}
+        ranked = time.monotonic()
+        assembled, expanded_scores, provenance = self.complete(candidates,scores,dense)
+        raw,selected = self.pack_entities(assembled,expanded_scores,dense,unit_dense,budget)
+        finished = time.monotonic()
+        return raw,{'version':VERSION,'elapsedMs':round((finished-start)*1000),'tokens':len(self.encoding.encode(raw)),
+            'queryCache':False,'modelRequests':{'embedding':1,'rerank':1},'candidateCount':len(candidates),
+            'expandedCount':len(provenance),'assembledCount':len(assembled),'policy':POLICY,'plan':plan,
+            'timingMs':{'embedding':round((embedded-start)*1000),'recall':round((recalled-embedded)*1000),
+                'rerank':round((ranked-recalled)*1000),'rerankModel':response.get('meta',{}).get('elapsed_ms'),
+                'selection':round((finished-ranked)*1000)},'rerankInputTokens':response.get('usage',{}).get('input_tokens'),
+            'candidates':[{'entity':eid,'ids':self.entities[eid]['ids'],'symbol':self.entities[eid]['symbol'],
+                'score':expanded_scores[eid],'source':'neural+graph' if eid in scores and eid in provenance else 'neural' if eid in scores else 'graph',
+                'spans':[{'path':self.units[i]['path'],'start':self.units[i]['start'],'end':self.units[i]['end']} for i in self.entities[eid]['ids']]} for eid in assembled],
+            'selected':selected,'provenance':provenance}

package/src/retrieval/languages/__init__.py

@@ -0,0 +1,129 @@
+"""Snapshot-bound language adapters; downstream retrieval consumes only CodeUnit."""
+from collections import defaultdict
+from copy import deepcopy
+import hashlib
+import json
+from pathlib import Path, PurePosixPath
+import sys
+
+from . import python, typescript, text, go
+from .files import path_exclusion, read_text
+from .schema import SCHEMA_VERSION, SourceFile, validate_units
+
+ADAPTERS = {'python': python, 'typescript': typescript, 'javascript': typescript, 'go': go, 'text': text}
+EXTENSIONS = {'.py': 'python', '.ts': 'typescript', '.tsx': 'typescript',
+              '.mts': 'typescript', '.cts': 'typescript',
+              '.go': 'go', '.js': 'javascript', '.jsx': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript'}
+
+
+def language_for(path):
+    return EXTENSIONS.get(PurePosixPath(path).suffix.lower(), 'text')
+
+
+def adapter_manifest(files, language_options=None):
+    languages = sorted({language_for(file['path']) for file in files})
+    options = language_options or {}
+    if not isinstance(options, dict) or set(options) - set(ADAPTERS):
+        raise ValueError('Unknown language options')
+    paths = [Path(__file__), Path(__file__).with_name('schema.py'), Path(__file__).with_name('files.py')]
+    for language in languages:
+        paths.append(Path(ADAPTERS[language].__file__))
+        if language in {'typescript', 'javascript'}:
+            paths.append(Path(typescript.__file__).with_suffix('.mjs'))
+        if language == 'go':
+            paths.append(Path(go.__file__).with_name('go_ast.go'))
+            paths.append(Path(go.__file__).with_name('go_types.go'))
+    return {'schemaVersion': SCHEMA_VERSION, 'languages': languages, 'options': options,
+            'parsers': {language: (f'python-ast-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}'
+                                  if language == 'python' else f'typescript-{typescript.COMPILER_VERSION}'
+                                  if language in {'typescript', 'javascript'} else go.compiler()[1]
+                                  if language == 'go' else text.VERSION)
+                        for language in languages},
+            'sourceSha256': {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in paths}}
+
+
+def source_units(root, files, max_lines=65, language_options=None, report=None, cache=None):
+    if type(max_lines) is not int or max_lines < 1:
+        raise ValueError('max_lines must be a positive integer')
+    root = Path(root).resolve()
+    groups, sources, seen, excluded = defaultdict(list), [], set(), []
+    options = language_options or {}
+    adapter_manifest(files, options)  # Validate configuration before reading source.
+    for file in files:
+        name = file['path']
+        path = PurePosixPath(name)
+        if (path.is_absolute() or '..' in path.parts or '\\' in name or str(path) != name
+                or name in seen):
+            raise ValueError('Invalid or duplicate snapshot path: ' + name)
+        seen.add(name)
+        resolved = (root/name).resolve()
+        if not resolved.is_relative_to(root):
+            raise ValueError('Source escapes snapshot root: ' + name)
+        reason = path_exclusion(name)
+        if (root/name).is_symlink():
+            reason = 'symlink'
+        if reason:
+            excluded.append({'path': name, 'reason': reason})
+            continue
+        raw, content, reason = read_text(resolved)
+        if reason:
+            excluded.append({'path': name, 'reason': reason})
+            continue
+        if hashlib.sha256(raw).hexdigest() != file['sha256']:
+            raise ValueError('Source changed: ' + name)
+        source = SourceFile(name, content, file['sha256'])
+        sources.append(source)
+        language = language_for(name)
+        groups['typescript' if language == 'javascript' else language].append(source)
+    by_path = defaultdict(list)
+    identities = {}
+    pending_cache = {}
+    for language, subset in groups.items():
+        settings = options.get(language)
+        if language == 'go':
+            settings = go.settings(settings, sources)
+        if language == 'typescript' and 'javascript' in options:
+            if settings is not None and settings != options['javascript']:
+                raise ValueError('JavaScript and TypeScript share compiler options')
+            settings = options['javascript']
+        # Bound cache lifetime to the current snapshot. Structural languages are
+        # invalidated together so imports and callers in unchanged files refresh.
+        batches = [[source] for source in subset] if cache is not None and language == 'text' else [subset]
+        units = []
+        for batch in batches:
+            key = (language, batch[0].path if language == 'text' else '')
+            fingerprint = (hashlib.sha256(json.dumps([
+                [(s.path, s.sha256) for s in batch], max_lines, settings,
+                adapter_manifest([{'path': s.path} for s in batch], options),
+            ], sort_keys=True).encode()).hexdigest() if cache is not None else None)
+            previous = cache.get(key) if cache is not None else None
+            extracted = (deepcopy(previous[1]) if previous and previous[0] == fingerprint
+                         else ADAPTERS[language].extract(batch, max_lines, settings))
+            if cache is not None:
+                pending_cache[key] = (fingerprint, deepcopy(extracted))
+            offset = len(units)
+            for unit in extracted:
+                unit['id'] += offset
+                for relation in unit['relations']:
+                    relation['target'] += offset
+                unit['edges'] = [target + offset for target in unit['edges']]
+            units.extend(extracted)
+        validate_units(units, subset)
+        for unit in units:
+            by_path[unit['path']].append(unit)
+            identities[id(unit)] = (language, unit['id'])
+    units = [unit for source in sources for unit in by_path[source.path]]
+    remap = {identities[id(unit)]: i for i, unit in enumerate(units)}
+    for i, unit in enumerate(units):
+        unit['id'] = i
+        for relation in unit['relations']:
+            relation['target'] = remap[(identities[id(unit)][0], relation['target'])]
+        unit['edges'] = sorted({relation['target'] for relation in unit['relations']})
+    validate_units(units, sources)
+    if cache is not None:
+        cache.clear()
+        cache.update(pending_cache)
+    if report is not None:
+        report.update(inputFiles=len(files), acceptedFiles=len(sources), excluded=excluded,
+                      fallbackFiles=sum(language_for(source.path) == 'text' for source in sources))
+    return units