open-context-engine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +177 -0
- package/assets/brand/logo-lockup-dark.svg +14 -0
- package/assets/brand/logo-lockup.svg +14 -0
- package/assets/brand/logo.svg +9 -0
- package/bin/opencontextengine.mjs +64 -0
- package/docs/QUICKSTART.md +192 -0
- package/docs/RERANKER_API.md +32 -0
- package/package.json +81 -0
- package/requirements.txt +2 -0
- package/scripts/mcp-opencontextengine.mjs +37 -0
- package/scripts/retrieval-server.py +135 -0
- package/src/client.mjs +37 -0
- package/src/config.mjs +49 -0
- package/src/environment.mjs +10 -0
- package/src/eval/remote-models.mjs +70 -0
- package/src/mcp.mjs +58 -0
- package/src/retrieval/batched.py +220 -0
- package/src/retrieval/cascade.py +142 -0
- package/src/retrieval/engine.py +195 -0
- package/src/retrieval/entities.py +187 -0
- package/src/retrieval/languages/__init__.py +129 -0
- package/src/retrieval/languages/files.py +90 -0
- package/src/retrieval/languages/go.py +154 -0
- package/src/retrieval/languages/go_ast.go +204 -0
- package/src/retrieval/languages/go_types.go +169 -0
- package/src/retrieval/languages/python.py +113 -0
- package/src/retrieval/languages/schema.py +81 -0
- package/src/retrieval/languages/text.py +39 -0
- package/src/retrieval/languages/typescript.mjs +233 -0
- package/src/retrieval/languages/typescript.py +23 -0
- package/src/retrieval/live.py +273 -0
- package/src/retrieval/reranker.py +83 -0
- package/src/retrieval/routed.py +35 -0
- package/src/runtime.mjs +60 -0
- package/src/service.mjs +77 -0
- package/src/setup.mjs +66 -0
- package/src/workspaces.mjs +59 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Shared file eligibility for discovery and explicitly supplied snapshots."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path, PurePosixPath
|
|
5
|
+
import re
|
|
6
|
+
import subprocess
|
|
7
|
+
|
|
8
|
+
MAX_BYTES = 1024 * 1024
|
|
9
|
+
SKIP_DIRS = frozenset({'.git', '.hg', '.svn', 'node_modules', 'vendor', '.venv', 'venv',
|
|
10
|
+
'__pycache__', 'dist', 'build', 'target', '.next', '.cache',
|
|
11
|
+
'.pilot-state', 'runs'})
|
|
12
|
+
SKIP_SUFFIXES = ('.min.js', '.min.css', '.map', '.pyc', '.pyo', '.so', '.dylib', '.dll',
|
|
13
|
+
'.exe', '.o', '.a', '.class', '.jar', '.wasm', '.zip', '.gz', '.tar',
|
|
14
|
+
'.png', '.jpg', '.jpeg', '.gif', '.webp', '.ico', '.pdf', '.woff', '.woff2',
|
|
15
|
+
'.mp3', '.mp4', '.sqlite', '.db', '.pem', '.key')
|
|
16
|
+
LOCKFILES = frozenset({'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'poetry.lock',
|
|
17
|
+
'uv.lock', 'Cargo.lock', 'go.sum'})
|
|
18
|
+
GENERATED = re.compile(r'(?im)^\s*(?://|#|/\*|\*)\s*(?:code generated\b[^\n]*do not edit|'
|
|
19
|
+
r'(?:this file (?:is|was) |@)?(?:auto[- ]?)?generated\b[^\n]*do not edit)')
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def path_exclusion(name):
|
|
23
|
+
path = PurePosixPath(name)
|
|
24
|
+
if set(path.parts[:-1]) & SKIP_DIRS:
|
|
25
|
+
return 'dependency-or-build-directory'
|
|
26
|
+
if path.name in LOCKFILES or name.lower().endswith(SKIP_SUFFIXES):
|
|
27
|
+
return 'generated-or-non-source-name'
|
|
28
|
+
if path.name == '.env' or (path.name.startswith('.env.') and path.name not in {'.env.example', '.env.sample'}):
|
|
29
|
+
return 'local-environment'
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def read_text(path):
|
|
34
|
+
"""Return decoded source or an explicit skip reason; never replace bad bytes."""
|
|
35
|
+
if path.stat().st_size > MAX_BYTES:
|
|
36
|
+
return None, None, 'file-too-large'
|
|
37
|
+
raw = path.read_bytes()
|
|
38
|
+
if len(raw) > MAX_BYTES:
|
|
39
|
+
return None, None, 'file-too-large'
|
|
40
|
+
if any(byte < 32 and byte not in (9, 10, 12, 13) for byte in raw):
|
|
41
|
+
return raw, None, 'binary-content'
|
|
42
|
+
try:
|
|
43
|
+
text = raw.decode('utf-8-sig')
|
|
44
|
+
except UnicodeDecodeError:
|
|
45
|
+
return raw, None, 'non-utf8'
|
|
46
|
+
if GENERATED.search(text[:2048]):
|
|
47
|
+
return raw, None, 'generated-header'
|
|
48
|
+
return raw, text, None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def discover_snapshot(root):
|
|
52
|
+
"""Use Git's ignore rules when available, otherwise a bounded directory walk."""
|
|
53
|
+
root = Path(root).resolve()
|
|
54
|
+
if not root.is_dir():
|
|
55
|
+
raise ValueError('Source root must be a directory')
|
|
56
|
+
git = subprocess.run(['git', '-C', str(root), 'rev-parse', '--is-inside-work-tree'],
|
|
57
|
+
capture_output=True, text=True)
|
|
58
|
+
excluded, files = [], []
|
|
59
|
+
if git.returncode == 0 and git.stdout.strip() == 'true':
|
|
60
|
+
result = subprocess.run(['git', '-C', str(root), 'ls-files', '-c', '-o',
|
|
61
|
+
'--exclude-standard', '-z'], capture_output=True, check=True)
|
|
62
|
+
names = sorted(set(os.fsdecode(name) for name in result.stdout.split(b'\0') if name))
|
|
63
|
+
discovery = 'git-tracked-and-unignored'
|
|
64
|
+
else:
|
|
65
|
+
names = []
|
|
66
|
+
for directory, dirs, entries in os.walk(root, followlinks=False):
|
|
67
|
+
for name in sorted(dirs):
|
|
68
|
+
path = Path(directory)/name
|
|
69
|
+
if name in SKIP_DIRS or path.is_symlink():
|
|
70
|
+
excluded.append({'path': path.relative_to(root).as_posix()+'/',
|
|
71
|
+
'reason': 'symlink' if path.is_symlink() else 'dependency-or-build-directory'})
|
|
72
|
+
dirs[:] = [name for name in dirs if name not in SKIP_DIRS and not (Path(directory)/name).is_symlink()]
|
|
73
|
+
names.extend((Path(directory)/name).relative_to(root).as_posix() for name in entries)
|
|
74
|
+
names.sort()
|
|
75
|
+
discovery = 'directory-walk'
|
|
76
|
+
for name in names:
|
|
77
|
+
path = root/name
|
|
78
|
+
reason = path_exclusion(name)
|
|
79
|
+
if path.is_symlink() or not path.resolve().is_relative_to(root):
|
|
80
|
+
reason = 'symlink'
|
|
81
|
+
elif not path.is_file():
|
|
82
|
+
reason = 'not-a-regular-file'
|
|
83
|
+
if not reason:
|
|
84
|
+
raw, _, reason = read_text(path)
|
|
85
|
+
if reason:
|
|
86
|
+
excluded.append({'path': name, 'reason': reason})
|
|
87
|
+
else:
|
|
88
|
+
files.append({'path': name, 'bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()})
|
|
89
|
+
return {'schemaVersion': 1, 'scope': 'eligible UTF-8 text; structural adapters where available',
|
|
90
|
+
'discovery': discovery, 'files': files, 'excluded': sorted(excluded, key=lambda item: item['path'])}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Go AST bridge with a content-addressed, local-only helper build."""
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
from functools import lru_cache
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import shutil
|
|
10
|
+
import subprocess
|
|
11
|
+
import tempfile
|
|
12
|
+
|
|
13
|
+
from .schema import physical_lines
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@lru_cache(maxsize=8)
|
|
17
|
+
def toolchain(binary):
|
|
18
|
+
result = subprocess.run([binary, 'version'], capture_output=True, text=True, check=True, timeout=10)
|
|
19
|
+
return result.stdout.strip()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def compiler():
|
|
23
|
+
binary = os.environ.get('OCE_GO_BINARY') or os.environ.get('REPONERVE_GO_BINARY') or shutil.which('go')
|
|
24
|
+
if not binary:
|
|
25
|
+
raise RuntimeError('Go indexing requires Go 1.22+ on PATH or OCE_GO_BINARY')
|
|
26
|
+
return binary, toolchain(binary)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def settings(options=None, sources=()):
|
|
30
|
+
options = {} if options is None else options
|
|
31
|
+
if not isinstance(options, dict) or set(options)-{'mode', 'modulePath', 'goos', 'goarch'}:
|
|
32
|
+
raise ValueError('Go options support mode, modulePath, goos and goarch')
|
|
33
|
+
result = {'mode': 'syntax', 'modulePath': 'snapshot', 'goos': 'linux', 'goarch': 'amd64', **options}
|
|
34
|
+
if result['mode'] not in {'syntax', 'types'} or result['goos'] not in {'linux', 'darwin', 'windows', 'freebsd'} or result['goarch'] not in {'amd64', 'arm64', '386', 'arm'}:
|
|
35
|
+
raise ValueError('Unsupported Go analysis mode or build target')
|
|
36
|
+
if 'modulePath' not in options:
|
|
37
|
+
for source in sources:
|
|
38
|
+
if source.path == 'go.mod':
|
|
39
|
+
match = re.search(r'(?m)^module\s+("[^"\n]+"|[^\s/]+(?:/[^\s]+)*)', source.text)
|
|
40
|
+
if match:
|
|
41
|
+
result['modulePath'] = match[1].strip('"')
|
|
42
|
+
if not isinstance(result['modulePath'], str) or not re.fullmatch(r'[A-Za-z0-9_.~/-]+', result['modulePath']) or '..' in result['modulePath'].split('/') or result['modulePath'].startswith('/'):
|
|
43
|
+
raise ValueError('Invalid Go modulePath')
|
|
44
|
+
return result
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def parse(sources, options):
|
|
48
|
+
binary, version = compiler()
|
|
49
|
+
source = Path(__file__).with_name('go_ast.go')
|
|
50
|
+
semantic = source.with_name('go_types.go')
|
|
51
|
+
identity = hashlib.sha256(source.read_bytes()+semantic.read_bytes()+version.encode()).hexdigest()
|
|
52
|
+
cache = Path(tempfile.gettempdir())/f'reponerve-go-adapter-{getattr(os, "getuid", lambda: 0)()}'
|
|
53
|
+
cache.mkdir(mode=0o700, exist_ok=True)
|
|
54
|
+
if cache.is_symlink() or (os.name == 'posix' and
|
|
55
|
+
(cache.stat().st_uid != os.getuid() or cache.stat().st_mode & 0o077)):
|
|
56
|
+
raise RuntimeError('Go parser cache must be private to the current user')
|
|
57
|
+
executable = cache/identity
|
|
58
|
+
if executable.is_symlink():
|
|
59
|
+
raise RuntimeError('Invalid Go parser cache entry')
|
|
60
|
+
if not executable.exists():
|
|
61
|
+
with tempfile.TemporaryDirectory(dir=cache) as directory:
|
|
62
|
+
target = Path(directory)/'parser'
|
|
63
|
+
env = {**os.environ, 'GOENV': 'off', 'GOWORK': 'off', 'GOTOOLCHAIN': 'local',
|
|
64
|
+
'GOPROXY': 'off', 'GO111MODULE': 'off', 'CGO_ENABLED': '0', 'GOFLAGS': '',
|
|
65
|
+
'GOCACHE': str(cache/'build')}
|
|
66
|
+
built = subprocess.run([binary, 'build', '-o', str(target), str(source), str(semantic)],
|
|
67
|
+
env=env, capture_output=True, text=True, timeout=120)
|
|
68
|
+
if built.returncode:
|
|
69
|
+
raise RuntimeError('Go parser build failed: '+built.stderr[:2000])
|
|
70
|
+
os.replace(target, executable)
|
|
71
|
+
result = subprocess.run([str(executable)], input=json.dumps({'options': options, 'files': [
|
|
72
|
+
{'path': source.path, 'text': source.text} for source in sources]}),
|
|
73
|
+
text=True, capture_output=True, timeout=120)
|
|
74
|
+
if result.returncode:
|
|
75
|
+
raise ValueError('Go adapter failed: '+result.stderr.strip()[:2000])
|
|
76
|
+
return json.loads(result.stdout)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def extract(sources, max_lines=65, options=None):
|
|
80
|
+
options = settings(options)
|
|
81
|
+
parsed = parse(sources, options)
|
|
82
|
+
units, records, targets = [], {}, defaultdict(list)
|
|
83
|
+
source_by_path = {source.path: source for source in sources}
|
|
84
|
+
for file in parsed['files']:
|
|
85
|
+
path = file['path']
|
|
86
|
+
lines = physical_lines(source_by_path[path].text)
|
|
87
|
+
module = str(Path(path).parent)+'::'+file['package']
|
|
88
|
+
root = {'name': '', 'kind': 'module', 'start': 1, 'end': len(lines), 'decl': 0}
|
|
89
|
+
owners = [root]*len(lines)
|
|
90
|
+
for entry in file['entries']:
|
|
91
|
+
for i in range(entry['start']-1, entry['end']):
|
|
92
|
+
owners[i] = entry
|
|
93
|
+
line_units = {}
|
|
94
|
+
start = 1
|
|
95
|
+
while start <= len(lines):
|
|
96
|
+
owner = owners[start-1]
|
|
97
|
+
end = start
|
|
98
|
+
while end < len(lines) and owners[end] is owner:
|
|
99
|
+
end += 1
|
|
100
|
+
while start <= end:
|
|
101
|
+
stop = min(end, start+max_lines-1)
|
|
102
|
+
if stop < end:
|
|
103
|
+
boundaries = [b for b in file['boundaries']
|
|
104
|
+
if start+max(1, max_lines//2) <= b <= stop+1 and b > start]
|
|
105
|
+
if boundaries:
|
|
106
|
+
stop = max(boundaries)-1
|
|
107
|
+
value = '\n'.join(lines[start-1:stop])
|
|
108
|
+
if value.strip():
|
|
109
|
+
uid = len(units)
|
|
110
|
+
symbol = f"{path}::{owner['name']}@{owner['decl']}"
|
|
111
|
+
unit = {'id': uid, 'language': 'go', 'path': path, 'module': module,
|
|
112
|
+
'name': owner['name'], 'symbol': symbol, 'kind': owner['kind'],
|
|
113
|
+
'scope': module, 'owner': None, 'start': start, 'end': stop,
|
|
114
|
+
'text': value, 'relations': [], 'edges': [], 'calls': [], 'unresolved': []}
|
|
115
|
+
units.append(unit)
|
|
116
|
+
targets[(path, owner['decl'])].append(uid)
|
|
117
|
+
for i in range(start, stop+1):
|
|
118
|
+
line_units[i] = uid
|
|
119
|
+
start = stop+1
|
|
120
|
+
records[path] = line_units
|
|
121
|
+
if line_units:
|
|
122
|
+
units[next(iter(line_units.values()))]['goAnalysis'] = {
|
|
123
|
+
**options, 'selectedForTypes': file['semantic'], 'diagnostics': file['diagnostics']}
|
|
124
|
+
def add(uid, others, kind, resolution):
|
|
125
|
+
for target in others:
|
|
126
|
+
if uid != target:
|
|
127
|
+
units[uid]['relations'].append({'target': target, 'kind': kind,
|
|
128
|
+
'confidence': 1 if kind == 'same_symbol' else .9, 'resolution': resolution})
|
|
129
|
+
for (path, decl), ids in targets.items():
|
|
130
|
+
if decl:
|
|
131
|
+
for uid in ids:
|
|
132
|
+
add(uid, ids, 'same_symbol', 'syntax')
|
|
133
|
+
for file in parsed['files']:
|
|
134
|
+
for reference in file['references']:
|
|
135
|
+
uid = records[file['path']].get(reference['line'])
|
|
136
|
+
if uid is not None:
|
|
137
|
+
add(uid, targets.get((reference['targetPath'], reference['targetLine']), []),
|
|
138
|
+
'references_type', reference['resolution'])
|
|
139
|
+
for call in file['calls']:
|
|
140
|
+
uid = records[file['path']].get(call['line'])
|
|
141
|
+
if uid is None:
|
|
142
|
+
continue
|
|
143
|
+
units[uid]['calls'].append(call['text'])
|
|
144
|
+
ids = targets.get((call.get('targetPath'), call.get('targetLine')), [])
|
|
145
|
+
if ids:
|
|
146
|
+
add(uid, ids, 'calls', call['resolution'])
|
|
147
|
+
else:
|
|
148
|
+
units[uid]['unresolved'].append({'kind': 'calls', 'text': call['text'],
|
|
149
|
+
'line': call['line'], 'reason': 'dynamic, external or not statically bound in snapshot'})
|
|
150
|
+
for unit in units:
|
|
151
|
+
unit['relations'] = sorted({(r['target'], r['kind']): r for r in unit['relations']}.values(),
|
|
152
|
+
key=lambda r: (r['target'], r['kind']))
|
|
153
|
+
unit['edges'] = sorted({r['target'] for r in unit['relations']})
|
|
154
|
+
return units
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
// Read only supplied source strings. Never load modules, run source or emit code.
|
|
2
|
+
package main
|
|
3
|
+
|
|
4
|
+
import (
|
|
5
|
+
"encoding/json"
|
|
6
|
+
"fmt"
|
|
7
|
+
"go/ast"
|
|
8
|
+
"go/parser"
|
|
9
|
+
"go/token"
|
|
10
|
+
"os"
|
|
11
|
+
"path"
|
|
12
|
+
"runtime"
|
|
13
|
+
"sort"
|
|
14
|
+
"strings"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
type Source struct {
|
|
18
|
+
Path string `json:"path"`
|
|
19
|
+
Text string `json:"text"`
|
|
20
|
+
}
|
|
21
|
+
type Input struct {
|
|
22
|
+
Files []Source `json:"files"`
|
|
23
|
+
Options Settings `json:"options"`
|
|
24
|
+
}
|
|
25
|
+
type Settings struct {
|
|
26
|
+
Mode string `json:"mode"`
|
|
27
|
+
ModulePath string `json:"modulePath"`
|
|
28
|
+
GOOS string `json:"goos"`
|
|
29
|
+
GOARCH string `json:"goarch"`
|
|
30
|
+
}
|
|
31
|
+
type Entry struct {
|
|
32
|
+
Name string `json:"name"`
|
|
33
|
+
Kind string `json:"kind"`
|
|
34
|
+
Start int `json:"start"`
|
|
35
|
+
End int `json:"end"`
|
|
36
|
+
Decl int `json:"decl"`
|
|
37
|
+
}
|
|
38
|
+
type Call struct {
|
|
39
|
+
Line int `json:"line"`
|
|
40
|
+
Text string `json:"text"`
|
|
41
|
+
Path string `json:"targetPath,omitempty"`
|
|
42
|
+
Decl int `json:"targetLine,omitempty"`
|
|
43
|
+
Resolution string `json:"resolution,omitempty"`
|
|
44
|
+
}
|
|
45
|
+
type File struct {
|
|
46
|
+
Path string `json:"path"`
|
|
47
|
+
Package string `json:"package"`
|
|
48
|
+
Entries []Entry `json:"entries"`
|
|
49
|
+
Boundaries []int `json:"boundaries"`
|
|
50
|
+
Calls []Call `json:"calls"`
|
|
51
|
+
References []Call `json:"references"`
|
|
52
|
+
Diagnostics []string `json:"diagnostics"`
|
|
53
|
+
Semantic bool `json:"semantic"`
|
|
54
|
+
}
|
|
55
|
+
type Target struct {
|
|
56
|
+
path string
|
|
57
|
+
line int
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
func main() {
|
|
61
|
+
if err := run(); err != nil {
|
|
62
|
+
fmt.Fprintln(os.Stderr, err)
|
|
63
|
+
os.Exit(1)
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
func run() error {
|
|
67
|
+
var input Input
|
|
68
|
+
if err := json.NewDecoder(os.Stdin).Decode(&input); err != nil {
|
|
69
|
+
return err
|
|
70
|
+
}
|
|
71
|
+
fset := token.NewFileSet()
|
|
72
|
+
trees := map[string]*ast.File{}
|
|
73
|
+
texts := map[string]string{}
|
|
74
|
+
objects := map[*ast.Object]Target{}
|
|
75
|
+
functions := map[string][]Target{}
|
|
76
|
+
line := func(p token.Pos) int { return fset.PositionFor(p, false).Line }
|
|
77
|
+
for _, src := range input.Files {
|
|
78
|
+
text := strings.ReplaceAll(strings.ReplaceAll(src.Text, "\r\n", "\n"), "\r", "\n")
|
|
79
|
+
tree, err := parser.ParseFile(fset, src.Path, text, parser.ParseComments|parser.AllErrors)
|
|
80
|
+
if err != nil {
|
|
81
|
+
return err
|
|
82
|
+
}
|
|
83
|
+
trees[src.Path] = tree
|
|
84
|
+
texts[src.Path] = text
|
|
85
|
+
for _, decl := range tree.Decls {
|
|
86
|
+
if fn, ok := decl.(*ast.FuncDecl); ok && fn.Recv == nil {
|
|
87
|
+
t := Target{src.Path, line(fn.Pos())}
|
|
88
|
+
if fn.Name.Obj != nil {
|
|
89
|
+
objects[fn.Name.Obj] = t
|
|
90
|
+
}
|
|
91
|
+
key := path.Dir(src.Path) + "::" + tree.Name.Name + "::" + fn.Name.Name
|
|
92
|
+
functions[key] = append(functions[key], t)
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
var typed TypeEvidence
|
|
97
|
+
if input.Options.Mode == "types" {
|
|
98
|
+
typed = resolveTypes(fset, trees, texts, input.Options.ModulePath, input.Options.GOOS, input.Options.GOARCH)
|
|
99
|
+
}
|
|
100
|
+
files := []File{}
|
|
101
|
+
for _, src := range input.Files {
|
|
102
|
+
tree := trees[src.Path]
|
|
103
|
+
out := File{Path: src.Path, Package: tree.Name.Name, Entries: []Entry{}, Boundaries: []int{}, Calls: []Call{}, References: []Call{}, Diagnostics: []string{}}
|
|
104
|
+
if input.Options.Mode == "types" {
|
|
105
|
+
out.Semantic = typed.Selected[src.Path]
|
|
106
|
+
out.Diagnostics = append(out.Diagnostics, typed.Diagnostics[src.Path]...)
|
|
107
|
+
}
|
|
108
|
+
add := func(node ast.Node, name, kind string, doc *ast.CommentGroup) {
|
|
109
|
+
start := line(node.Pos())
|
|
110
|
+
if doc != nil {
|
|
111
|
+
start = line(doc.Pos())
|
|
112
|
+
}
|
|
113
|
+
out.Entries = append(out.Entries, Entry{name, kind, start, line(node.End() - 1), line(node.Pos())})
|
|
114
|
+
}
|
|
115
|
+
for _, decl := range tree.Decls {
|
|
116
|
+
switch d := decl.(type) {
|
|
117
|
+
case *ast.FuncDecl:
|
|
118
|
+
name, kind := d.Name.Name, "function"
|
|
119
|
+
if d.Recv != nil && len(d.Recv.List) > 0 {
|
|
120
|
+
receiver := d.Recv.List[0].Type
|
|
121
|
+
if pointer, ok := receiver.(*ast.StarExpr); ok {
|
|
122
|
+
receiver = pointer.X
|
|
123
|
+
}
|
|
124
|
+
if indexed, ok := receiver.(*ast.IndexExpr); ok {
|
|
125
|
+
receiver = indexed.X
|
|
126
|
+
}
|
|
127
|
+
if indexed, ok := receiver.(*ast.IndexListExpr); ok {
|
|
128
|
+
receiver = indexed.X
|
|
129
|
+
}
|
|
130
|
+
if id, ok := receiver.(*ast.Ident); ok {
|
|
131
|
+
name = id.Name + "." + name
|
|
132
|
+
}
|
|
133
|
+
kind = "method"
|
|
134
|
+
}
|
|
135
|
+
add(d, name, kind, d.Doc)
|
|
136
|
+
case *ast.GenDecl:
|
|
137
|
+
for _, spec := range d.Specs {
|
|
138
|
+
if t, ok := spec.(*ast.TypeSpec); ok {
|
|
139
|
+
add(t, t.Name.Name, "type", t.Doc)
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
ast.Inspect(tree, func(node ast.Node) bool {
|
|
145
|
+
if node == nil {
|
|
146
|
+
return true
|
|
147
|
+
}
|
|
148
|
+
if _, ok := node.(ast.Stmt); ok {
|
|
149
|
+
out.Boundaries = append(out.Boundaries, line(node.Pos()))
|
|
150
|
+
}
|
|
151
|
+
if call, ok := node.(*ast.CallExpr); ok {
|
|
152
|
+
expression := call.Fun
|
|
153
|
+
if generic, ok := expression.(*ast.IndexExpr); ok {
|
|
154
|
+
expression = generic.X
|
|
155
|
+
}
|
|
156
|
+
if generic, ok := expression.(*ast.IndexListExpr); ok {
|
|
157
|
+
expression = generic.X
|
|
158
|
+
}
|
|
159
|
+
start, end := fset.PositionFor(expression.Pos(), false).Offset, fset.PositionFor(expression.End(), false).Offset
|
|
160
|
+
text := texts[src.Path][start:end]
|
|
161
|
+
if len(text) > 200 {
|
|
162
|
+
text = text[:200]
|
|
163
|
+
}
|
|
164
|
+
c := Call{Line: line(call.Pos()), Text: text}
|
|
165
|
+
if id, ok := expression.(*ast.Ident); ok {
|
|
166
|
+
var target Target
|
|
167
|
+
if id.Obj != nil {
|
|
168
|
+
target = objects[id.Obj]
|
|
169
|
+
} else {
|
|
170
|
+
candidates := functions[path.Dir(src.Path)+"::"+tree.Name.Name+"::"+id.Name]
|
|
171
|
+
if len(candidates) == 1 {
|
|
172
|
+
target = candidates[0]
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
if target.path != "" {
|
|
176
|
+
c.Path = target.path
|
|
177
|
+
c.Decl = target.line
|
|
178
|
+
c.Resolution = "syntax-binding"
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
if input.Options.Mode == "types" {
|
|
182
|
+
c.Path = ""
|
|
183
|
+
c.Decl = 0
|
|
184
|
+
c.Resolution = ""
|
|
185
|
+
if target, ok := typed.Calls[call.Pos()]; ok {
|
|
186
|
+
c.Path = target.path
|
|
187
|
+
c.Decl = target.line
|
|
188
|
+
c.Resolution = "compiler-symbol"
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
out.Calls = append(out.Calls, c)
|
|
192
|
+
}
|
|
193
|
+
if input.Options.Mode == "types" {
|
|
194
|
+
if target, ok := typed.References[node.Pos()]; ok {
|
|
195
|
+
out.References = append(out.References, Call{Line: line(node.Pos()), Path: target.path, Decl: target.line, Resolution: "compiler-symbol"})
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return true
|
|
199
|
+
})
|
|
200
|
+
sort.Ints(out.Boundaries)
|
|
201
|
+
files = append(files, out)
|
|
202
|
+
}
|
|
203
|
+
return json.NewEncoder(os.Stdout).Encode(map[string]any{"compilerVersion": runtime.Version(), "files": files})
|
|
204
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// Snapshot-only type checking. Missing dependencies remain unresolved; no loader,
|
|
2
|
+
// shell command, module download, package initializer or analyzed code is run.
|
|
3
|
+
package main
|
|
4
|
+
|
|
5
|
+
import (
|
|
6
|
+
"fmt"
|
|
7
|
+
"go/ast"
|
|
8
|
+
"go/build"
|
|
9
|
+
"go/token"
|
|
10
|
+
"go/types"
|
|
11
|
+
"io"
|
|
12
|
+
"path"
|
|
13
|
+
"sort"
|
|
14
|
+
"strings"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
type TypeEvidence struct {
|
|
18
|
+
Calls map[token.Pos]Target
|
|
19
|
+
References map[token.Pos]Target
|
|
20
|
+
Diagnostics map[string][]string
|
|
21
|
+
Selected map[string]bool
|
|
22
|
+
}
|
|
23
|
+
type snapshotImporter struct {
|
|
24
|
+
fset *token.FileSet
|
|
25
|
+
files map[string][]*ast.File
|
|
26
|
+
packages map[string]*types.Package
|
|
27
|
+
visiting map[string]bool
|
|
28
|
+
info *types.Info
|
|
29
|
+
errors map[string][]string
|
|
30
|
+
sizes types.Sizes
|
|
31
|
+
blocked map[string]bool
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
func (s *snapshotImporter) Import(name string) (*types.Package, error) {
|
|
35
|
+
if p := s.packages[name]; p != nil {
|
|
36
|
+
return p, nil
|
|
37
|
+
}
|
|
38
|
+
files := s.files[name]
|
|
39
|
+
if len(files) == 0 {
|
|
40
|
+
return nil, fmt.Errorf("dependency outside source snapshot: %s", name)
|
|
41
|
+
}
|
|
42
|
+
if s.visiting[name] {
|
|
43
|
+
return nil, fmt.Errorf("snapshot import cycle: %s", name)
|
|
44
|
+
}
|
|
45
|
+
s.visiting[name] = true
|
|
46
|
+
conf := types.Config{Importer: s, DisableUnusedImportCheck: true, Sizes: s.sizes, Error: func(err error) {
|
|
47
|
+
if typed, ok := err.(types.Error); ok {
|
|
48
|
+
pos := s.fset.PositionFor(typed.Pos, false)
|
|
49
|
+
if strings.Contains(typed.Msg, "redeclared") || strings.Contains(typed.Msg, "already declared") {
|
|
50
|
+
s.blocked[name] = true
|
|
51
|
+
}
|
|
52
|
+
if len(s.errors[pos.Filename]) < 20 {
|
|
53
|
+
s.errors[pos.Filename] = append(s.errors[pos.Filename], fmt.Sprintf("%d: %s", pos.Line, typed.Msg))
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}}
|
|
57
|
+
pkg, err := conf.Check(name, s.fset, files, s.info)
|
|
58
|
+
delete(s.visiting, name)
|
|
59
|
+
// go/types returns a partial package on ordinary type errors. Only concrete
|
|
60
|
+
// declarations with positions in this snapshot can become relations later.
|
|
61
|
+
if pkg != nil {
|
|
62
|
+
s.packages[name] = pkg
|
|
63
|
+
return pkg, nil
|
|
64
|
+
}
|
|
65
|
+
return nil, err
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
func resolveTypes(fset *token.FileSet, trees map[string]*ast.File, texts map[string]string,
|
|
69
|
+
modulePath, goos, goarch string) TypeEvidence {
|
|
70
|
+
evidence := TypeEvidence{Calls: map[token.Pos]Target{}, References: map[token.Pos]Target{},
|
|
71
|
+
Diagnostics: map[string][]string{}, Selected: map[string]bool{}}
|
|
72
|
+
context := build.Default
|
|
73
|
+
context.GOOS = goos
|
|
74
|
+
context.GOARCH = goarch
|
|
75
|
+
context.CgoEnabled = false
|
|
76
|
+
context.OpenFile = func(name string) (io.ReadCloser, error) {
|
|
77
|
+
if text, ok := texts[path.Clean(name)]; ok {
|
|
78
|
+
return io.NopCloser(strings.NewReader(text)), nil
|
|
79
|
+
}
|
|
80
|
+
return nil, fmt.Errorf("source outside snapshot: %s", name)
|
|
81
|
+
}
|
|
82
|
+
info := &types.Info{Uses: map[*ast.Ident]types.Object{}, Defs: map[*ast.Ident]types.Object{}, Selections: map[*ast.SelectorExpr]*types.Selection{}}
|
|
83
|
+
loader := &snapshotImporter{fset: fset, files: map[string][]*ast.File{}, packages: map[string]*types.Package{}, visiting: map[string]bool{}, info: info, errors: evidence.Diagnostics, sizes: types.SizesFor("gc", goarch), blocked: map[string]bool{}}
|
|
84
|
+
functions, definitions := map[token.Pos]Target{}, map[token.Pos]Target{}
|
|
85
|
+
names := make([]string, 0, len(trees))
|
|
86
|
+
for name := range trees {
|
|
87
|
+
names = append(names, name)
|
|
88
|
+
}
|
|
89
|
+
sort.Strings(names)
|
|
90
|
+
for _, name := range names {
|
|
91
|
+
tree := trees[name]
|
|
92
|
+
// All files remain in the structural index. Semantic links use one explicit
|
|
93
|
+
// build target and exclude external test packages and test-only declarations.
|
|
94
|
+
match, err := context.MatchFile(path.Dir(name), path.Base(name))
|
|
95
|
+
if err != nil {
|
|
96
|
+
evidence.Diagnostics[name] = append(evidence.Diagnostics[name], err.Error())
|
|
97
|
+
continue
|
|
98
|
+
}
|
|
99
|
+
if !match || strings.HasSuffix(name, "_test.go") {
|
|
100
|
+
continue
|
|
101
|
+
}
|
|
102
|
+
evidence.Selected[name] = true
|
|
103
|
+
packagePath := path.Join(modulePath, path.Dir(name))
|
|
104
|
+
loader.files[packagePath] = append(loader.files[packagePath], tree)
|
|
105
|
+
for _, decl := range tree.Decls {
|
|
106
|
+
switch d := decl.(type) {
|
|
107
|
+
case *ast.FuncDecl:
|
|
108
|
+
if d.Body != nil {
|
|
109
|
+
functions[d.Name.Pos()] = Target{name, fset.PositionFor(d.Pos(), false).Line}
|
|
110
|
+
}
|
|
111
|
+
case *ast.GenDecl:
|
|
112
|
+
for _, spec := range d.Specs {
|
|
113
|
+
if t, ok := spec.(*ast.TypeSpec); ok {
|
|
114
|
+
definitions[t.Name.Pos()] = Target{name, fset.PositionFor(t.Pos(), false).Line}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
packageNames := make([]string, 0, len(loader.files))
|
|
121
|
+
for name := range loader.files {
|
|
122
|
+
packageNames = append(packageNames, name)
|
|
123
|
+
}
|
|
124
|
+
sort.Strings(packageNames)
|
|
125
|
+
for _, name := range packageNames {
|
|
126
|
+
_, _ = loader.Import(name)
|
|
127
|
+
}
|
|
128
|
+
for _, name := range names {
|
|
129
|
+
if !evidence.Selected[name] || loader.blocked[path.Join(modulePath, path.Dir(name))] {
|
|
130
|
+
continue
|
|
131
|
+
}
|
|
132
|
+
ast.Inspect(trees[name], func(node ast.Node) bool {
|
|
133
|
+
if call, ok := node.(*ast.CallExpr); ok {
|
|
134
|
+
expression := call.Fun
|
|
135
|
+
if indexed, ok := expression.(*ast.IndexExpr); ok {
|
|
136
|
+
expression = indexed.X
|
|
137
|
+
}
|
|
138
|
+
if indexed, ok := expression.(*ast.IndexListExpr); ok {
|
|
139
|
+
expression = indexed.X
|
|
140
|
+
}
|
|
141
|
+
var object types.Object
|
|
142
|
+
switch x := expression.(type) {
|
|
143
|
+
case *ast.Ident:
|
|
144
|
+
object = info.Uses[x]
|
|
145
|
+
case *ast.SelectorExpr:
|
|
146
|
+
if selection := info.Selections[x]; selection != nil {
|
|
147
|
+
object = selection.Obj()
|
|
148
|
+
} else {
|
|
149
|
+
object = info.Uses[x.Sel]
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if fn, ok := object.(*types.Func); ok {
|
|
153
|
+
if target, found := functions[fn.Pos()]; found && !loader.blocked[path.Join(modulePath, path.Dir(target.path))] {
|
|
154
|
+
evidence.Calls[call.Pos()] = target
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
if id, ok := node.(*ast.Ident); ok {
|
|
159
|
+
if object, ok := info.Uses[id].(*types.TypeName); ok {
|
|
160
|
+
if target, found := definitions[object.Pos()]; found && !loader.blocked[path.Join(modulePath, path.Dir(target.path))] {
|
|
161
|
+
evidence.References[id.Pos()] = target
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return true
|
|
166
|
+
})
|
|
167
|
+
}
|
|
168
|
+
return evidence
|
|
169
|
+
}
|