open-context-engine 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/README.zh-CN.md +4 -2
- package/bin/opencontextengine.mjs +2 -2
- package/docs/QUICKSTART.md +39 -7
- package/package.json +5 -3
- package/scripts/retrieval-server.py +72 -15
- package/src/client.mjs +5 -1
- package/src/config.mjs +1 -1
- package/src/eval/remote-models.mjs +1 -1
- package/src/mcp.mjs +4 -2
- package/src/retrieval/batched.py +1 -1
- package/src/retrieval/cascade.py +1 -1
- package/src/retrieval/engine.py +2 -2
- package/src/retrieval/entities.py +1 -1
- package/src/retrieval/evidence.py +312 -0
- package/src/retrieval/languages/__init__.py +33 -4
- package/src/retrieval/languages/go.py +10 -6
- package/src/retrieval/languages/go_ast.go +11 -0
- package/src/retrieval/languages/go_types.go +2 -0
- package/src/retrieval/languages/python.py +23 -13
- package/src/retrieval/languages/python_calls.py +188 -0
- package/src/retrieval/languages/schema.py +9 -2
- package/src/retrieval/languages/typescript.mjs +14 -6
- package/src/retrieval/languages/typescript.py +4 -1
- package/src/retrieval/live.py +13 -12
- package/src/retrieval/planning.py +15 -0
- package/src/retrieval/reranker.py +12 -6
- package/src/retrieval/shared_worker.py +115 -0
- package/src/retrieval/writer_lock.py +29 -0
- package/src/runtime.mjs +9 -2
- package/src/service.mjs +26 -8
- package/src/setup.mjs +8 -5
- package/src/shared-service.mjs +142 -0
- package/src/workspaces.mjs +8 -6
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Conservative lexical import/call resolution; never infer dynamic receivers."""
|
|
2
|
+
import ast
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def import_bindings(node, module, package):
|
|
6
|
+
if isinstance(node, ast.Import):
|
|
7
|
+
return {a.asname or a.name.split('.')[0]: a.name if a.asname else a.name.split('.')[0]
|
|
8
|
+
for a in node.names}
|
|
9
|
+
prefix = node.module or ''
|
|
10
|
+
if node.level:
|
|
11
|
+
parts = package.split('.') if package else []
|
|
12
|
+
if node.level > len(parts):
|
|
13
|
+
return {}
|
|
14
|
+
prefix = '.'.join(parts[:len(parts) - node.level + 1] + ([prefix] if prefix else []))
|
|
15
|
+
return {a.asname or a.name: '.'.join(filter(None, [prefix, a.name]))
|
|
16
|
+
for a in node.names if a.name != '*'}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Bindings(ast.NodeVisitor):
|
|
20
|
+
def __init__(self):
|
|
21
|
+
self.names = set()
|
|
22
|
+
|
|
23
|
+
def visit_Name(self, node):
|
|
24
|
+
if isinstance(node.ctx, (ast.Store, ast.Del)):
|
|
25
|
+
self.names.add(node.id)
|
|
26
|
+
|
|
27
|
+
def visit_Import(self, node):
|
|
28
|
+
self.names.update(a.asname or a.name.split('.')[0] for a in node.names)
|
|
29
|
+
|
|
30
|
+
def visit_ImportFrom(self, node):
|
|
31
|
+
self.names.update(a.asname or a.name for a in node.names)
|
|
32
|
+
|
|
33
|
+
def visit_FunctionDef(self, node):
|
|
34
|
+
self.names.add(node.name)
|
|
35
|
+
|
|
36
|
+
visit_AsyncFunctionDef = visit_FunctionDef
|
|
37
|
+
visit_ClassDef = visit_FunctionDef
|
|
38
|
+
def visit_Lambda(self, node):
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
def visit_ExceptHandler(self, node):
|
|
42
|
+
if node.name:
|
|
43
|
+
self.names.add(node.name)
|
|
44
|
+
self.generic_visit(node)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def bound(body):
|
|
48
|
+
visitor = Bindings()
|
|
49
|
+
for node in body:
|
|
50
|
+
visitor.visit(node)
|
|
51
|
+
return visitor.names
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def resolved_links(tree, module, package):
|
|
55
|
+
"""Return calls with source positions and scoped, qualified targets.
|
|
56
|
+
|
|
57
|
+
Imports under uncertain control flow are not assumed to have executed.
|
|
58
|
+
Function-local names shadow outer bindings even before their assignment.
|
|
59
|
+
"""
|
|
60
|
+
calls, references = [], []
|
|
61
|
+
globals_ = {}
|
|
62
|
+
for node in tree.body:
|
|
63
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
64
|
+
globals_.update(import_bindings(node, module, package))
|
|
65
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
66
|
+
globals_[node.name] = module + '.' + node.name
|
|
67
|
+
# A module-level reassignment makes the runtime binding indeterminate.
|
|
68
|
+
for node in tree.body:
|
|
69
|
+
if not isinstance(node, (ast.Import, ast.ImportFrom, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
70
|
+
for name in bound([node]):
|
|
71
|
+
globals_.pop(name, None)
|
|
72
|
+
|
|
73
|
+
def expression(node, env):
|
|
74
|
+
if node is None:
|
|
75
|
+
return
|
|
76
|
+
if isinstance(node, ast.Lambda):
|
|
77
|
+
local = env.copy()
|
|
78
|
+
for arg in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
|
|
79
|
+
local.pop(arg.arg, None)
|
|
80
|
+
for arg in (node.args.vararg, node.args.kwarg):
|
|
81
|
+
if arg:
|
|
82
|
+
local.pop(arg.arg, None)
|
|
83
|
+
expression(node.body, local)
|
|
84
|
+
return
|
|
85
|
+
if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)):
|
|
86
|
+
local = env.copy()
|
|
87
|
+
for gen in node.generators:
|
|
88
|
+
expression(gen.iter, local)
|
|
89
|
+
for name in bound([gen.target]):
|
|
90
|
+
local.pop(name, None)
|
|
91
|
+
for condition in gen.ifs:
|
|
92
|
+
expression(condition, local)
|
|
93
|
+
for field in ('elt', 'key', 'value'):
|
|
94
|
+
expression(getattr(node, field, None), local)
|
|
95
|
+
return
|
|
96
|
+
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) and node.id in env:
|
|
97
|
+
references.append((node.lineno, env[node.id]))
|
|
98
|
+
if isinstance(node, ast.Call):
|
|
99
|
+
pieces = []
|
|
100
|
+
head = node.func
|
|
101
|
+
while isinstance(head, ast.Attribute):
|
|
102
|
+
pieces.insert(0, head.attr)
|
|
103
|
+
head = head.value
|
|
104
|
+
if isinstance(head, ast.Name) and head.id in env:
|
|
105
|
+
calls.append((node.lineno, '.'.join([env[head.id]] + pieces)))
|
|
106
|
+
for child in ast.iter_child_nodes(node):
|
|
107
|
+
expression(child, env)
|
|
108
|
+
|
|
109
|
+
def statements(body, env, owner=None, prefix='', class_closure=None):
|
|
110
|
+
for node in body:
|
|
111
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
112
|
+
env.update(import_bindings(node, module, package))
|
|
113
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
114
|
+
for item in node.decorator_list + node.args.defaults + [d for d in node.args.kw_defaults if d]:
|
|
115
|
+
expression(item, env)
|
|
116
|
+
local = (class_closure if owner and class_closure is not None else env).copy()
|
|
117
|
+
for name in bound(node.body):
|
|
118
|
+
local.pop(name, None)
|
|
119
|
+
for arg in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
|
|
120
|
+
local.pop(arg.arg, None)
|
|
121
|
+
for arg in (node.args.vararg, node.args.kwarg):
|
|
122
|
+
if arg:
|
|
123
|
+
local.pop(arg.arg, None)
|
|
124
|
+
args = node.args.posonlyargs + node.args.args
|
|
125
|
+
if owner and args and args[0].arg in {'self', 'cls'} and args[0].arg not in bound(node.body):
|
|
126
|
+
local[args[0].arg] = owner
|
|
127
|
+
statements(node.body, local, prefix=prefix + node.name + '.')
|
|
128
|
+
env[node.name] = module + '.' + prefix + node.name
|
|
129
|
+
elif isinstance(node, ast.ClassDef):
|
|
130
|
+
for item in node.decorator_list + node.bases:
|
|
131
|
+
expression(item, env)
|
|
132
|
+
name = module + '.' + prefix + node.name
|
|
133
|
+
# Methods close over the containing function/module, not class locals.
|
|
134
|
+
statements(node.body, env.copy(), owner=name, prefix=prefix + node.name + '.', class_closure=env.copy())
|
|
135
|
+
env[node.name] = name
|
|
136
|
+
elif isinstance(node, (ast.If, ast.For, ast.AsyncFor, ast.While, ast.With, ast.AsyncWith, ast.Try, ast.TryStar, ast.Match)):
|
|
137
|
+
# Visit each branch in an isolated scope; invalidate any names
|
|
138
|
+
# it could write, so uncertain imports cannot leak past it.
|
|
139
|
+
changed = bound([node])
|
|
140
|
+
branch_env = {k: v for k, v in env.items() if k not in changed}
|
|
141
|
+
for field, value in ast.iter_fields(node):
|
|
142
|
+
if field in {'body', 'orelse', 'finalbody'}:
|
|
143
|
+
statements(value, branch_env.copy(), owner, prefix, class_closure)
|
|
144
|
+
elif field == 'handlers':
|
|
145
|
+
for handler in value:
|
|
146
|
+
statements(handler.body, branch_env.copy(), owner, prefix, class_closure)
|
|
147
|
+
elif field == 'cases':
|
|
148
|
+
for case in value:
|
|
149
|
+
expression(case.guard, branch_env)
|
|
150
|
+
statements(case.body, branch_env.copy(), owner, prefix, class_closure)
|
|
151
|
+
elif isinstance(value, ast.AST):
|
|
152
|
+
expression(value, env)
|
|
153
|
+
elif isinstance(value, list):
|
|
154
|
+
for item in value:
|
|
155
|
+
if isinstance(item, ast.AST):
|
|
156
|
+
expression(item, env)
|
|
157
|
+
for name in changed:
|
|
158
|
+
env.pop(name, None)
|
|
159
|
+
else:
|
|
160
|
+
expression(node, env)
|
|
161
|
+
for name in bound([node]):
|
|
162
|
+
env.pop(name, None)
|
|
163
|
+
statements(tree.body, globals_.copy())
|
|
164
|
+
return {'calls': calls, 'references_value': references}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def symbol_resolver(modules, symbols):
|
|
168
|
+
"""Allow source-root prefixes only when a module suffix is unambiguous."""
|
|
169
|
+
suffixes = {}
|
|
170
|
+
for module in modules:
|
|
171
|
+
parts = module.split('.')
|
|
172
|
+
for i in range(len(parts)):
|
|
173
|
+
suffixes.setdefault('.'.join(parts[i:]), set()).add(module)
|
|
174
|
+
|
|
175
|
+
def resolve(target):
|
|
176
|
+
if target in symbols:
|
|
177
|
+
return symbols[target]
|
|
178
|
+
parts = target.split('.')
|
|
179
|
+
for split in range(len(parts) - 1, 0, -1):
|
|
180
|
+
name = '.'.join(parts[:split])
|
|
181
|
+
matches = {name} if name in modules else suffixes.get(name, set())
|
|
182
|
+
if len(matches) == 1:
|
|
183
|
+
qualified = next(iter(matches)) + '.' + '.'.join(parts[split:])
|
|
184
|
+
return symbols.get(qualified, [])
|
|
185
|
+
if matches:
|
|
186
|
+
return []
|
|
187
|
+
return []
|
|
188
|
+
return resolve
|
|
@@ -3,9 +3,16 @@ from dataclasses import dataclass
|
|
|
3
3
|
import math
|
|
4
4
|
from typing import TypedDict
|
|
5
5
|
|
|
6
|
-
SCHEMA_VERSION = 'source-units-
|
|
6
|
+
SCHEMA_VERSION = 'source-units-v3'
|
|
7
7
|
RELATION_KINDS = frozenset({'calls', 'member_of', 'inherits', 'implements',
|
|
8
|
-
'same_symbol', 'imports', 'references_type'})
|
|
8
|
+
'same_symbol', 'imports', 'references_type', 'references_value'})
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SourceSyntaxError(ValueError):
|
|
12
|
+
"""File-local syntax diagnostics, distinct from adapter or validation failures."""
|
|
13
|
+
def __init__(self, diagnostics):
|
|
14
|
+
super().__init__('Source syntax errors')
|
|
15
|
+
self.diagnostics = diagnostics
|
|
9
16
|
|
|
10
17
|
|
|
11
18
|
@dataclass(frozen=True)
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/** Parse and bind only the frozen source set. Never emit, execute, or load repo plugins. */
|
|
2
2
|
import ts from 'typescript';
|
|
3
|
-
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { readFileSync, existsSync, realpathSync } from 'node:fs';
|
|
4
4
|
import path from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
5
6
|
|
|
6
7
|
const COMPILER_VERSION = '5.9.3';
|
|
7
8
|
if (ts.version !== COMPILER_VERSION) throw new Error(`Expected TypeScript ${COMPILER_VERSION}; run npm ci`);
|
|
@@ -44,10 +45,16 @@ export function extract(files, maxLines = 65, settings = {}) {
|
|
|
44
45
|
const program = ts.createProgram([...texts.keys()], options, host);
|
|
45
46
|
const errors = program.getSyntacticDiagnostics();
|
|
46
47
|
if (errors.length) {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
if (errors.some(d => !d.file)) throw new Error('TypeScript syntax diagnostic is missing a source file');
|
|
49
|
+
const diagnostics = new Map();
|
|
50
|
+
for (const d of errors) {
|
|
51
|
+
const name = path.posix.relative(ROOT, d.file.fileName);
|
|
52
|
+
const position = d.file.getLineAndCharacterOfPosition(d.start ?? 0);
|
|
53
|
+
if (!diagnostics.has(name)) diagnostics.set(name, {path:name,
|
|
54
|
+
language:/\.(?:jsx?|mjs|cjs)$/i.test(name) ? 'javascript' : 'typescript',
|
|
55
|
+
errorType:'SyntaxError', line:position.line + 1, column:position.character + 1});
|
|
56
|
+
}
|
|
57
|
+
return {compilerVersion:ts.version, units:[], syntaxErrors:[...diagnostics.values()]};
|
|
51
58
|
}
|
|
52
59
|
const checker = program.getTypeChecker();
|
|
53
60
|
const units = [], records = new Map(), nodeEntries = new Map();
|
|
@@ -223,7 +230,8 @@ export function extract(files, maxLines = 65, settings = {}) {
|
|
|
223
230
|
return { compilerVersion: ts.version, units };
|
|
224
231
|
}
|
|
225
232
|
|
|
226
|
-
if (process.argv[1] &&
|
|
233
|
+
if (process.argv[1] && existsSync(process.argv[1])
|
|
234
|
+
&& realpathSync.native(fileURLToPath(import.meta.url)) === realpathSync.native(process.argv[1])) {
|
|
227
235
|
try {
|
|
228
236
|
const input = JSON.parse(readFileSync(0, 'utf8'));
|
|
229
237
|
process.stdout.write(JSON.stringify(extract(input.files, input.maxLines, input.options)));
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import json
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
import subprocess
|
|
5
|
+
from .schema import SourceSyntaxError
|
|
5
6
|
|
|
6
7
|
COMPILER_VERSION = '5.9.3'
|
|
7
8
|
|
|
@@ -11,7 +12,7 @@ def extract(sources, max_lines=65, options=None):
|
|
|
11
12
|
'maxLines': max_lines, 'options': options or {}}
|
|
12
13
|
try:
|
|
13
14
|
result = subprocess.run(['node', str(Path(__file__).with_suffix('.mjs'))],
|
|
14
|
-
input=json.dumps(payload),
|
|
15
|
+
input=json.dumps(payload), encoding='utf-8', capture_output=True,
|
|
15
16
|
timeout=120, check=False)
|
|
16
17
|
except FileNotFoundError as error:
|
|
17
18
|
raise RuntimeError('TypeScript indexing requires Node.js and npm ci') from error
|
|
@@ -20,4 +21,6 @@ def extract(sources, max_lines=65, options=None):
|
|
|
20
21
|
output = json.loads(result.stdout)
|
|
21
22
|
if output['compilerVersion'] != COMPILER_VERSION:
|
|
22
23
|
raise ValueError('Unexpected TypeScript compiler version; run npm ci')
|
|
24
|
+
if output.get('syntaxErrors'):
|
|
25
|
+
raise SourceSyntaxError(output['syntaxErrors'])
|
|
23
26
|
return output['units']
|
package/src/retrieval/live.py
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
from collections import Counter
|
|
3
3
|
from contextlib import closing
|
|
4
4
|
from dataclasses import dataclass
|
|
5
|
-
import fcntl
|
|
6
5
|
import hashlib
|
|
7
6
|
import json
|
|
8
7
|
import os
|
|
@@ -18,7 +17,8 @@ import numpy as np
|
|
|
18
17
|
from engine import document, post
|
|
19
18
|
from languages import adapter_manifest, source_units
|
|
20
19
|
from languages.files import discover_snapshot
|
|
21
|
-
from
|
|
20
|
+
from evidence import EvidenceEngine
|
|
21
|
+
from writer_lock import acquire_writer_lock
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def digest(value):
|
|
@@ -42,7 +42,7 @@ class Generation:
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
class LiveIndex:
|
|
45
|
-
def __init__(self, config, *, embed=post, engine_factory=
|
|
45
|
+
def __init__(self, config, *, embed=post, engine_factory=EvidenceEngine):
|
|
46
46
|
self.config = config
|
|
47
47
|
self.root = Path(config['root']).resolve()
|
|
48
48
|
self.state = Path(config['state']).resolve()
|
|
@@ -64,6 +64,9 @@ class LiveIndex:
|
|
|
64
64
|
self.dimensions = self.embedding['dimensions']
|
|
65
65
|
if type(self.dimensions) is not int or not 1 <= self.dimensions <= 65536:
|
|
66
66
|
raise ValueError('Invalid embedding dimensions')
|
|
67
|
+
self.batch_size = config.get('embeddingBatchSize', 64)
|
|
68
|
+
if type(self.batch_size) is not int or not 1 <= self.batch_size <= 64:
|
|
69
|
+
raise ValueError('Embedding batch size must be an integer from 1 to 64')
|
|
67
70
|
self.embed, self.engine_factory = embed, engine_factory
|
|
68
71
|
self.condition = threading.Condition()
|
|
69
72
|
self.stop_event = threading.Event()
|
|
@@ -72,12 +75,7 @@ class LiveIndex:
|
|
|
72
75
|
self.phase = 'starting'
|
|
73
76
|
self.parse_cache = {}
|
|
74
77
|
self.state.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
75
|
-
self.lock_file = (self.state/'writer.lock')
|
|
76
|
-
try:
|
|
77
|
-
fcntl.flock(self.lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
78
|
-
except OSError:
|
|
79
|
-
self.lock_file.close()
|
|
80
|
-
raise ValueError('This index directory already has a running writer') from None
|
|
78
|
+
self.lock_file = acquire_writer_lock(self.state/'writer.lock')
|
|
81
79
|
self.thread = threading.Thread(target=self._run, name='repository-index', daemon=True)
|
|
82
80
|
|
|
83
81
|
def scan(self):
|
|
@@ -169,7 +167,9 @@ class LiveIndex:
|
|
|
169
167
|
|
|
170
168
|
def _build(self, snapshot, identity):
|
|
171
169
|
start = time.monotonic()
|
|
172
|
-
|
|
170
|
+
report = {}
|
|
171
|
+
units = source_units(self.root, snapshot['files'], language_options=self.options,
|
|
172
|
+
cache=self.parse_cache, report=report)
|
|
173
173
|
documents = [document(unit) for unit in units]
|
|
174
174
|
keys = [digest([self.embedding, text]) for text in documents]
|
|
175
175
|
vectors, missing = {}, {}
|
|
@@ -184,10 +184,10 @@ class LiveIndex:
|
|
|
184
184
|
continue
|
|
185
185
|
missing[key] = text
|
|
186
186
|
entries = list(missing.items())
|
|
187
|
-
for offset in range(0, len(entries),
|
|
187
|
+
for offset in range(0, len(entries), self.batch_size):
|
|
188
188
|
if self.stop_event.is_set():
|
|
189
189
|
raise SourceChanged()
|
|
190
|
-
batch = entries[offset:offset+
|
|
190
|
+
batch = entries[offset:offset+self.batch_size]
|
|
191
191
|
result = self.embed(self.config['embeddingUrl']+'/embeddings',
|
|
192
192
|
{'model': self.embedding['model'], 'input': [text for _, text in batch]},
|
|
193
193
|
self.config.get('embeddingKey', 'local-only'), timeout=60)
|
|
@@ -212,6 +212,7 @@ class LiveIndex:
|
|
|
212
212
|
'changedFiles': sum(previous.get(path) != sha for path, sha in now.items()),
|
|
213
213
|
'deletedFiles': len(previous.keys() - now.keys()), 'embedding': self.embedding,
|
|
214
214
|
'languageUnits': dict(Counter(unit['language'] for unit in units)),
|
|
215
|
+
'degradedFiles': report['degradedFiles'], 'parseDiagnostics': report['parseDiagnostics'],
|
|
215
216
|
'indexingMs': round((time.monotonic()-start)*1000), 'completedAt': time.time()}
|
|
216
217
|
folder = self.state/('generation-'+uuid.uuid4().hex)
|
|
217
218
|
folder.mkdir(mode=0o700)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Deterministic behavior facets; every facet retains its full query context in retrieval."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def plan_query(query):
|
|
6
|
+
# Keep conditions in the intent, but do not spend a facet on a condition
|
|
7
|
+
# such as "when a user reopens a chat" without its requested behavior.
|
|
8
|
+
parts = [part.strip() for part in re.split(
|
|
9
|
+
r'[::;;,]|,\s+(?=how|which|why|where|what)|\s+and\s+(?=how|which|why|where|what|preserve|restore|validate|enforce|handle|load|save|reject|filter|check|merge|retain|stop)',
|
|
10
|
+
query, flags=re.I) if len(part.strip()) > 7]
|
|
11
|
+
actionable = [part for part in parts if not re.match(r'^(when|if|after|before|unless|given)\b', part, re.I)
|
|
12
|
+
and not (part.endswith('时') and not re.search(r'如何|怎么|哪里|怎样', part))]
|
|
13
|
+
facets = actionable if 1 < len(actionable) <= 4 else [query]
|
|
14
|
+
return {'intent': query, 'facets': [{'question': part, 'terms': re.findall(r'[A-Za-z][A-Za-z0-9_]*', part)}
|
|
15
|
+
for part in facets]}
|
|
@@ -33,8 +33,8 @@ def checked_rows(data, count):
|
|
|
33
33
|
|
|
34
34
|
def rerank_pairs(config, queries, documents, pairs, post):
|
|
35
35
|
api = config.get('api', 'rerank')
|
|
36
|
-
if api not in ('rerank', 'rerank-batch'):
|
|
37
|
-
raise ValueError('Rerank API must be rerank
|
|
36
|
+
if api not in ('rerank', 'rerank-batch', 'dashscope'):
|
|
37
|
+
raise ValueError('Rerank API must be rerank, rerank-batch or dashscope')
|
|
38
38
|
concurrency = bounded_integer(config.get('concurrency', 2), 'Rerank concurrency', 8)
|
|
39
39
|
limit = bounded_integer(config.get('maxDocuments', 128), 'Rerank document limit', 1024)
|
|
40
40
|
for q, d in pairs:
|
|
@@ -42,7 +42,8 @@ def rerank_pairs(config, queries, documents, pairs, post):
|
|
|
42
42
|
raise ValueError('Invalid requested rerank pair')
|
|
43
43
|
if not pairs:
|
|
44
44
|
return {'results': [], 'meta': {'request_count': 0}}
|
|
45
|
-
|
|
45
|
+
path = 'services/rerank/text-rerank/text-rerank' if api == 'dashscope' else api
|
|
46
|
+
url = config['baseUrl'].rstrip('/') + '/' + path
|
|
46
47
|
if api == 'rerank-batch':
|
|
47
48
|
data = post(url, {'model': config['model'], 'queries': queries,
|
|
48
49
|
'documents': documents, 'pairs': pairs}, config['apiKey'])
|
|
@@ -61,9 +62,14 @@ def rerank_pairs(config, queries, documents, pairs, post):
|
|
|
61
62
|
|
|
62
63
|
def request(job):
|
|
63
64
|
q, ids = job
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
65
|
+
inputs = {'query': queries[q], 'documents': [documents[d] for d in ids]}
|
|
66
|
+
body = ({'model': config['model'], 'input': inputs, 'parameters': {'top_n': len(ids)}}
|
|
67
|
+
if api == 'dashscope' else {'model': config['model'], **inputs, 'top_n': len(ids)})
|
|
68
|
+
data = post(url, body, config['apiKey'])
|
|
69
|
+
rows = checked_rows(data.get('output') if api == 'dashscope' and isinstance(data, dict) else data, len(ids))
|
|
70
|
+
if api == 'dashscope':
|
|
71
|
+
usage = data.get('usage', {})
|
|
72
|
+
data = {**data, 'usage': {'input_tokens': usage.get('total_tokens')} if isinstance(usage, dict) else {}}
|
|
67
73
|
return {(q, ids[row['index']]): row['relevance_score'] for row in rows}, data
|
|
68
74
|
|
|
69
75
|
# Fail the whole wave on any error; never silently drop a query or switch API.
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Discovery and expiring client leases for a single locked index writer."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import secrets
|
|
6
|
+
import re
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
PROTOCOL = 1
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def restrict_permissions(path):
|
|
16
|
+
if sys.platform == 'win32':
|
|
17
|
+
# os.open(mode=0o600) does not set a Windows DACL. Resolve the current
|
|
18
|
+
# user's SID (independent of localized account names) and remove inheritance.
|
|
19
|
+
result = subprocess.run(['whoami', '/user', '/fo', 'csv', '/nh'],
|
|
20
|
+
capture_output=True, check=True, timeout=5)
|
|
21
|
+
match = re.search(rb'S-1-[0-9-]+', result.stdout)
|
|
22
|
+
if not match:
|
|
23
|
+
raise PermissionError('Cannot determine current Windows user SID')
|
|
24
|
+
sid = match.group().decode('ascii')
|
|
25
|
+
subprocess.run(['icacls', str(path), '/inheritance:r', '/grant:r', '*' + sid + ':F'],
|
|
26
|
+
capture_output=True, check=True, timeout=5)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SharedWorker:
|
|
30
|
+
def __init__(self, config):
|
|
31
|
+
options = config['shared']
|
|
32
|
+
self.path = Path(config['state']) / 'worker.json'
|
|
33
|
+
self.instance = secrets.token_hex(24)
|
|
34
|
+
self.fingerprint = options['fingerprint']
|
|
35
|
+
self.key = config['serviceKey']
|
|
36
|
+
self.ttl = options.get('leaseSeconds', 15)
|
|
37
|
+
self.idle = options.get('idleSeconds', 30)
|
|
38
|
+
self.leases = {}
|
|
39
|
+
self.active = 0
|
|
40
|
+
self.empty_since = time.monotonic()
|
|
41
|
+
self.stopping = False
|
|
42
|
+
self.lock = threading.Lock()
|
|
43
|
+
self.finished = threading.Event()
|
|
44
|
+
|
|
45
|
+
def publish(self, port):
|
|
46
|
+
# Called only by the writer-lock holder, after HTTP bind succeeds.
|
|
47
|
+
self.record = {'protocol': PROTOCOL, 'instanceId': self.instance,
|
|
48
|
+
'pid': os.getpid(), 'port': port, 'apiKey': self.key}
|
|
49
|
+
temporary = self.path.with_name('worker-' + self.instance + '.tmp')
|
|
50
|
+
try:
|
|
51
|
+
with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as file:
|
|
52
|
+
restrict_permissions(temporary)
|
|
53
|
+
json.dump(self.record, file)
|
|
54
|
+
os.replace(temporary, self.path)
|
|
55
|
+
finally:
|
|
56
|
+
temporary.unlink(missing_ok=True)
|
|
57
|
+
|
|
58
|
+
def lease(self, body, release=False):
|
|
59
|
+
with self.lock:
|
|
60
|
+
if body.get('instanceId') != self.instance:
|
|
61
|
+
return 409, {'error': 'Worker instance changed', 'code': 'INSTANCE_CHANGED'}
|
|
62
|
+
if not secrets.compare_digest(str(body.get('fingerprint', '')), self.fingerprint):
|
|
63
|
+
return 409, {'error': 'This index is used by a worker with incompatible configuration or runtime. '
|
|
64
|
+
'Close its clients before changing configuration, or use a separate --state directory.',
|
|
65
|
+
'code': 'CONFIG_MISMATCH'}
|
|
66
|
+
identifier = body.get('leaseId')
|
|
67
|
+
if not isinstance(identifier, str) or not 1 <= len(identifier) <= 128:
|
|
68
|
+
return 422, {'error': 'Invalid lease identifier'}
|
|
69
|
+
if self.stopping:
|
|
70
|
+
return 503, {'error': 'Worker is stopping'}
|
|
71
|
+
if release:
|
|
72
|
+
self.leases.pop(identifier, None)
|
|
73
|
+
else:
|
|
74
|
+
self.leases[identifier] = time.monotonic() + self.ttl
|
|
75
|
+
self.empty_since = time.monotonic()
|
|
76
|
+
return 200, {'instanceId': self.instance, 'pid': os.getpid(), 'leaseSeconds': self.ttl}
|
|
77
|
+
|
|
78
|
+
def enter(self):
|
|
79
|
+
with self.lock:
|
|
80
|
+
if self.stopping:
|
|
81
|
+
return False
|
|
82
|
+
self.active += 1
|
|
83
|
+
return True
|
|
84
|
+
|
|
85
|
+
def leave(self):
|
|
86
|
+
with self.lock:
|
|
87
|
+
self.active -= 1
|
|
88
|
+
self.empty_since = time.monotonic()
|
|
89
|
+
|
|
90
|
+
def should_stop(self):
|
|
91
|
+
with self.lock:
|
|
92
|
+
now = time.monotonic()
|
|
93
|
+
self.leases = {key: until for key, until in self.leases.items() if until > now}
|
|
94
|
+
if self.leases or self.active:
|
|
95
|
+
self.empty_since = now
|
|
96
|
+
elif now - self.empty_since >= self.idle:
|
|
97
|
+
self.stopping = True
|
|
98
|
+
return self.stopping
|
|
99
|
+
|
|
100
|
+
def monitor(self, server):
|
|
101
|
+
def run():
|
|
102
|
+
while not self.finished.wait(min(1, self.idle / 2)):
|
|
103
|
+
if self.should_stop():
|
|
104
|
+
server.shutdown()
|
|
105
|
+
return
|
|
106
|
+
threading.Thread(target=run, name='worker-leases', daemon=True).start()
|
|
107
|
+
|
|
108
|
+
def close(self):
|
|
109
|
+
self.finished.set()
|
|
110
|
+
# Cleanup precedes releasing the writer lock; never unlink a successor's record.
|
|
111
|
+
try:
|
|
112
|
+
if json.loads(self.path.read_text()).get('instanceId') == self.instance:
|
|
113
|
+
self.path.unlink()
|
|
114
|
+
except (FileNotFoundError, ValueError):
|
|
115
|
+
pass
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Nonblocking process lock, released when its file handle is closed."""
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
if sys.platform == 'win32':
|
|
5
|
+
import msvcrt
|
|
6
|
+
|
|
7
|
+
def lock(file):
|
|
8
|
+
# Every writer locks the same byte, including when the file is empty.
|
|
9
|
+
file.seek(0)
|
|
10
|
+
msvcrt.locking(file.fileno(), msvcrt.LK_NBLCK, 1)
|
|
11
|
+
else:
|
|
12
|
+
import fcntl
|
|
13
|
+
|
|
14
|
+
def lock(file):
|
|
15
|
+
fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class WriterBusy(ValueError):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def acquire_writer_lock(path):
|
|
23
|
+
file = path.open('a+b')
|
|
24
|
+
try:
|
|
25
|
+
lock(file)
|
|
26
|
+
except OSError:
|
|
27
|
+
file.close()
|
|
28
|
+
raise WriterBusy(f'This index directory already has a running writer: {path}') from None
|
|
29
|
+
return file
|
package/src/runtime.mjs
CHANGED
|
@@ -6,6 +6,12 @@ import { createHash } from 'node:crypto';
|
|
|
6
6
|
import { configDirectory, projectRoot } from './config.mjs';
|
|
7
7
|
|
|
8
8
|
const exec = promisify(execFile);
|
|
9
|
+
export function defaultPython(platform = process.platform) {
|
|
10
|
+
return platform === 'win32' ? 'python' : 'python3';
|
|
11
|
+
}
|
|
12
|
+
export function venvPython(directory, platform = process.platform) {
|
|
13
|
+
return platform === 'win32' ? join(directory,'Scripts','python.exe') : join(directory,'bin','python');
|
|
14
|
+
}
|
|
9
15
|
export async function run(command, args, options = {}) {
|
|
10
16
|
try {return await exec(command,args,{timeout:600000,maxBuffer:4*1024*1024,...options});}
|
|
11
17
|
catch (error) {
|
|
@@ -21,7 +27,8 @@ export async function verifyPython(python, environment = process.env, execute =
|
|
|
21
27
|
return (await execute(python,['-c',probe],{env:environment,timeout:120000})).stdout.trim();
|
|
22
28
|
}
|
|
23
29
|
|
|
24
|
-
export async function ensureRuntime(environment, {
|
|
30
|
+
export async function ensureRuntime(environment, {platform = process.platform,
|
|
31
|
+
python = defaultPython(platform), execute = run, log = () => {}} = {}) {
|
|
25
32
|
const runtimes = join(configDirectory(environment),'runtimes');
|
|
26
33
|
const requirements = await readFile(join(projectRoot,'requirements.txt'));
|
|
27
34
|
const fingerprint = createHash('sha256').update(requirements).digest('hex');
|
|
@@ -43,7 +50,7 @@ export async function ensureRuntime(environment, {python = 'python3', execute =
|
|
|
43
50
|
await mkdir(runtimes,{recursive:true,mode:0o700});
|
|
44
51
|
// Virtual environments cannot be relocated. Each installation is built in its final directory.
|
|
45
52
|
const directory = await mkdtemp(join(runtimes,'python-'));
|
|
46
|
-
const executable =
|
|
53
|
+
const executable = venvPython(directory,platform);
|
|
47
54
|
const env = {...environment,TIKTOKEN_CACHE_DIR:join(directory,'tokenizer')};
|
|
48
55
|
try {
|
|
49
56
|
log('Creating an isolated Python environment...');
|