open-context-engine 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/docs/QUICKSTART.md +28 -4
- package/package.json +3 -2
- package/scripts/retrieval-server.py +65 -16
- package/src/client.mjs +6 -2
- package/src/config.mjs +1 -1
- package/src/eval/remote-models.mjs +1 -1
- package/src/mcp.mjs +5 -3
- package/src/retrieval/batched.py +1 -1
- package/src/retrieval/cascade.py +1 -1
- package/src/retrieval/engine.py +2 -2
- package/src/retrieval/entities.py +1 -1
- package/src/retrieval/evidence.py +312 -0
- package/src/retrieval/languages/__init__.py +33 -4
- package/src/retrieval/languages/go.py +3 -1
- package/src/retrieval/languages/go_ast.go +11 -0
- package/src/retrieval/languages/python.py +23 -13
- package/src/retrieval/languages/python_calls.py +188 -0
- package/src/retrieval/languages/schema.py +9 -2
- package/src/retrieval/languages/typescript.mjs +10 -4
- package/src/retrieval/languages/typescript.py +3 -0
- package/src/retrieval/live.py +11 -5
- package/src/retrieval/planning.py +15 -0
- package/src/retrieval/reranker.py +12 -6
- package/src/retrieval/shared_worker.py +115 -0
- package/src/retrieval/writer_lock.py +5 -1
- package/src/service.mjs +19 -3
- package/src/setup.mjs +2 -0
- package/src/shared-service.mjs +142 -0
- package/src/workspaces.mjs +7 -5
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Conservative lexical import/call resolution; never infer dynamic receivers."""
|
|
2
|
+
import ast
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def import_bindings(node, module, package):
|
|
6
|
+
if isinstance(node, ast.Import):
|
|
7
|
+
return {a.asname or a.name.split('.')[0]: a.name if a.asname else a.name.split('.')[0]
|
|
8
|
+
for a in node.names}
|
|
9
|
+
prefix = node.module or ''
|
|
10
|
+
if node.level:
|
|
11
|
+
parts = package.split('.') if package else []
|
|
12
|
+
if node.level > len(parts):
|
|
13
|
+
return {}
|
|
14
|
+
prefix = '.'.join(parts[:len(parts) - node.level + 1] + ([prefix] if prefix else []))
|
|
15
|
+
return {a.asname or a.name: '.'.join(filter(None, [prefix, a.name]))
|
|
16
|
+
for a in node.names if a.name != '*'}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Bindings(ast.NodeVisitor):
|
|
20
|
+
def __init__(self):
|
|
21
|
+
self.names = set()
|
|
22
|
+
|
|
23
|
+
def visit_Name(self, node):
|
|
24
|
+
if isinstance(node.ctx, (ast.Store, ast.Del)):
|
|
25
|
+
self.names.add(node.id)
|
|
26
|
+
|
|
27
|
+
def visit_Import(self, node):
|
|
28
|
+
self.names.update(a.asname or a.name.split('.')[0] for a in node.names)
|
|
29
|
+
|
|
30
|
+
def visit_ImportFrom(self, node):
|
|
31
|
+
self.names.update(a.asname or a.name for a in node.names)
|
|
32
|
+
|
|
33
|
+
def visit_FunctionDef(self, node):
|
|
34
|
+
self.names.add(node.name)
|
|
35
|
+
|
|
36
|
+
visit_AsyncFunctionDef = visit_FunctionDef
|
|
37
|
+
visit_ClassDef = visit_FunctionDef
|
|
38
|
+
def visit_Lambda(self, node):
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
def visit_ExceptHandler(self, node):
|
|
42
|
+
if node.name:
|
|
43
|
+
self.names.add(node.name)
|
|
44
|
+
self.generic_visit(node)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def bound(body):
|
|
48
|
+
visitor = Bindings()
|
|
49
|
+
for node in body:
|
|
50
|
+
visitor.visit(node)
|
|
51
|
+
return visitor.names
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def resolved_links(tree, module, package):
|
|
55
|
+
"""Return calls with source positions and scoped, qualified targets.
|
|
56
|
+
|
|
57
|
+
Imports under uncertain control flow are not assumed to have executed.
|
|
58
|
+
Function-local names shadow outer bindings even before their assignment.
|
|
59
|
+
"""
|
|
60
|
+
calls, references = [], []
|
|
61
|
+
globals_ = {}
|
|
62
|
+
for node in tree.body:
|
|
63
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
64
|
+
globals_.update(import_bindings(node, module, package))
|
|
65
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
66
|
+
globals_[node.name] = module + '.' + node.name
|
|
67
|
+
# A module-level reassignment makes the runtime binding indeterminate.
|
|
68
|
+
for node in tree.body:
|
|
69
|
+
if not isinstance(node, (ast.Import, ast.ImportFrom, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
70
|
+
for name in bound([node]):
|
|
71
|
+
globals_.pop(name, None)
|
|
72
|
+
|
|
73
|
+
def expression(node, env):
|
|
74
|
+
if node is None:
|
|
75
|
+
return
|
|
76
|
+
if isinstance(node, ast.Lambda):
|
|
77
|
+
local = env.copy()
|
|
78
|
+
for arg in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
|
|
79
|
+
local.pop(arg.arg, None)
|
|
80
|
+
for arg in (node.args.vararg, node.args.kwarg):
|
|
81
|
+
if arg:
|
|
82
|
+
local.pop(arg.arg, None)
|
|
83
|
+
expression(node.body, local)
|
|
84
|
+
return
|
|
85
|
+
if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)):
|
|
86
|
+
local = env.copy()
|
|
87
|
+
for gen in node.generators:
|
|
88
|
+
expression(gen.iter, local)
|
|
89
|
+
for name in bound([gen.target]):
|
|
90
|
+
local.pop(name, None)
|
|
91
|
+
for condition in gen.ifs:
|
|
92
|
+
expression(condition, local)
|
|
93
|
+
for field in ('elt', 'key', 'value'):
|
|
94
|
+
expression(getattr(node, field, None), local)
|
|
95
|
+
return
|
|
96
|
+
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) and node.id in env:
|
|
97
|
+
references.append((node.lineno, env[node.id]))
|
|
98
|
+
if isinstance(node, ast.Call):
|
|
99
|
+
pieces = []
|
|
100
|
+
head = node.func
|
|
101
|
+
while isinstance(head, ast.Attribute):
|
|
102
|
+
pieces.insert(0, head.attr)
|
|
103
|
+
head = head.value
|
|
104
|
+
if isinstance(head, ast.Name) and head.id in env:
|
|
105
|
+
calls.append((node.lineno, '.'.join([env[head.id]] + pieces)))
|
|
106
|
+
for child in ast.iter_child_nodes(node):
|
|
107
|
+
expression(child, env)
|
|
108
|
+
|
|
109
|
+
def statements(body, env, owner=None, prefix='', class_closure=None):
|
|
110
|
+
for node in body:
|
|
111
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
112
|
+
env.update(import_bindings(node, module, package))
|
|
113
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
114
|
+
for item in node.decorator_list + node.args.defaults + [d for d in node.args.kw_defaults if d]:
|
|
115
|
+
expression(item, env)
|
|
116
|
+
local = (class_closure if owner and class_closure is not None else env).copy()
|
|
117
|
+
for name in bound(node.body):
|
|
118
|
+
local.pop(name, None)
|
|
119
|
+
for arg in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
|
|
120
|
+
local.pop(arg.arg, None)
|
|
121
|
+
for arg in (node.args.vararg, node.args.kwarg):
|
|
122
|
+
if arg:
|
|
123
|
+
local.pop(arg.arg, None)
|
|
124
|
+
args = node.args.posonlyargs + node.args.args
|
|
125
|
+
if owner and args and args[0].arg in {'self', 'cls'} and args[0].arg not in bound(node.body):
|
|
126
|
+
local[args[0].arg] = owner
|
|
127
|
+
statements(node.body, local, prefix=prefix + node.name + '.')
|
|
128
|
+
env[node.name] = module + '.' + prefix + node.name
|
|
129
|
+
elif isinstance(node, ast.ClassDef):
|
|
130
|
+
for item in node.decorator_list + node.bases:
|
|
131
|
+
expression(item, env)
|
|
132
|
+
name = module + '.' + prefix + node.name
|
|
133
|
+
# Methods close over the containing function/module, not class locals.
|
|
134
|
+
statements(node.body, env.copy(), owner=name, prefix=prefix + node.name + '.', class_closure=env.copy())
|
|
135
|
+
env[node.name] = name
|
|
136
|
+
elif isinstance(node, (ast.If, ast.For, ast.AsyncFor, ast.While, ast.With, ast.AsyncWith, ast.Try, ast.TryStar, ast.Match)):
|
|
137
|
+
# Visit each branch in an isolated scope; invalidate any names
|
|
138
|
+
# it could write, so uncertain imports cannot leak past it.
|
|
139
|
+
changed = bound([node])
|
|
140
|
+
branch_env = {k: v for k, v in env.items() if k not in changed}
|
|
141
|
+
for field, value in ast.iter_fields(node):
|
|
142
|
+
if field in {'body', 'orelse', 'finalbody'}:
|
|
143
|
+
statements(value, branch_env.copy(), owner, prefix, class_closure)
|
|
144
|
+
elif field == 'handlers':
|
|
145
|
+
for handler in value:
|
|
146
|
+
statements(handler.body, branch_env.copy(), owner, prefix, class_closure)
|
|
147
|
+
elif field == 'cases':
|
|
148
|
+
for case in value:
|
|
149
|
+
expression(case.guard, branch_env)
|
|
150
|
+
statements(case.body, branch_env.copy(), owner, prefix, class_closure)
|
|
151
|
+
elif isinstance(value, ast.AST):
|
|
152
|
+
expression(value, env)
|
|
153
|
+
elif isinstance(value, list):
|
|
154
|
+
for item in value:
|
|
155
|
+
if isinstance(item, ast.AST):
|
|
156
|
+
expression(item, env)
|
|
157
|
+
for name in changed:
|
|
158
|
+
env.pop(name, None)
|
|
159
|
+
else:
|
|
160
|
+
expression(node, env)
|
|
161
|
+
for name in bound([node]):
|
|
162
|
+
env.pop(name, None)
|
|
163
|
+
statements(tree.body, globals_.copy())
|
|
164
|
+
return {'calls': calls, 'references_value': references}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def symbol_resolver(modules, symbols):
|
|
168
|
+
"""Allow source-root prefixes only when a module suffix is unambiguous."""
|
|
169
|
+
suffixes = {}
|
|
170
|
+
for module in modules:
|
|
171
|
+
parts = module.split('.')
|
|
172
|
+
for i in range(len(parts)):
|
|
173
|
+
suffixes.setdefault('.'.join(parts[i:]), set()).add(module)
|
|
174
|
+
|
|
175
|
+
def resolve(target):
|
|
176
|
+
if target in symbols:
|
|
177
|
+
return symbols[target]
|
|
178
|
+
parts = target.split('.')
|
|
179
|
+
for split in range(len(parts) - 1, 0, -1):
|
|
180
|
+
name = '.'.join(parts[:split])
|
|
181
|
+
matches = {name} if name in modules else suffixes.get(name, set())
|
|
182
|
+
if len(matches) == 1:
|
|
183
|
+
qualified = next(iter(matches)) + '.' + '.'.join(parts[split:])
|
|
184
|
+
return symbols.get(qualified, [])
|
|
185
|
+
if matches:
|
|
186
|
+
return []
|
|
187
|
+
return []
|
|
188
|
+
return resolve
|
|
@@ -3,9 +3,16 @@ from dataclasses import dataclass
|
|
|
3
3
|
import math
|
|
4
4
|
from typing import TypedDict
|
|
5
5
|
|
|
6
|
-
SCHEMA_VERSION = 'source-units-
|
|
6
|
+
SCHEMA_VERSION = 'source-units-v3'
|
|
7
7
|
RELATION_KINDS = frozenset({'calls', 'member_of', 'inherits', 'implements',
|
|
8
|
-
'same_symbol', 'imports', 'references_type'})
|
|
8
|
+
'same_symbol', 'imports', 'references_type', 'references_value'})
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SourceSyntaxError(ValueError):
|
|
12
|
+
"""File-local syntax diagnostics, distinct from adapter or validation failures."""
|
|
13
|
+
def __init__(self, diagnostics):
|
|
14
|
+
super().__init__('Source syntax errors')
|
|
15
|
+
self.diagnostics = diagnostics
|
|
9
16
|
|
|
10
17
|
|
|
11
18
|
@dataclass(frozen=True)
|
|
@@ -45,10 +45,16 @@ export function extract(files, maxLines = 65, settings = {}) {
|
|
|
45
45
|
const program = ts.createProgram([...texts.keys()], options, host);
|
|
46
46
|
const errors = program.getSyntacticDiagnostics();
|
|
47
47
|
if (errors.length) {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
48
|
+
if (errors.some(d => !d.file)) throw new Error('TypeScript syntax diagnostic is missing a source file');
|
|
49
|
+
const diagnostics = new Map();
|
|
50
|
+
for (const d of errors) {
|
|
51
|
+
const name = path.posix.relative(ROOT, d.file.fileName);
|
|
52
|
+
const position = d.file.getLineAndCharacterOfPosition(d.start ?? 0);
|
|
53
|
+
if (!diagnostics.has(name)) diagnostics.set(name, {path:name,
|
|
54
|
+
language:/\.(?:jsx?|mjs|cjs)$/i.test(name) ? 'javascript' : 'typescript',
|
|
55
|
+
errorType:'SyntaxError', line:position.line + 1, column:position.character + 1});
|
|
56
|
+
}
|
|
57
|
+
return {compilerVersion:ts.version, units:[], syntaxErrors:[...diagnostics.values()]};
|
|
52
58
|
}
|
|
53
59
|
const checker = program.getTypeChecker();
|
|
54
60
|
const units = [], records = new Map(), nodeEntries = new Map();
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import json
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
import subprocess
|
|
5
|
+
from .schema import SourceSyntaxError
|
|
5
6
|
|
|
6
7
|
COMPILER_VERSION = '5.9.3'
|
|
7
8
|
|
|
@@ -20,4 +21,6 @@ def extract(sources, max_lines=65, options=None):
|
|
|
20
21
|
output = json.loads(result.stdout)
|
|
21
22
|
if output['compilerVersion'] != COMPILER_VERSION:
|
|
22
23
|
raise ValueError('Unexpected TypeScript compiler version; run npm ci')
|
|
24
|
+
if output.get('syntaxErrors'):
|
|
25
|
+
raise SourceSyntaxError(output['syntaxErrors'])
|
|
23
26
|
return output['units']
|
package/src/retrieval/live.py
CHANGED
|
@@ -17,7 +17,7 @@ import numpy as np
|
|
|
17
17
|
from engine import document, post
|
|
18
18
|
from languages import adapter_manifest, source_units
|
|
19
19
|
from languages.files import discover_snapshot
|
|
20
|
-
from
|
|
20
|
+
from evidence import EvidenceEngine
|
|
21
21
|
from writer_lock import acquire_writer_lock
|
|
22
22
|
|
|
23
23
|
|
|
@@ -42,7 +42,7 @@ class Generation:
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
class LiveIndex:
|
|
45
|
-
def __init__(self, config, *, embed=post, engine_factory=
|
|
45
|
+
def __init__(self, config, *, embed=post, engine_factory=EvidenceEngine):
|
|
46
46
|
self.config = config
|
|
47
47
|
self.root = Path(config['root']).resolve()
|
|
48
48
|
self.state = Path(config['state']).resolve()
|
|
@@ -64,6 +64,9 @@ class LiveIndex:
|
|
|
64
64
|
self.dimensions = self.embedding['dimensions']
|
|
65
65
|
if type(self.dimensions) is not int or not 1 <= self.dimensions <= 65536:
|
|
66
66
|
raise ValueError('Invalid embedding dimensions')
|
|
67
|
+
self.batch_size = config.get('embeddingBatchSize', 64)
|
|
68
|
+
if type(self.batch_size) is not int or not 1 <= self.batch_size <= 64:
|
|
69
|
+
raise ValueError('Embedding batch size must be an integer from 1 to 64')
|
|
67
70
|
self.embed, self.engine_factory = embed, engine_factory
|
|
68
71
|
self.condition = threading.Condition()
|
|
69
72
|
self.stop_event = threading.Event()
|
|
@@ -164,7 +167,9 @@ class LiveIndex:
|
|
|
164
167
|
|
|
165
168
|
def _build(self, snapshot, identity):
|
|
166
169
|
start = time.monotonic()
|
|
167
|
-
|
|
170
|
+
report = {}
|
|
171
|
+
units = source_units(self.root, snapshot['files'], language_options=self.options,
|
|
172
|
+
cache=self.parse_cache, report=report)
|
|
168
173
|
documents = [document(unit) for unit in units]
|
|
169
174
|
keys = [digest([self.embedding, text]) for text in documents]
|
|
170
175
|
vectors, missing = {}, {}
|
|
@@ -179,10 +184,10 @@ class LiveIndex:
|
|
|
179
184
|
continue
|
|
180
185
|
missing[key] = text
|
|
181
186
|
entries = list(missing.items())
|
|
182
|
-
for offset in range(0, len(entries),
|
|
187
|
+
for offset in range(0, len(entries), self.batch_size):
|
|
183
188
|
if self.stop_event.is_set():
|
|
184
189
|
raise SourceChanged()
|
|
185
|
-
batch = entries[offset:offset+
|
|
190
|
+
batch = entries[offset:offset+self.batch_size]
|
|
186
191
|
result = self.embed(self.config['embeddingUrl']+'/embeddings',
|
|
187
192
|
{'model': self.embedding['model'], 'input': [text for _, text in batch]},
|
|
188
193
|
self.config.get('embeddingKey', 'local-only'), timeout=60)
|
|
@@ -207,6 +212,7 @@ class LiveIndex:
|
|
|
207
212
|
'changedFiles': sum(previous.get(path) != sha for path, sha in now.items()),
|
|
208
213
|
'deletedFiles': len(previous.keys() - now.keys()), 'embedding': self.embedding,
|
|
209
214
|
'languageUnits': dict(Counter(unit['language'] for unit in units)),
|
|
215
|
+
'degradedFiles': report['degradedFiles'], 'parseDiagnostics': report['parseDiagnostics'],
|
|
210
216
|
'indexingMs': round((time.monotonic()-start)*1000), 'completedAt': time.time()}
|
|
211
217
|
folder = self.state/('generation-'+uuid.uuid4().hex)
|
|
212
218
|
folder.mkdir(mode=0o700)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Deterministic behavior facets; every facet retains its full query context in retrieval."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def plan_query(query):
|
|
6
|
+
# Keep conditions in the intent, but do not spend a facet on a condition
|
|
7
|
+
# such as "when a user reopens a chat" without its requested behavior.
|
|
8
|
+
parts = [part.strip() for part in re.split(
|
|
9
|
+
r'[::;;,]|,\s+(?=how|which|why|where|what)|\s+and\s+(?=how|which|why|where|what|preserve|restore|validate|enforce|handle|load|save|reject|filter|check|merge|retain|stop)',
|
|
10
|
+
query, flags=re.I) if len(part.strip()) > 7]
|
|
11
|
+
actionable = [part for part in parts if not re.match(r'^(when|if|after|before|unless|given)\b', part, re.I)
|
|
12
|
+
and not (part.endswith('时') and not re.search(r'如何|怎么|哪里|怎样', part))]
|
|
13
|
+
facets = actionable if 1 < len(actionable) <= 4 else [query]
|
|
14
|
+
return {'intent': query, 'facets': [{'question': part, 'terms': re.findall(r'[A-Za-z][A-Za-z0-9_]*', part)}
|
|
15
|
+
for part in facets]}
|
|
@@ -33,8 +33,8 @@ def checked_rows(data, count):
|
|
|
33
33
|
|
|
34
34
|
def rerank_pairs(config, queries, documents, pairs, post):
|
|
35
35
|
api = config.get('api', 'rerank')
|
|
36
|
-
if api not in ('rerank', 'rerank-batch'):
|
|
37
|
-
raise ValueError('Rerank API must be rerank
|
|
36
|
+
if api not in ('rerank', 'rerank-batch', 'dashscope'):
|
|
37
|
+
raise ValueError('Rerank API must be rerank, rerank-batch or dashscope')
|
|
38
38
|
concurrency = bounded_integer(config.get('concurrency', 2), 'Rerank concurrency', 8)
|
|
39
39
|
limit = bounded_integer(config.get('maxDocuments', 128), 'Rerank document limit', 1024)
|
|
40
40
|
for q, d in pairs:
|
|
@@ -42,7 +42,8 @@ def rerank_pairs(config, queries, documents, pairs, post):
|
|
|
42
42
|
raise ValueError('Invalid requested rerank pair')
|
|
43
43
|
if not pairs:
|
|
44
44
|
return {'results': [], 'meta': {'request_count': 0}}
|
|
45
|
-
|
|
45
|
+
path = 'services/rerank/text-rerank/text-rerank' if api == 'dashscope' else api
|
|
46
|
+
url = config['baseUrl'].rstrip('/') + '/' + path
|
|
46
47
|
if api == 'rerank-batch':
|
|
47
48
|
data = post(url, {'model': config['model'], 'queries': queries,
|
|
48
49
|
'documents': documents, 'pairs': pairs}, config['apiKey'])
|
|
@@ -61,9 +62,14 @@ def rerank_pairs(config, queries, documents, pairs, post):
|
|
|
61
62
|
|
|
62
63
|
def request(job):
|
|
63
64
|
q, ids = job
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
65
|
+
inputs = {'query': queries[q], 'documents': [documents[d] for d in ids]}
|
|
66
|
+
body = ({'model': config['model'], 'input': inputs, 'parameters': {'top_n': len(ids)}}
|
|
67
|
+
if api == 'dashscope' else {'model': config['model'], **inputs, 'top_n': len(ids)})
|
|
68
|
+
data = post(url, body, config['apiKey'])
|
|
69
|
+
rows = checked_rows(data.get('output') if api == 'dashscope' and isinstance(data, dict) else data, len(ids))
|
|
70
|
+
if api == 'dashscope':
|
|
71
|
+
usage = data.get('usage', {})
|
|
72
|
+
data = {**data, 'usage': {'input_tokens': usage.get('total_tokens')} if isinstance(usage, dict) else {}}
|
|
67
73
|
return {(q, ids[row['index']]): row['relevance_score'] for row in rows}, data
|
|
68
74
|
|
|
69
75
|
# Fail the whole wave on any error; never silently drop a query or switch API.
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Discovery and expiring client leases for a single locked index writer."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import secrets
|
|
6
|
+
import re
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
PROTOCOL = 1
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def restrict_permissions(path):
|
|
16
|
+
if sys.platform == 'win32':
|
|
17
|
+
# os.open(mode=0o600) does not set a Windows DACL. Resolve the current
|
|
18
|
+
# user's SID (independent of localized account names) and remove inheritance.
|
|
19
|
+
result = subprocess.run(['whoami', '/user', '/fo', 'csv', '/nh'],
|
|
20
|
+
capture_output=True, check=True, timeout=5)
|
|
21
|
+
match = re.search(rb'S-1-[0-9-]+', result.stdout)
|
|
22
|
+
if not match:
|
|
23
|
+
raise PermissionError('Cannot determine current Windows user SID')
|
|
24
|
+
sid = match.group().decode('ascii')
|
|
25
|
+
subprocess.run(['icacls', str(path), '/inheritance:r', '/grant:r', '*' + sid + ':F'],
|
|
26
|
+
capture_output=True, check=True, timeout=5)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SharedWorker:
|
|
30
|
+
def __init__(self, config):
|
|
31
|
+
options = config['shared']
|
|
32
|
+
self.path = Path(config['state']) / 'worker.json'
|
|
33
|
+
self.instance = secrets.token_hex(24)
|
|
34
|
+
self.fingerprint = options['fingerprint']
|
|
35
|
+
self.key = config['serviceKey']
|
|
36
|
+
self.ttl = options.get('leaseSeconds', 15)
|
|
37
|
+
self.idle = options.get('idleSeconds', 30)
|
|
38
|
+
self.leases = {}
|
|
39
|
+
self.active = 0
|
|
40
|
+
self.empty_since = time.monotonic()
|
|
41
|
+
self.stopping = False
|
|
42
|
+
self.lock = threading.Lock()
|
|
43
|
+
self.finished = threading.Event()
|
|
44
|
+
|
|
45
|
+
def publish(self, port):
|
|
46
|
+
# Called only by the writer-lock holder, after HTTP bind succeeds.
|
|
47
|
+
self.record = {'protocol': PROTOCOL, 'instanceId': self.instance,
|
|
48
|
+
'pid': os.getpid(), 'port': port, 'apiKey': self.key}
|
|
49
|
+
temporary = self.path.with_name('worker-' + self.instance + '.tmp')
|
|
50
|
+
try:
|
|
51
|
+
with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as file:
|
|
52
|
+
restrict_permissions(temporary)
|
|
53
|
+
json.dump(self.record, file)
|
|
54
|
+
os.replace(temporary, self.path)
|
|
55
|
+
finally:
|
|
56
|
+
temporary.unlink(missing_ok=True)
|
|
57
|
+
|
|
58
|
+
def lease(self, body, release=False):
|
|
59
|
+
with self.lock:
|
|
60
|
+
if body.get('instanceId') != self.instance:
|
|
61
|
+
return 409, {'error': 'Worker instance changed', 'code': 'INSTANCE_CHANGED'}
|
|
62
|
+
if not secrets.compare_digest(str(body.get('fingerprint', '')), self.fingerprint):
|
|
63
|
+
return 409, {'error': 'This index is used by a worker with incompatible configuration or runtime. '
|
|
64
|
+
'Close its clients before changing configuration, or use a separate --state directory.',
|
|
65
|
+
'code': 'CONFIG_MISMATCH'}
|
|
66
|
+
identifier = body.get('leaseId')
|
|
67
|
+
if not isinstance(identifier, str) or not 1 <= len(identifier) <= 128:
|
|
68
|
+
return 422, {'error': 'Invalid lease identifier'}
|
|
69
|
+
if self.stopping:
|
|
70
|
+
return 503, {'error': 'Worker is stopping'}
|
|
71
|
+
if release:
|
|
72
|
+
self.leases.pop(identifier, None)
|
|
73
|
+
else:
|
|
74
|
+
self.leases[identifier] = time.monotonic() + self.ttl
|
|
75
|
+
self.empty_since = time.monotonic()
|
|
76
|
+
return 200, {'instanceId': self.instance, 'pid': os.getpid(), 'leaseSeconds': self.ttl}
|
|
77
|
+
|
|
78
|
+
def enter(self):
|
|
79
|
+
with self.lock:
|
|
80
|
+
if self.stopping:
|
|
81
|
+
return False
|
|
82
|
+
self.active += 1
|
|
83
|
+
return True
|
|
84
|
+
|
|
85
|
+
def leave(self):
|
|
86
|
+
with self.lock:
|
|
87
|
+
self.active -= 1
|
|
88
|
+
self.empty_since = time.monotonic()
|
|
89
|
+
|
|
90
|
+
def should_stop(self):
|
|
91
|
+
with self.lock:
|
|
92
|
+
now = time.monotonic()
|
|
93
|
+
self.leases = {key: until for key, until in self.leases.items() if until > now}
|
|
94
|
+
if self.leases or self.active:
|
|
95
|
+
self.empty_since = now
|
|
96
|
+
elif now - self.empty_since >= self.idle:
|
|
97
|
+
self.stopping = True
|
|
98
|
+
return self.stopping
|
|
99
|
+
|
|
100
|
+
def monitor(self, server):
|
|
101
|
+
def run():
|
|
102
|
+
while not self.finished.wait(min(1, self.idle / 2)):
|
|
103
|
+
if self.should_stop():
|
|
104
|
+
server.shutdown()
|
|
105
|
+
return
|
|
106
|
+
threading.Thread(target=run, name='worker-leases', daemon=True).start()
|
|
107
|
+
|
|
108
|
+
def close(self):
|
|
109
|
+
self.finished.set()
|
|
110
|
+
# Cleanup precedes releasing the writer lock; never unlink a successor's record.
|
|
111
|
+
try:
|
|
112
|
+
if json.loads(self.path.read_text()).get('instanceId') == self.instance:
|
|
113
|
+
self.path.unlink()
|
|
114
|
+
except (FileNotFoundError, ValueError):
|
|
115
|
+
pass
|
|
@@ -15,11 +15,15 @@ else:
|
|
|
15
15
|
fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
class WriterBusy(ValueError):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
18
22
|
def acquire_writer_lock(path):
|
|
19
23
|
file = path.open('a+b')
|
|
20
24
|
try:
|
|
21
25
|
lock(file)
|
|
22
26
|
except OSError:
|
|
23
27
|
file.close()
|
|
24
|
-
raise
|
|
28
|
+
raise WriterBusy(f'This index directory already has a running writer: {path}') from None
|
|
25
29
|
return file
|
package/src/service.mjs
CHANGED
|
@@ -30,6 +30,7 @@ export function serviceConfig({root, state, port = 0} = {}, environment = proces
|
|
|
30
30
|
embeddingKey: env.EMBEDDING_API_KEY,
|
|
31
31
|
embeddingModel: env.EMBEDDING_MODEL || 'Qwen3-Embedding-4B',
|
|
32
32
|
embeddingDimensions: Number(env.OCE_EMBEDDING_DIMENSIONS || 1024),
|
|
33
|
+
embeddingBatchSize: Number(env.OCE_EMBEDDING_BATCH_SIZE ?? 64),
|
|
33
34
|
embeddingRevision: env.OCE_EMBEDDING_REVISION || '1',
|
|
34
35
|
reranker: {...reranker, baseUrl: runtime.requestBaseUrl},
|
|
35
36
|
languageOptions: env.OCE_LANGUAGE_OPTIONS ? JSON.parse(env.OCE_LANGUAGE_OPTIONS) : {},
|
|
@@ -38,13 +39,13 @@ export function serviceConfig({root, state, port = 0} = {}, environment = proces
|
|
|
38
39
|
};
|
|
39
40
|
}
|
|
40
41
|
|
|
41
|
-
export function startService(settings, {log = line => process.stderr.write(line + '\n')} = {}) {
|
|
42
|
+
export function startService(settings, {log = line => process.stderr.write(line + '\n'), startupMs = 15000} = {}) {
|
|
42
43
|
const {python, config} = settings;
|
|
43
44
|
const child = spawn(python, [resolve(projectRoot, 'scripts/retrieval-server.py')], {
|
|
44
45
|
cwd: projectRoot, env: {...process.env, ...settings.workerEnv,
|
|
45
46
|
PATH:dirname(process.execPath)+delimiter+(process.env.PATH || ''),
|
|
46
47
|
PYTHONUTF8:'1', PYTHONIOENCODING:'utf-8', OPENBLAS_NUM_THREADS:'2', OMP_NUM_THREADS:'2'},
|
|
47
|
-
stdio:['pipe', 'pipe', 'pipe'],
|
|
48
|
+
stdio:['pipe', 'pipe', 'pipe'], detached:Boolean(config.shared), windowsHide:true,
|
|
48
49
|
});
|
|
49
50
|
const lines = createInterface({input: child.stdout});
|
|
50
51
|
const errors = createInterface({input: child.stderr});
|
|
@@ -52,14 +53,29 @@ export function startService(settings, {log = line => process.stderr.write(line
|
|
|
52
53
|
child.stdin.on('error', () => {}); // Spawn/exit handlers report early failures.
|
|
53
54
|
child.stdin.end(JSON.stringify(config) + '\n');
|
|
54
55
|
const ready = new Promise((resolveReady, reject) => {
|
|
55
|
-
const timer = setTimeout(() => {
|
|
56
|
+
const timer = setTimeout(() => {
|
|
57
|
+
child.kill();
|
|
58
|
+
reject(Object.assign(new Error('Retrieval worker startup timed out'), {code:'STARTUP_TIMEOUT'}));
|
|
59
|
+
}, startupMs);
|
|
56
60
|
child.once('error', error => {clearTimeout(timer); reject(error);});
|
|
57
61
|
child.once('exit', code => {clearTimeout(timer); reject(new Error(`Retrieval worker exited (${code})`));});
|
|
58
62
|
lines.on('line', line => {
|
|
59
63
|
try {
|
|
60
64
|
const value = JSON.parse(line);
|
|
65
|
+
if (value.startupError) {
|
|
66
|
+
clearTimeout(timer);
|
|
67
|
+
const error = new Error(`Retrieval worker failed to start: ${value.startupError.message}`);
|
|
68
|
+
error.code = value.startupError.code;
|
|
69
|
+
reject(error);
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
61
72
|
if (value.listening) {
|
|
62
73
|
clearTimeout(timer);
|
|
74
|
+
if (config.shared) {
|
|
75
|
+
lines.close(); errors.close();
|
|
76
|
+
child.stdout.destroy(); child.stderr.destroy();
|
|
77
|
+
child.unref();
|
|
78
|
+
}
|
|
63
79
|
resolveReady({baseUrl:value.listening, apiKey:config.serviceKey});
|
|
64
80
|
return;
|
|
65
81
|
}
|
package/src/setup.mjs
CHANGED
|
@@ -13,6 +13,8 @@ export function validateModels(env) {
|
|
|
13
13
|
}
|
|
14
14
|
const dimensions = Number(env.OCE_EMBEDDING_DIMENSIONS);
|
|
15
15
|
if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error('Embedding dimensions must be a positive integer');
|
|
16
|
+
const batchSize = Number(env.OCE_EMBEDDING_BATCH_SIZE ?? 64);
|
|
17
|
+
if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 64) throw new Error('Embedding batch size must be an integer from 1 to 64');
|
|
16
18
|
}
|
|
17
19
|
|
|
18
20
|
export function terminalPrompt(input = process.stdin, output = process.stderr) {
|