open-context-engine 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,188 @@
1
+ """Conservative lexical import/call resolution; never infer dynamic receivers."""
2
+ import ast
3
+
4
+
5
+ def import_bindings(node, module, package):
6
+ if isinstance(node, ast.Import):
7
+ return {a.asname or a.name.split('.')[0]: a.name if a.asname else a.name.split('.')[0]
8
+ for a in node.names}
9
+ prefix = node.module or ''
10
+ if node.level:
11
+ parts = package.split('.') if package else []
12
+ if node.level > len(parts):
13
+ return {}
14
+ prefix = '.'.join(parts[:len(parts) - node.level + 1] + ([prefix] if prefix else []))
15
+ return {a.asname or a.name: '.'.join(filter(None, [prefix, a.name]))
16
+ for a in node.names if a.name != '*'}
17
+
18
+
19
+ class Bindings(ast.NodeVisitor):
20
+ def __init__(self):
21
+ self.names = set()
22
+
23
+ def visit_Name(self, node):
24
+ if isinstance(node.ctx, (ast.Store, ast.Del)):
25
+ self.names.add(node.id)
26
+
27
+ def visit_Import(self, node):
28
+ self.names.update(a.asname or a.name.split('.')[0] for a in node.names)
29
+
30
+ def visit_ImportFrom(self, node):
31
+ self.names.update(a.asname or a.name for a in node.names)
32
+
33
+ def visit_FunctionDef(self, node):
34
+ self.names.add(node.name)
35
+
36
+ visit_AsyncFunctionDef = visit_FunctionDef
37
+ visit_ClassDef = visit_FunctionDef
38
+ def visit_Lambda(self, node):
39
+ pass
40
+
41
+ def visit_ExceptHandler(self, node):
42
+ if node.name:
43
+ self.names.add(node.name)
44
+ self.generic_visit(node)
45
+
46
+
47
+ def bound(body):
48
+ visitor = Bindings()
49
+ for node in body:
50
+ visitor.visit(node)
51
+ return visitor.names
52
+
53
+
54
+ def resolved_links(tree, module, package):
55
+ """Return calls with source positions and scoped, qualified targets.
56
+
57
+ Imports under uncertain control flow are not assumed to have executed.
58
+ Function-local names shadow outer bindings even before their assignment.
59
+ """
60
+ calls, references = [], []
61
+ globals_ = {}
62
+ for node in tree.body:
63
+ if isinstance(node, (ast.Import, ast.ImportFrom)):
64
+ globals_.update(import_bindings(node, module, package))
65
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
66
+ globals_[node.name] = module + '.' + node.name
67
+ # A module-level reassignment makes the runtime binding indeterminate.
68
+ for node in tree.body:
69
+ if not isinstance(node, (ast.Import, ast.ImportFrom, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
70
+ for name in bound([node]):
71
+ globals_.pop(name, None)
72
+
73
+ def expression(node, env):
74
+ if node is None:
75
+ return
76
+ if isinstance(node, ast.Lambda):
77
+ local = env.copy()
78
+ for arg in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
79
+ local.pop(arg.arg, None)
80
+ for arg in (node.args.vararg, node.args.kwarg):
81
+ if arg:
82
+ local.pop(arg.arg, None)
83
+ expression(node.body, local)
84
+ return
85
+ if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)):
86
+ local = env.copy()
87
+ for gen in node.generators:
88
+ expression(gen.iter, local)
89
+ for name in bound([gen.target]):
90
+ local.pop(name, None)
91
+ for condition in gen.ifs:
92
+ expression(condition, local)
93
+ for field in ('elt', 'key', 'value'):
94
+ expression(getattr(node, field, None), local)
95
+ return
96
+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) and node.id in env:
97
+ references.append((node.lineno, env[node.id]))
98
+ if isinstance(node, ast.Call):
99
+ pieces = []
100
+ head = node.func
101
+ while isinstance(head, ast.Attribute):
102
+ pieces.insert(0, head.attr)
103
+ head = head.value
104
+ if isinstance(head, ast.Name) and head.id in env:
105
+ calls.append((node.lineno, '.'.join([env[head.id]] + pieces)))
106
+ for child in ast.iter_child_nodes(node):
107
+ expression(child, env)
108
+
109
+ def statements(body, env, owner=None, prefix='', class_closure=None):
110
+ for node in body:
111
+ if isinstance(node, (ast.Import, ast.ImportFrom)):
112
+ env.update(import_bindings(node, module, package))
113
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
114
+ for item in node.decorator_list + node.args.defaults + [d for d in node.args.kw_defaults if d]:
115
+ expression(item, env)
116
+ local = (class_closure if owner and class_closure is not None else env).copy()
117
+ for name in bound(node.body):
118
+ local.pop(name, None)
119
+ for arg in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
120
+ local.pop(arg.arg, None)
121
+ for arg in (node.args.vararg, node.args.kwarg):
122
+ if arg:
123
+ local.pop(arg.arg, None)
124
+ args = node.args.posonlyargs + node.args.args
125
+ if owner and args and args[0].arg in {'self', 'cls'} and args[0].arg not in bound(node.body):
126
+ local[args[0].arg] = owner
127
+ statements(node.body, local, prefix=prefix + node.name + '.')
128
+ env[node.name] = module + '.' + prefix + node.name
129
+ elif isinstance(node, ast.ClassDef):
130
+ for item in node.decorator_list + node.bases:
131
+ expression(item, env)
132
+ name = module + '.' + prefix + node.name
133
+ # Methods close over the containing function/module, not class locals.
134
+ statements(node.body, env.copy(), owner=name, prefix=prefix + node.name + '.', class_closure=env.copy())
135
+ env[node.name] = name
136
+ elif isinstance(node, (ast.If, ast.For, ast.AsyncFor, ast.While, ast.With, ast.AsyncWith, ast.Try, ast.TryStar, ast.Match)):
137
+ # Visit each branch in an isolated scope; invalidate any names
138
+ # it could write, so uncertain imports cannot leak past it.
139
+ changed = bound([node])
140
+ branch_env = {k: v for k, v in env.items() if k not in changed}
141
+ for field, value in ast.iter_fields(node):
142
+ if field in {'body', 'orelse', 'finalbody'}:
143
+ statements(value, branch_env.copy(), owner, prefix, class_closure)
144
+ elif field == 'handlers':
145
+ for handler in value:
146
+ statements(handler.body, branch_env.copy(), owner, prefix, class_closure)
147
+ elif field == 'cases':
148
+ for case in value:
149
+ expression(case.guard, branch_env)
150
+ statements(case.body, branch_env.copy(), owner, prefix, class_closure)
151
+ elif isinstance(value, ast.AST):
152
+ expression(value, env)
153
+ elif isinstance(value, list):
154
+ for item in value:
155
+ if isinstance(item, ast.AST):
156
+ expression(item, env)
157
+ for name in changed:
158
+ env.pop(name, None)
159
+ else:
160
+ expression(node, env)
161
+ for name in bound([node]):
162
+ env.pop(name, None)
163
+ statements(tree.body, globals_.copy())
164
+ return {'calls': calls, 'references_value': references}
165
+
166
+
167
+ def symbol_resolver(modules, symbols):
168
+ """Allow source-root prefixes only when a module suffix is unambiguous."""
169
+ suffixes = {}
170
+ for module in modules:
171
+ parts = module.split('.')
172
+ for i in range(len(parts)):
173
+ suffixes.setdefault('.'.join(parts[i:]), set()).add(module)
174
+
175
+ def resolve(target):
176
+ if target in symbols:
177
+ return symbols[target]
178
+ parts = target.split('.')
179
+ for split in range(len(parts) - 1, 0, -1):
180
+ name = '.'.join(parts[:split])
181
+ matches = {name} if name in modules else suffixes.get(name, set())
182
+ if len(matches) == 1:
183
+ qualified = next(iter(matches)) + '.' + '.'.join(parts[split:])
184
+ return symbols.get(qualified, [])
185
+ if matches:
186
+ return []
187
+ return []
188
+ return resolve
@@ -3,9 +3,16 @@ from dataclasses import dataclass
3
3
  import math
4
4
  from typing import TypedDict
5
5
 
6
- SCHEMA_VERSION = 'source-units-v2'
6
+ SCHEMA_VERSION = 'source-units-v3'
7
7
  RELATION_KINDS = frozenset({'calls', 'member_of', 'inherits', 'implements',
8
- 'same_symbol', 'imports', 'references_type'})
8
+ 'same_symbol', 'imports', 'references_type', 'references_value'})
9
+
10
+
11
+ class SourceSyntaxError(ValueError):
12
+ """File-local syntax diagnostics, distinct from adapter or validation failures."""
13
+ def __init__(self, diagnostics):
14
+ super().__init__('Source syntax errors')
15
+ self.diagnostics = diagnostics
9
16
 
10
17
 
11
18
  @dataclass(frozen=True)
@@ -1,7 +1,8 @@
1
1
  /** Parse and bind only the frozen source set. Never emit, execute, or load repo plugins. */
2
2
  import ts from 'typescript';
3
- import { readFileSync } from 'node:fs';
3
+ import { readFileSync, existsSync, realpathSync } from 'node:fs';
4
4
  import path from 'node:path';
5
+ import { fileURLToPath } from 'node:url';
5
6
 
6
7
  const COMPILER_VERSION = '5.9.3';
7
8
  if (ts.version !== COMPILER_VERSION) throw new Error(`Expected TypeScript ${COMPILER_VERSION}; run npm ci`);
@@ -44,10 +45,16 @@ export function extract(files, maxLines = 65, settings = {}) {
44
45
  const program = ts.createProgram([...texts.keys()], options, host);
45
46
  const errors = program.getSyntacticDiagnostics();
46
47
  if (errors.length) {
47
- throw new Error(errors.slice(0, 5).map(d => {
48
- const line = d.file.getLineAndCharacterOfPosition(d.start ?? 0).line + 1;
49
- return `${path.posix.relative(ROOT, d.file.fileName)}:${line}: ${ts.flattenDiagnosticMessageText(d.messageText, ' ')}`;
50
- }).join('\n'));
48
+ if (errors.some(d => !d.file)) throw new Error('TypeScript syntax diagnostic is missing a source file');
49
+ const diagnostics = new Map();
50
+ for (const d of errors) {
51
+ const name = path.posix.relative(ROOT, d.file.fileName);
52
+ const position = d.file.getLineAndCharacterOfPosition(d.start ?? 0);
53
+ if (!diagnostics.has(name)) diagnostics.set(name, {path:name,
54
+ language:/\.(?:jsx?|mjs|cjs)$/i.test(name) ? 'javascript' : 'typescript',
55
+ errorType:'SyntaxError', line:position.line + 1, column:position.character + 1});
56
+ }
57
+ return {compilerVersion:ts.version, units:[], syntaxErrors:[...diagnostics.values()]};
51
58
  }
52
59
  const checker = program.getTypeChecker();
53
60
  const units = [], records = new Map(), nodeEntries = new Map();
@@ -223,7 +230,8 @@ export function extract(files, maxLines = 65, settings = {}) {
223
230
  return { compilerVersion: ts.version, units };
224
231
  }
225
232
 
226
- if (process.argv[1] && new URL(import.meta.url).pathname === path.resolve(process.argv[1])) {
233
+ if (process.argv[1] && existsSync(process.argv[1])
234
+ && realpathSync.native(fileURLToPath(import.meta.url)) === realpathSync.native(process.argv[1])) {
227
235
  try {
228
236
  const input = JSON.parse(readFileSync(0, 'utf8'));
229
237
  process.stdout.write(JSON.stringify(extract(input.files, input.maxLines, input.options)));
@@ -2,6 +2,7 @@
2
2
  import json
3
3
  from pathlib import Path
4
4
  import subprocess
5
+ from .schema import SourceSyntaxError
5
6
 
6
7
  COMPILER_VERSION = '5.9.3'
7
8
 
@@ -11,7 +12,7 @@ def extract(sources, max_lines=65, options=None):
11
12
  'maxLines': max_lines, 'options': options or {}}
12
13
  try:
13
14
  result = subprocess.run(['node', str(Path(__file__).with_suffix('.mjs'))],
14
- input=json.dumps(payload), text=True, capture_output=True,
15
+ input=json.dumps(payload), encoding='utf-8', capture_output=True,
15
16
  timeout=120, check=False)
16
17
  except FileNotFoundError as error:
17
18
  raise RuntimeError('TypeScript indexing requires Node.js and npm ci') from error
@@ -20,4 +21,6 @@ def extract(sources, max_lines=65, options=None):
20
21
  output = json.loads(result.stdout)
21
22
  if output['compilerVersion'] != COMPILER_VERSION:
22
23
  raise ValueError('Unexpected TypeScript compiler version; run npm ci')
24
+ if output.get('syntaxErrors'):
25
+ raise SourceSyntaxError(output['syntaxErrors'])
23
26
  return output['units']
@@ -2,7 +2,6 @@
2
2
  from collections import Counter
3
3
  from contextlib import closing
4
4
  from dataclasses import dataclass
5
- import fcntl
6
5
  import hashlib
7
6
  import json
8
7
  import os
@@ -18,7 +17,8 @@ import numpy as np
18
17
  from engine import document, post
19
18
  from languages import adapter_manifest, source_units
20
19
  from languages.files import discover_snapshot
21
- from routed import RoutedEngine
20
+ from evidence import EvidenceEngine
21
+ from writer_lock import acquire_writer_lock
22
22
 
23
23
 
24
24
  def digest(value):
@@ -42,7 +42,7 @@ class Generation:
42
42
 
43
43
 
44
44
  class LiveIndex:
45
- def __init__(self, config, *, embed=post, engine_factory=RoutedEngine):
45
+ def __init__(self, config, *, embed=post, engine_factory=EvidenceEngine):
46
46
  self.config = config
47
47
  self.root = Path(config['root']).resolve()
48
48
  self.state = Path(config['state']).resolve()
@@ -64,6 +64,9 @@ class LiveIndex:
64
64
  self.dimensions = self.embedding['dimensions']
65
65
  if type(self.dimensions) is not int or not 1 <= self.dimensions <= 65536:
66
66
  raise ValueError('Invalid embedding dimensions')
67
+ self.batch_size = config.get('embeddingBatchSize', 64)
68
+ if type(self.batch_size) is not int or not 1 <= self.batch_size <= 64:
69
+ raise ValueError('Embedding batch size must be an integer from 1 to 64')
67
70
  self.embed, self.engine_factory = embed, engine_factory
68
71
  self.condition = threading.Condition()
69
72
  self.stop_event = threading.Event()
@@ -72,12 +75,7 @@ class LiveIndex:
72
75
  self.phase = 'starting'
73
76
  self.parse_cache = {}
74
77
  self.state.mkdir(parents=True, exist_ok=True, mode=0o700)
75
- self.lock_file = (self.state/'writer.lock').open('a')
76
- try:
77
- fcntl.flock(self.lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
78
- except OSError:
79
- self.lock_file.close()
80
- raise ValueError('This index directory already has a running writer') from None
78
+ self.lock_file = acquire_writer_lock(self.state/'writer.lock')
81
79
  self.thread = threading.Thread(target=self._run, name='repository-index', daemon=True)
82
80
 
83
81
  def scan(self):
@@ -169,7 +167,9 @@ class LiveIndex:
169
167
 
170
168
  def _build(self, snapshot, identity):
171
169
  start = time.monotonic()
172
- units = source_units(self.root, snapshot['files'], language_options=self.options, cache=self.parse_cache)
170
+ report = {}
171
+ units = source_units(self.root, snapshot['files'], language_options=self.options,
172
+ cache=self.parse_cache, report=report)
173
173
  documents = [document(unit) for unit in units]
174
174
  keys = [digest([self.embedding, text]) for text in documents]
175
175
  vectors, missing = {}, {}
@@ -184,10 +184,10 @@ class LiveIndex:
184
184
  continue
185
185
  missing[key] = text
186
186
  entries = list(missing.items())
187
- for offset in range(0, len(entries), 64):
187
+ for offset in range(0, len(entries), self.batch_size):
188
188
  if self.stop_event.is_set():
189
189
  raise SourceChanged()
190
- batch = entries[offset:offset+64]
190
+ batch = entries[offset:offset+self.batch_size]
191
191
  result = self.embed(self.config['embeddingUrl']+'/embeddings',
192
192
  {'model': self.embedding['model'], 'input': [text for _, text in batch]},
193
193
  self.config.get('embeddingKey', 'local-only'), timeout=60)
@@ -212,6 +212,7 @@ class LiveIndex:
212
212
  'changedFiles': sum(previous.get(path) != sha for path, sha in now.items()),
213
213
  'deletedFiles': len(previous.keys() - now.keys()), 'embedding': self.embedding,
214
214
  'languageUnits': dict(Counter(unit['language'] for unit in units)),
215
+ 'degradedFiles': report['degradedFiles'], 'parseDiagnostics': report['parseDiagnostics'],
215
216
  'indexingMs': round((time.monotonic()-start)*1000), 'completedAt': time.time()}
216
217
  folder = self.state/('generation-'+uuid.uuid4().hex)
217
218
  folder.mkdir(mode=0o700)
@@ -0,0 +1,15 @@
1
+ """Deterministic behavior facets; every facet retains its full query context in retrieval."""
2
+ import re
3
+
4
+
5
+ def plan_query(query):
6
+ # Keep conditions in the intent, but do not spend a facet on a condition
7
+ # such as "when a user reopens a chat" without its requested behavior.
8
+ parts = [part.strip() for part in re.split(
9
+ r'[::;;,]|,\s+(?=how|which|why|where|what)|\s+and\s+(?=how|which|why|where|what|preserve|restore|validate|enforce|handle|load|save|reject|filter|check|merge|retain|stop)',
10
+ query, flags=re.I) if len(part.strip()) > 7]
11
+ actionable = [part for part in parts if not re.match(r'^(when|if|after|before|unless|given)\b', part, re.I)
12
+ and not (part.endswith('时') and not re.search(r'如何|怎么|哪里|怎样', part))]
13
+ facets = actionable if 1 < len(actionable) <= 4 else [query]
14
+ return {'intent': query, 'facets': [{'question': part, 'terms': re.findall(r'[A-Za-z][A-Za-z0-9_]*', part)}
15
+ for part in facets]}
@@ -33,8 +33,8 @@ def checked_rows(data, count):
33
33
 
34
34
  def rerank_pairs(config, queries, documents, pairs, post):
35
35
  api = config.get('api', 'rerank')
36
- if api not in ('rerank', 'rerank-batch'):
37
- raise ValueError('Rerank API must be rerank or rerank-batch')
36
+ if api not in ('rerank', 'rerank-batch', 'dashscope'):
37
+ raise ValueError('Rerank API must be rerank, rerank-batch or dashscope')
38
38
  concurrency = bounded_integer(config.get('concurrency', 2), 'Rerank concurrency', 8)
39
39
  limit = bounded_integer(config.get('maxDocuments', 128), 'Rerank document limit', 1024)
40
40
  for q, d in pairs:
@@ -42,7 +42,8 @@ def rerank_pairs(config, queries, documents, pairs, post):
42
42
  raise ValueError('Invalid requested rerank pair')
43
43
  if not pairs:
44
44
  return {'results': [], 'meta': {'request_count': 0}}
45
- url = config['baseUrl'].rstrip('/') + '/' + api
45
+ path = 'services/rerank/text-rerank/text-rerank' if api == 'dashscope' else api
46
+ url = config['baseUrl'].rstrip('/') + '/' + path
46
47
  if api == 'rerank-batch':
47
48
  data = post(url, {'model': config['model'], 'queries': queries,
48
49
  'documents': documents, 'pairs': pairs}, config['apiKey'])
@@ -61,9 +62,14 @@ def rerank_pairs(config, queries, documents, pairs, post):
61
62
 
62
63
  def request(job):
63
64
  q, ids = job
64
- data = post(url, {'model': config['model'], 'query': queries[q],
65
- 'documents': [documents[d] for d in ids], 'top_n': len(ids)}, config['apiKey'])
66
- rows = checked_rows(data, len(ids))
65
+ inputs = {'query': queries[q], 'documents': [documents[d] for d in ids]}
66
+ body = ({'model': config['model'], 'input': inputs, 'parameters': {'top_n': len(ids)}}
67
+ if api == 'dashscope' else {'model': config['model'], **inputs, 'top_n': len(ids)})
68
+ data = post(url, body, config['apiKey'])
69
+ rows = checked_rows(data.get('output') if api == 'dashscope' and isinstance(data, dict) else data, len(ids))
70
+ if api == 'dashscope':
71
+ usage = data.get('usage', {})
72
+ data = {**data, 'usage': {'input_tokens': usage.get('total_tokens')} if isinstance(usage, dict) else {}}
67
73
  return {(q, ids[row['index']]): row['relevance_score'] for row in rows}, data
68
74
 
69
75
  # Fail the whole wave on any error; never silently drop a query or switch API.
@@ -0,0 +1,115 @@
1
+ """Discovery and expiring client leases for a single locked index writer."""
2
+ import json
3
+ import os
4
+ from pathlib import Path
5
+ import secrets
6
+ import re
7
+ import subprocess
8
+ import sys
9
+ import threading
10
+ import time
11
+
12
+ PROTOCOL = 1
13
+
14
+
15
+ def restrict_permissions(path):
16
+ if sys.platform == 'win32':
17
+ # os.open(mode=0o600) does not set a Windows DACL. Resolve the current
18
+ # user's SID (independent of localized account names) and remove inheritance.
19
+ result = subprocess.run(['whoami', '/user', '/fo', 'csv', '/nh'],
20
+ capture_output=True, check=True, timeout=5)
21
+ match = re.search(rb'S-1-[0-9-]+', result.stdout)
22
+ if not match:
23
+ raise PermissionError('Cannot determine current Windows user SID')
24
+ sid = match.group().decode('ascii')
25
+ subprocess.run(['icacls', str(path), '/inheritance:r', '/grant:r', '*' + sid + ':F'],
26
+ capture_output=True, check=True, timeout=5)
27
+
28
+
29
+ class SharedWorker:
30
+ def __init__(self, config):
31
+ options = config['shared']
32
+ self.path = Path(config['state']) / 'worker.json'
33
+ self.instance = secrets.token_hex(24)
34
+ self.fingerprint = options['fingerprint']
35
+ self.key = config['serviceKey']
36
+ self.ttl = options.get('leaseSeconds', 15)
37
+ self.idle = options.get('idleSeconds', 30)
38
+ self.leases = {}
39
+ self.active = 0
40
+ self.empty_since = time.monotonic()
41
+ self.stopping = False
42
+ self.lock = threading.Lock()
43
+ self.finished = threading.Event()
44
+
45
+ def publish(self, port):
46
+ # Called only by the writer-lock holder, after HTTP bind succeeds.
47
+ self.record = {'protocol': PROTOCOL, 'instanceId': self.instance,
48
+ 'pid': os.getpid(), 'port': port, 'apiKey': self.key}
49
+ temporary = self.path.with_name('worker-' + self.instance + '.tmp')
50
+ try:
51
+ with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'w') as file:
52
+ restrict_permissions(temporary)
53
+ json.dump(self.record, file)
54
+ os.replace(temporary, self.path)
55
+ finally:
56
+ temporary.unlink(missing_ok=True)
57
+
58
+ def lease(self, body, release=False):
59
+ with self.lock:
60
+ if body.get('instanceId') != self.instance:
61
+ return 409, {'error': 'Worker instance changed', 'code': 'INSTANCE_CHANGED'}
62
+ if not secrets.compare_digest(str(body.get('fingerprint', '')), self.fingerprint):
63
+ return 409, {'error': 'This index is used by a worker with incompatible configuration or runtime. '
64
+ 'Close its clients before changing configuration, or use a separate --state directory.',
65
+ 'code': 'CONFIG_MISMATCH'}
66
+ identifier = body.get('leaseId')
67
+ if not isinstance(identifier, str) or not 1 <= len(identifier) <= 128:
68
+ return 422, {'error': 'Invalid lease identifier'}
69
+ if self.stopping:
70
+ return 503, {'error': 'Worker is stopping'}
71
+ if release:
72
+ self.leases.pop(identifier, None)
73
+ else:
74
+ self.leases[identifier] = time.monotonic() + self.ttl
75
+ self.empty_since = time.monotonic()
76
+ return 200, {'instanceId': self.instance, 'pid': os.getpid(), 'leaseSeconds': self.ttl}
77
+
78
+ def enter(self):
79
+ with self.lock:
80
+ if self.stopping:
81
+ return False
82
+ self.active += 1
83
+ return True
84
+
85
+ def leave(self):
86
+ with self.lock:
87
+ self.active -= 1
88
+ self.empty_since = time.monotonic()
89
+
90
+ def should_stop(self):
91
+ with self.lock:
92
+ now = time.monotonic()
93
+ self.leases = {key: until for key, until in self.leases.items() if until > now}
94
+ if self.leases or self.active:
95
+ self.empty_since = now
96
+ elif now - self.empty_since >= self.idle:
97
+ self.stopping = True
98
+ return self.stopping
99
+
100
+ def monitor(self, server):
101
+ def run():
102
+ while not self.finished.wait(min(1, self.idle / 2)):
103
+ if self.should_stop():
104
+ server.shutdown()
105
+ return
106
+ threading.Thread(target=run, name='worker-leases', daemon=True).start()
107
+
108
+ def close(self):
109
+ self.finished.set()
110
+ # Cleanup precedes releasing the writer lock; never unlink a successor's record.
111
+ try:
112
+ if json.loads(self.path.read_text()).get('instanceId') == self.instance:
113
+ self.path.unlink()
114
+ except (FileNotFoundError, ValueError):
115
+ pass
@@ -0,0 +1,29 @@
1
+ """Nonblocking process lock, released when its file handle is closed."""
2
+ import sys
3
+
4
+ if sys.platform == 'win32':
5
+ import msvcrt
6
+
7
+ def lock(file):
8
+ # Every writer locks the same byte, including when the file is empty.
9
+ file.seek(0)
10
+ msvcrt.locking(file.fileno(), msvcrt.LK_NBLCK, 1)
11
+ else:
12
+ import fcntl
13
+
14
+ def lock(file):
15
+ fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB)
16
+
17
+
18
+ class WriterBusy(ValueError):
19
+ pass
20
+
21
+
22
+ def acquire_writer_lock(path):
23
+ file = path.open('a+b')
24
+ try:
25
+ lock(file)
26
+ except OSError:
27
+ file.close()
28
+ raise WriterBusy(f'This index directory already has a running writer: {path}') from None
29
+ return file
package/src/runtime.mjs CHANGED
@@ -6,6 +6,12 @@ import { createHash } from 'node:crypto';
6
6
  import { configDirectory, projectRoot } from './config.mjs';
7
7
 
8
8
  const exec = promisify(execFile);
9
+ export function defaultPython(platform = process.platform) {
10
+ return platform === 'win32' ? 'python' : 'python3';
11
+ }
12
+ export function venvPython(directory, platform = process.platform) {
13
+ return platform === 'win32' ? join(directory,'Scripts','python.exe') : join(directory,'bin','python');
14
+ }
9
15
  export async function run(command, args, options = {}) {
10
16
  try {return await exec(command,args,{timeout:600000,maxBuffer:4*1024*1024,...options});}
11
17
  catch (error) {
@@ -21,7 +27,8 @@ export async function verifyPython(python, environment = process.env, execute =
21
27
  return (await execute(python,['-c',probe],{env:environment,timeout:120000})).stdout.trim();
22
28
  }
23
29
 
24
- export async function ensureRuntime(environment, {python = 'python3', execute = run, log = () => {}} = {}) {
30
+ export async function ensureRuntime(environment, {platform = process.platform,
31
+ python = defaultPython(platform), execute = run, log = () => {}} = {}) {
25
32
  const runtimes = join(configDirectory(environment),'runtimes');
26
33
  const requirements = await readFile(join(projectRoot,'requirements.txt'));
27
34
  const fingerprint = createHash('sha256').update(requirements).digest('hex');
@@ -43,7 +50,7 @@ export async function ensureRuntime(environment, {python = 'python3', execute =
43
50
  await mkdir(runtimes,{recursive:true,mode:0o700});
44
51
  // Virtual environments cannot be relocated. Each installation is built in its final directory.
45
52
  const directory = await mkdtemp(join(runtimes,'python-'));
46
- const executable = join(directory,'bin','python');
53
+ const executable = venvPython(directory,platform);
47
54
  const env = {...environment,TIKTOKEN_CACHE_DIR:join(directory,'tokenizer')};
48
55
  try {
49
56
  log('Creating an isolated Python environment...');