@minaim/openclaw-channel 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -2
- package/bridge.mjs +92 -0
- package/diagnostics.mjs +47 -0
- package/index.mjs +128 -0
- package/openclaw.plugin.json +25 -0
- package/package.json +55 -4
- package/setup.mjs +37 -0
- package/vendor/__init__.py +0 -0
- package/vendor/compatibility.json +7 -0
- package/vendor/mina_agent.py +363 -0
- package/vendor/mina_media.py +172 -0
- package/vendor/mina_native.py +293 -0
- package/vendor/mina_setup.py +133 -0
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Mina Bot protocol v1 reference adapter. Python 3.9+, standard library only.
|
|
3
|
+
|
|
4
|
+
A host adapter supplies handle(event) -> final text. This program never turns
|
|
5
|
+
message text into a command, changes the destination, or retries an Agent run.
|
|
6
|
+
"""
|
|
7
|
+
import argparse
|
|
8
|
+
import fcntl
|
|
9
|
+
import hashlib
|
|
10
|
+
import hmac
|
|
11
|
+
import importlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import pathlib
|
|
15
|
+
import re
|
|
16
|
+
import secrets
|
|
17
|
+
import sqlite3
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
import urllib.error
|
|
21
|
+
import urllib.parse
|
|
22
|
+
import urllib.request
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ConfigurationError(Exception):
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ProtocolError(Exception):
|
|
30
|
+
def __init__(self, status, code):
|
|
31
|
+
self.status, self.code = status, code
|
|
32
|
+
super().__init__('Bot API %s: %s' % (status, code))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def sign(secret, route, app_id, version, timestamp, nonce, body):
|
|
36
|
+
canonical = '\n'.join(('POST', route, app_id, str(version), str(timestamp),
|
|
37
|
+
nonce, hashlib.sha256(body).hexdigest()))
|
|
38
|
+
return hmac.new(secret.encode('utf-8'), canonical.encode('utf-8'), hashlib.sha256).hexdigest()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def encode(value):
|
|
42
|
+
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), sort_keys=True)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
46
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class HttpGateway:
|
|
51
|
+
def __init__(self, url, app_id, secret, version, *, allow_loopback_http=False):
|
|
52
|
+
parsed = urllib.parse.urlsplit(url)
|
|
53
|
+
local = parsed.hostname in ('127.0.0.1', '::1', 'localhost')
|
|
54
|
+
if (parsed.scheme != 'https' and not (allow_loopback_http and local and parsed.scheme == 'http')
|
|
55
|
+
or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment
|
|
56
|
+
or not re.fullmatch(r'(/[A-Za-z0-9_-]+)*/?', parsed.path)):
|
|
57
|
+
raise ConfigurationError('gateway must be an HTTPS base URL without credentials or query')
|
|
58
|
+
self.url = url.rstrip('/')
|
|
59
|
+
self.app_id, self.secret, self.version = app_id, secret, version
|
|
60
|
+
self.opener = urllib.request.build_opener(NoRedirect())
|
|
61
|
+
|
|
62
|
+
def post(self, route, body):
|
|
63
|
+
if not re.fullmatch(r'/bot/v1/[A-Za-z0-9_/-]+', route) or '//' in route:
|
|
64
|
+
raise ConfigurationError('invalid protocol route')
|
|
65
|
+
raw = encode(body).encode('utf-8')
|
|
66
|
+
timestamp, nonce = int(time.time() * 1000), secrets.token_urlsafe(24)
|
|
67
|
+
headers = {'Content-Type': 'application/json', 'X-Mina-App-Id': self.app_id,
|
|
68
|
+
'X-Mina-Credential-Version': str(self.version), 'X-Mina-Timestamp': str(timestamp),
|
|
69
|
+
'X-Mina-Nonce': nonce,
|
|
70
|
+
'X-Mina-Signature': sign(self.secret, route, self.app_id, self.version, timestamp, nonce, raw)}
|
|
71
|
+
request = urllib.request.Request(self.url + route, data=raw, headers=headers, method='POST')
|
|
72
|
+
try:
|
|
73
|
+
with self.opener.open(request, timeout=75 if '/media/' in route else 30) as response:
|
|
74
|
+
limit=28*1024*1024 if re.fullmatch(r'/bot/v1/events/evt_[a-f0-9]{32}/media/att_[a-f0-9]{32}/download',route) else 1048576
|
|
75
|
+
data = response.read(limit+1)
|
|
76
|
+
if len(data) > limit:
|
|
77
|
+
raise ProtocolError(502, 'oversized_response')
|
|
78
|
+
return json.loads(data)
|
|
79
|
+
except urllib.error.HTTPError as error:
|
|
80
|
+
# Never print URLs, headers, response bodies or remote diagnostic text.
|
|
81
|
+
try:
|
|
82
|
+
code = json.loads(error.read(4096)).get('error', 'http_error')
|
|
83
|
+
except (ValueError, AttributeError):
|
|
84
|
+
code = 'http_error'
|
|
85
|
+
if not isinstance(code, str) or not re.fullmatch(r'[a-z_]{1,64}', code):
|
|
86
|
+
code = 'http_error'
|
|
87
|
+
raise ProtocolError(error.code, code) from None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class Journal:
|
|
91
|
+
"""Durable local inbox; scope and run decision survive process crashes."""
|
|
92
|
+
def __init__(self, path, app_id, credential_version):
|
|
93
|
+
path = pathlib.Path(path)
|
|
94
|
+
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
95
|
+
if path.is_symlink():
|
|
96
|
+
raise ConfigurationError('state file must not be a symbolic link')
|
|
97
|
+
fd = os.open(path, os.O_CREAT | os.O_RDWR | getattr(os, 'O_NOFOLLOW', 0), 0o600)
|
|
98
|
+
try:
|
|
99
|
+
if os.fstat(fd).st_mode & 0o077:
|
|
100
|
+
raise ConfigurationError('state file permissions must be 0600')
|
|
101
|
+
try:
|
|
102
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
103
|
+
except BlockingIOError:
|
|
104
|
+
raise ConfigurationError('another adapter is already using this journal') from None
|
|
105
|
+
except BaseException:
|
|
106
|
+
os.close(fd)
|
|
107
|
+
raise
|
|
108
|
+
self.file_lock = fd
|
|
109
|
+
self.lock = threading.RLock()
|
|
110
|
+
self.db = sqlite3.connect(str(path), check_same_thread=False)
|
|
111
|
+
self.db.row_factory = sqlite3.Row
|
|
112
|
+
self.db.execute('PRAGMA synchronous=FULL')
|
|
113
|
+
self.db.execute('PRAGMA journal_mode=DELETE')
|
|
114
|
+
with self.db:
|
|
115
|
+
self.db.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
|
|
116
|
+
self.db.execute('''CREATE TABLE IF NOT EXISTS events (id TEXT PRIMARY KEY, payload TEXT NOT NULL,
|
|
117
|
+
generation INTEGER NOT NULL, state TEXT NOT NULL, execution_id TEXT NOT NULL DEFAULT '',
|
|
118
|
+
result TEXT NOT NULL DEFAULT '', created_ms INTEGER NOT NULL)''')
|
|
119
|
+
if 'result_format' not in {row[1] for row in self.db.execute('PRAGMA table_info(events)')}:
|
|
120
|
+
with self.db:self.db.execute("ALTER TABLE events ADD COLUMN result_format TEXT NOT NULL DEFAULT 'text'")
|
|
121
|
+
identity = encode([app_id, credential_version])
|
|
122
|
+
old = self.meta('identity')
|
|
123
|
+
if old is not None and old != identity:
|
|
124
|
+
self.close()
|
|
125
|
+
raise ConfigurationError('use a new state file for different credentials')
|
|
126
|
+
with self.db:
|
|
127
|
+
for key, value in [('identity', identity), ('consumer_id', secrets.token_urlsafe(24)),
|
|
128
|
+
('lease_token', secrets.token_urlsafe(32))]:
|
|
129
|
+
self.db.execute('INSERT OR IGNORE INTO meta VALUES (?,?)', (key, value))
|
|
130
|
+
# The external side effect may have happened. A restart is never permission to rerun it.
|
|
131
|
+
self.db.execute("UPDATE events SET state='unknown' WHERE state IN ('start_requested','running')")
|
|
132
|
+
|
|
133
|
+
def meta(self, key):
|
|
134
|
+
with self.lock:
|
|
135
|
+
row = self.db.execute('SELECT value FROM meta WHERE key=?', (key,)).fetchone()
|
|
136
|
+
return row['value'] if row else None
|
|
137
|
+
|
|
138
|
+
def event(self, event_id):
|
|
139
|
+
with self.lock:
|
|
140
|
+
row = self.db.execute('SELECT * FROM events WHERE id=?', (event_id,)).fetchone()
|
|
141
|
+
return dict(row) if row else None
|
|
142
|
+
|
|
143
|
+
def put_event(self, event, generation):
|
|
144
|
+
event_id = event.get('event_id', '')
|
|
145
|
+
if not re.fullmatch(r'evt_[A-Za-z0-9_-]{1,40}', event_id):
|
|
146
|
+
raise ConfigurationError('invalid event ID')
|
|
147
|
+
payload = encode(event)
|
|
148
|
+
with self.lock, self.db:
|
|
149
|
+
existing = self.event(event_id)
|
|
150
|
+
if existing and (existing['payload'] != payload or existing['generation'] != generation):
|
|
151
|
+
raise ConfigurationError('event scope or content conflict')
|
|
152
|
+
self.db.execute('INSERT OR IGNORE INTO events(id,payload,generation,state,created_ms) VALUES (?,?,?,?,?)',
|
|
153
|
+
(event_id, payload, generation, 'stored', int(time.time() * 1000)))
|
|
154
|
+
return self.event(event_id)
|
|
155
|
+
|
|
156
|
+
def transition(self, event_id, state, *, execution_id=None, result=None, result_format=None):
|
|
157
|
+
with self.lock, self.db:
|
|
158
|
+
self.db.execute('UPDATE events SET state=?,execution_id=COALESCE(?,execution_id),'
|
|
159
|
+
'result=COALESCE(?,result),result_format=COALESCE(?,result_format) WHERE id=?', (state, execution_id, result, result_format, event_id))
|
|
160
|
+
|
|
161
|
+
def pending(self, generation):
|
|
162
|
+
with self.lock:
|
|
163
|
+
return [json.loads(row['payload']) for row in self.db.execute(
|
|
164
|
+
"SELECT payload FROM events WHERE generation=? AND state IN ('stored','acked','result_ready') ORDER BY rowid",
|
|
165
|
+
(generation,)).fetchall()]
|
|
166
|
+
|
|
167
|
+
def fence(self, generation):
|
|
168
|
+
with self.lock, self.db:
|
|
169
|
+
self.db.execute("UPDATE events SET state='fenced' WHERE generation<>? AND state NOT IN ('submitted','fenced')", (generation,))
|
|
170
|
+
|
|
171
|
+
def close(self):
|
|
172
|
+
with self.lock:
|
|
173
|
+
self.db.close()
|
|
174
|
+
if self.file_lock is not None:
|
|
175
|
+
os.close(self.file_lock)
|
|
176
|
+
self.file_lock = None
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class Runner:
|
|
180
|
+
def __init__(self, gateway, journal, handler, client_type='Generic Agent'):
|
|
181
|
+
self.gateway, self.journal, self.handler = gateway, journal, handler
|
|
182
|
+
self.client_type = client_type
|
|
183
|
+
self.generation = None
|
|
184
|
+
self.stop = threading.Event()
|
|
185
|
+
|
|
186
|
+
def connect(self, takeover=False):
|
|
187
|
+
response = self.gateway.post('/bot/v1/connect', {
|
|
188
|
+
'protocol_version': '1', 'consumer_id': self.journal.meta('consumer_id'),
|
|
189
|
+
'lease_token': self.journal.meta('lease_token'), 'client_type': self.client_type, 'takeover': takeover,
|
|
190
|
+
**({'capabilities':self.capabilities} if hasattr(self,'capabilities') else {})})
|
|
191
|
+
self.generation = response['lease_generation']
|
|
192
|
+
self.journal.fence(self.generation)
|
|
193
|
+
return response
|
|
194
|
+
|
|
195
|
+
def request(self, route, **body):
|
|
196
|
+
return self.gateway.post(route, dict(body, lease_generation=self.generation,
|
|
197
|
+
lease_token=self.journal.meta('lease_token')))
|
|
198
|
+
|
|
199
|
+
def handle(self, event):
|
|
200
|
+
try:
|
|
201
|
+
self._handle(event)
|
|
202
|
+
self.report_status("connected")
|
|
203
|
+
except ProtocolError as error:
|
|
204
|
+
if error.code == 'event_expired':
|
|
205
|
+
self.journal.transition(event['event_id'], 'expired')
|
|
206
|
+
elif error.code in ('execution_unknown', 'invalid_state', 'stale_execution',
|
|
207
|
+
'recovery_required', 'content_conflict', 'event_not_found', 'invalid_result'):
|
|
208
|
+
self.journal.transition(event['event_id'], 'unknown')
|
|
209
|
+
elif error.code in ('stale_lease', 'authorization_revoked') or error.status in (401, 403, 404):
|
|
210
|
+
self.journal.transition(event['event_id'], 'fenced')
|
|
211
|
+
self.stop.set()
|
|
212
|
+
elif error.status in (400, 413):
|
|
213
|
+
self.journal.transition(event['event_id'], 'unknown')
|
|
214
|
+
self.stop.set()
|
|
215
|
+
raise
|
|
216
|
+
|
|
217
|
+
def _handle(self, event):
|
|
218
|
+
row = self.journal.put_event(event, self.generation)
|
|
219
|
+
event_id = event['event_id']
|
|
220
|
+
task = '/bot/v1/tasks/' + event_id
|
|
221
|
+
if row['state'] in ('submitted', 'unknown', 'fenced', 'expired'):
|
|
222
|
+
return
|
|
223
|
+
if row['state'] == 'stored':
|
|
224
|
+
self.request('/bot/v1/events/' + event_id + '/ack')
|
|
225
|
+
self.journal.transition(event_id, 'acked')
|
|
226
|
+
row['state'] = 'acked'
|
|
227
|
+
if row['state'] == 'acked':
|
|
228
|
+
if self.stop.is_set():
|
|
229
|
+
return
|
|
230
|
+
self.before_start(dict(event))
|
|
231
|
+
self.journal.transition(event_id, 'start_requested')
|
|
232
|
+
try:
|
|
233
|
+
permit = self.request(task + '/start')
|
|
234
|
+
except ProtocolError as error:
|
|
235
|
+
# A definitive rejection did not grant permission; keep serial ordering for a later poll.
|
|
236
|
+
if error.code == 'session_busy':
|
|
237
|
+
self.journal.transition(event_id, 'acked')
|
|
238
|
+
else:
|
|
239
|
+
self.journal.transition(event_id, 'unknown')
|
|
240
|
+
raise
|
|
241
|
+
except Exception:
|
|
242
|
+
self.journal.transition(event_id, 'unknown')
|
|
243
|
+
raise
|
|
244
|
+
execution = permit.get('execution_id', '')
|
|
245
|
+
if permit.get('allow_execute') is not True:
|
|
246
|
+
self.journal.transition(event_id, 'unknown', execution_id=execution)
|
|
247
|
+
return
|
|
248
|
+
self.journal.transition(event_id, 'running', execution_id=execution)
|
|
249
|
+
try:
|
|
250
|
+
if self.stop.is_set():
|
|
251
|
+
raise RuntimeError('connection no longer valid')
|
|
252
|
+
result = self.store_result(dict(event), self.invoke_handler(dict(event)))
|
|
253
|
+
except BaseException:
|
|
254
|
+
self.journal.transition(event_id, 'unknown')
|
|
255
|
+
try:
|
|
256
|
+
self.request(task + '/fail', execution_id=execution)
|
|
257
|
+
except Exception:
|
|
258
|
+
pass
|
|
259
|
+
raise
|
|
260
|
+
self.journal.transition(event_id, 'result_ready', result=result, result_format=self.result_format(dict(event)))
|
|
261
|
+
row = self.journal.event(event_id)
|
|
262
|
+
if row['state'] == 'result_ready':
|
|
263
|
+
self.submit_result(dict(event),row)
|
|
264
|
+
self.journal.transition(event_id, 'submitted')
|
|
265
|
+
|
|
266
|
+
def report_status(self,code):
|
|
267
|
+
pass
|
|
268
|
+
|
|
269
|
+
def before_start(self,event):
|
|
270
|
+
pass
|
|
271
|
+
|
|
272
|
+
def invoke_handler(self,event):
|
|
273
|
+
return self.handler(event)
|
|
274
|
+
|
|
275
|
+
def store_result(self,event,result):
|
|
276
|
+
if not isinstance(result,str) or not result or len(result.encode('utf-8'))>32768:
|
|
277
|
+
raise ConfigurationError('handler result must be 1..32768 UTF-8 bytes')
|
|
278
|
+
return result
|
|
279
|
+
|
|
280
|
+
def result_format(self,event):
|
|
281
|
+
return 'text'
|
|
282
|
+
|
|
283
|
+
def submit_result(self,event,row):
|
|
284
|
+
self.request('/bot/v1/tasks/'+event['event_id']+'/reply',execution_id=row['execution_id'],content=row['result'])
|
|
285
|
+
|
|
286
|
+
def heartbeat(self):
|
|
287
|
+
while not self.stop.wait(20):
|
|
288
|
+
try:
|
|
289
|
+
self.request('/bot/v1/heartbeat')
|
|
290
|
+
except ProtocolError as error:
|
|
291
|
+
self.report_status(error.code)
|
|
292
|
+
if error.status in (401, 403, 404, 409):
|
|
293
|
+
self.stop.set()
|
|
294
|
+
except Exception:
|
|
295
|
+
self.report_status("network_unavailable")
|
|
296
|
+
# Main loop reconnects before accepting further execution after a prolonged outage.
|
|
297
|
+
self.stop.set()
|
|
298
|
+
|
|
299
|
+
def run(self, takeover=False):
|
|
300
|
+
self.connect(takeover)
|
|
301
|
+
heart = threading.Thread(target=self.heartbeat, name='mina-heartbeat', daemon=True)
|
|
302
|
+
heart.start()
|
|
303
|
+
try:
|
|
304
|
+
backoff = 1
|
|
305
|
+
while not self.stop.is_set():
|
|
306
|
+
try:
|
|
307
|
+
pending = self.journal.pending(self.generation)
|
|
308
|
+
if pending:
|
|
309
|
+
for event in pending:
|
|
310
|
+
self.handle(event)
|
|
311
|
+
self.stop.wait(1)
|
|
312
|
+
continue
|
|
313
|
+
response = self.request('/bot/v1/events/poll', wait_ms=20000)
|
|
314
|
+
self.report_status('connected')
|
|
315
|
+
if response['lease_generation'] != self.generation:
|
|
316
|
+
raise ConfigurationError('lease generation mismatch')
|
|
317
|
+
# Persist the whole batch before any ACK; ACK each event only when ready to process it.
|
|
318
|
+
for event in response['events']:
|
|
319
|
+
self.journal.put_event(event, self.generation)
|
|
320
|
+
backoff = 1
|
|
321
|
+
except ProtocolError as error:
|
|
322
|
+
self.report_status(error.code)
|
|
323
|
+
if error.status in (401, 403, 404) or error.code in ('stale_lease', 'authorization_revoked'):
|
|
324
|
+
raise
|
|
325
|
+
self.stop.wait(min(backoff, 20))
|
|
326
|
+
backoff = min(backoff * 2, 20)
|
|
327
|
+
except OSError:
|
|
328
|
+
self.report_status('network_unavailable')
|
|
329
|
+
self.stop.wait(min(backoff, 20))
|
|
330
|
+
backoff = min(backoff * 2, 20)
|
|
331
|
+
finally:
|
|
332
|
+
self.stop.set()
|
|
333
|
+
heart.join(timeout=2)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def main():
|
|
337
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
338
|
+
parser.add_argument('--config', required=True, help='private JSON file (mode 0600)')
|
|
339
|
+
parser.add_argument('--takeover', action='store_true', help='explicitly replace another active consumer')
|
|
340
|
+
args = parser.parse_args()
|
|
341
|
+
config_path = pathlib.Path(args.config)
|
|
342
|
+
if config_path.stat().st_mode & 0o077:
|
|
343
|
+
raise ConfigurationError('config file permissions must be 0600')
|
|
344
|
+
config = json.loads(config_path.read_text(encoding='utf-8'))
|
|
345
|
+
module, function = config['handler'].split(':', 1)
|
|
346
|
+
handler = getattr(importlib.import_module(module), function)
|
|
347
|
+
gateway = HttpGateway(config['gateway'], config['app_id'], config['app_secret'], config['credential_version'])
|
|
348
|
+
journal = Journal(config['state_file'], config['app_id'], config['credential_version'])
|
|
349
|
+
try:
|
|
350
|
+
Runner(gateway, journal, handler, config.get('client_type', 'Generic Agent')).run(args.takeover)
|
|
351
|
+
finally:
|
|
352
|
+
journal.close()
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
if __name__ == '__main__':
|
|
356
|
+
try:
|
|
357
|
+
main()
|
|
358
|
+
except KeyboardInterrupt:
|
|
359
|
+
pass
|
|
360
|
+
except ProtocolError as error:
|
|
361
|
+
raise SystemExit(str(error)) from None
|
|
362
|
+
except Exception:
|
|
363
|
+
raise SystemExit('Adapter stopped; check local configuration and connection. No task was automatically re-executed.') from None
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""Bounded, durable media snapshots; only explicit host attachments become uploads."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import http.client
|
|
4
|
+
import ipaddress
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import socket
|
|
9
|
+
import ssl
|
|
10
|
+
import stat
|
|
11
|
+
import urllib.parse
|
|
12
|
+
if __package__:
|
|
13
|
+
from .mina_agent import ConfigurationError
|
|
14
|
+
else:
|
|
15
|
+
from mina_agent import ConfigurationError
|
|
16
|
+
|
|
17
|
+
MAX_FILE = 20*1024*1024
|
|
18
|
+
MAX_IMAGE = 8*1024*1024
|
|
19
|
+
|
|
20
|
+
class PublicHTTPS(http.client.HTTPSConnection):
|
|
21
|
+
def connect(self):
|
|
22
|
+
# Resolve once, reject the entire answer if any address is non-public,
|
|
23
|
+
# and connect to that IP while retaining TLS hostname verification.
|
|
24
|
+
infos = socket.getaddrinfo(self.host, self.port, type=socket.SOCK_STREAM)
|
|
25
|
+
if not infos or any(not ipaddress.ip_address(info[4][0]).is_global for info in infos):
|
|
26
|
+
raise ConfigurationError('remote attachment must use public HTTPS')
|
|
27
|
+
sock = socket.create_connection(infos[0][4][:2], timeout=self.timeout)
|
|
28
|
+
try:
|
|
29
|
+
self.sock = self._context.wrap_socket(sock, server_hostname=self.host)
|
|
30
|
+
except BaseException:
|
|
31
|
+
sock.close()
|
|
32
|
+
raise
|
|
33
|
+
|
|
34
|
+
class MediaFiles:
|
|
35
|
+
def __init__(self, config):
|
|
36
|
+
self.config = config
|
|
37
|
+
identity = hashlib.sha256((config['app_id']+':'+str(config['credential_version'])).encode()).hexdigest()[:24]
|
|
38
|
+
self.root = Path(config['state_file']).parent / ('mina-media-'+identity)
|
|
39
|
+
self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
40
|
+
if self.root.is_symlink() or self.root.stat().st_mode & 0o077:
|
|
41
|
+
raise ConfigurationError('media directory must be private')
|
|
42
|
+
self.roots = [Path(p).resolve(strict=True) for p in config.get('media_roots', [])]
|
|
43
|
+
if any(not p.is_dir() or p == Path('/') for p in self.roots):
|
|
44
|
+
raise ConfigurationError('media roots must be explicit directories')
|
|
45
|
+
self.hosts = config.get('media_hosts', [])
|
|
46
|
+
|
|
47
|
+
@staticmethod
|
|
48
|
+
def _check_name(name):
|
|
49
|
+
if (not isinstance(name,str) or not name or name != name.strip() or len(name.encode())>180
|
|
50
|
+
or name in ('.','..') or any(c in name for c in '/\\:') or any(ord(c)<32 or ord(c)==127 for c in name)):
|
|
51
|
+
raise ConfigurationError('unsafe attachment name')
|
|
52
|
+
return name
|
|
53
|
+
|
|
54
|
+
def _local(self, path, limit):
|
|
55
|
+
candidate = Path(path)
|
|
56
|
+
if not candidate.is_absolute() or '..' in candidate.parts:
|
|
57
|
+
raise ConfigurationError('attachment path must be inside a configured root')
|
|
58
|
+
for forbidden in [self.config['state_file'], self.config.get('_config_path','')]:
|
|
59
|
+
if forbidden and (candidate == Path(forbidden) or str(candidate).startswith(str(forbidden)+'-')):
|
|
60
|
+
raise ConfigurationError('credential/state files cannot be attachments')
|
|
61
|
+
for root in self.roots:
|
|
62
|
+
try: relative = candidate.relative_to(root)
|
|
63
|
+
except ValueError: continue
|
|
64
|
+
if not relative.parts: continue
|
|
65
|
+
fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
66
|
+
try:
|
|
67
|
+
for part in relative.parts[:-1]:
|
|
68
|
+
next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
69
|
+
os.close(fd); fd = next_fd
|
|
70
|
+
file_fd = os.open(relative.parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd)
|
|
71
|
+
with os.fdopen(file_fd,'rb') as stream:
|
|
72
|
+
info = os.fstat(stream.fileno())
|
|
73
|
+
if not stat.S_ISREG(info.st_mode) or not 1 <= info.st_size <= limit:
|
|
74
|
+
raise ConfigurationError('attachment is not a bounded regular file')
|
|
75
|
+
data = stream.read(limit+1)
|
|
76
|
+
if len(data)!=info.st_size or len(data)>limit:
|
|
77
|
+
raise ConfigurationError('attachment changed while being read')
|
|
78
|
+
return data
|
|
79
|
+
except OSError:
|
|
80
|
+
raise ConfigurationError('attachment cannot be opened safely') from None
|
|
81
|
+
finally: os.close(fd)
|
|
82
|
+
raise ConfigurationError('attachment path is outside configured media roots')
|
|
83
|
+
|
|
84
|
+
def _remote(self, url, limit):
|
|
85
|
+
parsed = urllib.parse.urlsplit(url)
|
|
86
|
+
if (parsed.scheme!='https' or parsed.username or parsed.password or parsed.fragment
|
|
87
|
+
or parsed.hostname not in self.hosts or parsed.port not in (None,443)):
|
|
88
|
+
raise ConfigurationError('remote attachment host is not allowed')
|
|
89
|
+
connection = PublicHTTPS(parsed.hostname,443,timeout=30,context=ssl.create_default_context())
|
|
90
|
+
try:
|
|
91
|
+
target = urllib.parse.urlunsplit(('', '', parsed.path or '/', parsed.query, ''))
|
|
92
|
+
connection.request('GET',target,headers={'Accept-Encoding':'identity'})
|
|
93
|
+
response = connection.getresponse()
|
|
94
|
+
if response.status!=200 or response.getheader('Content-Encoding','identity')!='identity':
|
|
95
|
+
raise ConfigurationError('remote attachment response rejected')
|
|
96
|
+
length=response.getheader('Content-Length')
|
|
97
|
+
if length is not None and (not length.isdigit() or int(length)>limit):
|
|
98
|
+
raise ConfigurationError('remote attachment too large')
|
|
99
|
+
data=response.read(limit+1)
|
|
100
|
+
if not data or len(data)>limit: raise ConfigurationError('remote attachment too large')
|
|
101
|
+
return data
|
|
102
|
+
finally: connection.close()
|
|
103
|
+
|
|
104
|
+
def _write(self, area, event_id, filename, data):
|
|
105
|
+
if not isinstance(event_id,str) or not __import__('re').fullmatch('evt_[a-f0-9]{32}',event_id):
|
|
106
|
+
raise ConfigurationError('invalid media event scope')
|
|
107
|
+
(self.root/area).mkdir(mode=0o700,exist_ok=True)
|
|
108
|
+
parent=self.root/area/event_id
|
|
109
|
+
parent.mkdir(mode=0o700,exist_ok=True)
|
|
110
|
+
for p in [self.root/area,parent]:
|
|
111
|
+
if p.is_symlink() or p.stat().st_mode & 0o077: raise ConfigurationError('media scope is not private')
|
|
112
|
+
destination=parent/filename
|
|
113
|
+
if destination.exists():
|
|
114
|
+
if destination.is_symlink() or destination.read_bytes()!=data: raise ConfigurationError('media snapshot conflict')
|
|
115
|
+
return str(destination)
|
|
116
|
+
fd=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
|
117
|
+
try:
|
|
118
|
+
with os.fdopen(fd,'wb',closefd=False) as stream:
|
|
119
|
+
stream.write(data);stream.flush();os.fsync(fd)
|
|
120
|
+
finally: os.close(fd)
|
|
121
|
+
directory=os.open(parent,os.O_RDONLY|os.O_DIRECTORY)
|
|
122
|
+
try: os.fsync(directory)
|
|
123
|
+
finally: os.close(directory)
|
|
124
|
+
return str(destination)
|
|
125
|
+
|
|
126
|
+
def cleanup(self,event_id,output_only=False):
|
|
127
|
+
if not __import__('re').fullmatch('evt_[a-f0-9]{32}',event_id): return
|
|
128
|
+
for area in (['out'] if output_only else ['in','out']):
|
|
129
|
+
parent=self.root/area/event_id
|
|
130
|
+
if not parent.exists() or parent.is_symlink(): continue
|
|
131
|
+
for child in parent.iterdir():
|
|
132
|
+
if child.is_file() or child.is_symlink(): child.unlink()
|
|
133
|
+
try: parent.rmdir()
|
|
134
|
+
except OSError: pass
|
|
135
|
+
|
|
136
|
+
def snapshot(self, item, event_id, index):
|
|
137
|
+
if not isinstance(item,dict) or set(item)-{'path','kind','name'}:
|
|
138
|
+
raise ConfigurationError('invalid explicit host attachment')
|
|
139
|
+
path,kind=item.get('path'),item.get('kind')
|
|
140
|
+
if not isinstance(path,str) or kind not in ('image','file'): raise ConfigurationError('invalid host attachment')
|
|
141
|
+
name=self._check_name(item.get('name') or Path(urllib.parse.urlsplit(path).path).name or 'attachment.bin')
|
|
142
|
+
limit=MAX_IMAGE if kind=='image' else MAX_FILE
|
|
143
|
+
data=self._remote(path,limit) if path.startswith('https://') else self._local(path,limit)
|
|
144
|
+
digest=hashlib.sha256(data).hexdigest()
|
|
145
|
+
saved=self._write('out',event_id,str(index)+'-'+digest,data)
|
|
146
|
+
return dict(path=saved,kind=kind,name=name,size=len(data),sha256=digest,
|
|
147
|
+
upload_id='media_'+hashlib.sha256((event_id+':'+str(index)+':'+digest).encode()).hexdigest())
|
|
148
|
+
|
|
149
|
+
def prepare_result(self, result, event_id):
|
|
150
|
+
if isinstance(result,str): result={'text':result,'attachments':[]}
|
|
151
|
+
if not isinstance(result,dict) or set(result)-{'text','attachments'}: raise ConfigurationError('invalid native result')
|
|
152
|
+
text,items=result.get('text',''),result.get('attachments',[])
|
|
153
|
+
if (not isinstance(text,str) or len(text.encode())>32768 or not isinstance(items,list)
|
|
154
|
+
or len(items)>5 or (not text and not items)):
|
|
155
|
+
raise ConfigurationError('native result is empty or too large')
|
|
156
|
+
return dict(text=text,attachments=[self.snapshot(item,event_id,n) for n,item in enumerate(items)])
|
|
157
|
+
|
|
158
|
+
def read_snapshot(self, item, event_id):
|
|
159
|
+
path=Path(item['path'])
|
|
160
|
+
if path.parent!=self.root/'out'/event_id or path.is_symlink(): raise ConfigurationError('invalid stored output snapshot')
|
|
161
|
+
with path.open('rb') as stream: data=stream.read(MAX_FILE+1)
|
|
162
|
+
if len(data)!=item['size'] or hashlib.sha256(data).hexdigest()!=item['sha256']:
|
|
163
|
+
raise ConfigurationError('stored output snapshot corrupted')
|
|
164
|
+
return data
|
|
165
|
+
|
|
166
|
+
def save_input(self, event_id, item, data):
|
|
167
|
+
if len(data)!=item['size'] or hashlib.sha256(data).hexdigest()!=item['sha256']:
|
|
168
|
+
raise ConfigurationError('input attachment integrity mismatch')
|
|
169
|
+
self._check_name(item['name'])
|
|
170
|
+
suffix=Path(item['name']).suffix.lower()
|
|
171
|
+
if not __import__('re').fullmatch(r'\.[a-z0-9]{1,12}',suffix): suffix='.bin'
|
|
172
|
+
return self._write('in',event_id,item['attachment_id']+suffix,data)
|