@melaya/runner 1.1.60 → 1.1.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assistantFiles.d.ts +29 -0
- package/dist/assistantFiles.js +113 -0
- package/dist/assistantHost.py +333 -37
- package/dist/cli.js +7 -0
- package/dist/connection.js +236 -65
- package/dist/folderPicker.d.ts +15 -0
- package/dist/folderPicker.js +69 -0
- package/dist/localRelay.d.ts +8 -1
- package/dist/localRelay.js +21 -1
- package/dist/localVault.py +745 -0
- package/dist/localVaultMirror.py +507 -0
- package/dist/obsidianApp.d.ts +324 -0
- package/dist/obsidianApp.js +1143 -0
- package/dist/obsidianAppCli.d.ts +15 -0
- package/dist/obsidianAppCli.js +123 -0
- package/dist/sharedVendor.js +33 -1
- package/dist/vaultMirror.d.ts +104 -0
- package/dist/vaultMirror.js +305 -0
- package/dist/vaults.d.ts +350 -0
- package/dist/vaults.js +1130 -0
- package/localVault.py +745 -0
- package/localVaultMirror.py +507 -0
- package/package.json +4 -2
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
"""
|
|
2
|
+
localVaultMirror.py — keeps the cloud copy of a device vault up to date.
|
|
3
|
+
=======================================================================
|
|
4
|
+
docs/plans/obsidian-vault-connector.md §6.4 (S4); used by localVault.py
|
|
5
|
+
(``mirror_sync`` / ``mirror_drop``) when the vault's policy is
|
|
6
|
+
``storage: mirror``. Bulk data goes over authenticated HTTPS to the Vault API
|
|
7
|
+
(through the server's /api/v1/vault-api/mirror/<vault_id>/ proxy); the socket
|
|
8
|
+
only coordinates (src/vaultMirror.ts).
|
|
9
|
+
|
|
10
|
+
What changed is NOT detected here: the vault index (index.sqlite, built by the
|
|
11
|
+
engine from fs.watch batches plus the 10-minute stat-only reconciliation, which
|
|
12
|
+
hashes only stat-changed files) already holds ``(path, size, mtime, sha256)``
|
|
13
|
+
for every visible file, under the same path policy as S1. This module diffs
|
|
14
|
+
that manifest against its own ledger of what the copy will hold once every
|
|
15
|
+
queued op is published:
|
|
16
|
+
|
|
17
|
+
~/.melaya-runner/vaults/<vault_id>/mirror.sqlite
|
|
18
|
+
files path → sha256, size: the copy's state once the outbox is published
|
|
19
|
+
outbox ordered ops (seq, put | delete, path, sha256, size, mtime_ms): the
|
|
20
|
+
durable queue; a delete is a tombstone with its own sequence
|
|
21
|
+
number, so a retried upload can never resurrect the note
|
|
22
|
+
cfg stat cache of the allow-listed settings files (config folder)
|
|
23
|
+
meta next_seq, replica_id, journal_seq
|
|
24
|
+
|
|
25
|
+
One sync round: plan → begin → for each batch of ops: upload the contents the
|
|
26
|
+
copy lacks (resumable chunks with per-chunk and whole-file checksums), send the
|
|
27
|
+
ops → publish (with the device change journal since the last round). The
|
|
28
|
+
outbox is trimmed only after a publish is acknowledged, so an interrupted
|
|
29
|
+
round resumes where the copy's staging area stopped.
|
|
30
|
+
|
|
31
|
+
Never read into memory as a whole: files are streamed chunk by chunk. Never
|
|
32
|
+
logged: note content, paths, the capability.
|
|
33
|
+
"""
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import hashlib
|
|
37
|
+
import json
|
|
38
|
+
import os
|
|
39
|
+
import sqlite3
|
|
40
|
+
import stat as _stat
|
|
41
|
+
import time
|
|
42
|
+
import urllib.error
|
|
43
|
+
import urllib.parse
|
|
44
|
+
import urllib.request
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
from typing import Any, Callable, Iterable
|
|
47
|
+
|
|
48
|
+
CONFIG_FILES = (
|
|
49
|
+
"app.json", "daily-notes.json", "templates.json", "zk-prefixer.json", "bookmarks.json",
|
|
50
|
+
"types.json", "workspace.json", "workspaces.json", "core-plugins.json",
|
|
51
|
+
"community-plugins.json", "appearance.json",
|
|
52
|
+
)
|
|
53
|
+
MAX_FILE_BYTES = 512 * 1024 * 1024
|
|
54
|
+
OPS_BATCH = 500
|
|
55
|
+
JOURNAL_BATCH = 2000
|
|
56
|
+
FRESH_JOURNAL_BACKLOG = 2000
|
|
57
|
+
MAX_BUSY_RETRIES = 6
|
|
58
|
+
DEFAULT_CHUNK = 4 * 1024 * 1024
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class MirrorError(Exception):
|
|
62
|
+
"""``code``: auth | quota | retry | error (fixed, short)."""
|
|
63
|
+
|
|
64
|
+
def __init__(self, code: str, message: str = ""):
|
|
65
|
+
super().__init__(code)
|
|
66
|
+
self.code = code
|
|
67
|
+
self.message = message or code
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class FileChanged(Exception):
|
|
71
|
+
"""The file no longer matches the content queued for it: rescan, then retry."""
|
|
72
|
+
|
|
73
|
+
def __init__(self, path: str):
|
|
74
|
+
super().__init__(path)
|
|
75
|
+
self.path = path
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# ── transport ─────────────────────────────────────────────────────────────
|
|
79
|
+
|
|
80
|
+
Response = tuple[int, dict, dict]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class HttpTransport:
|
|
84
|
+
"""HTTPS to ``base`` (…/api/v1/vault-api/mirror/<vault_id>), ``Authorization:
|
|
85
|
+
Mirror <capability>``. Paths are relative: begin, ops, publish,
|
|
86
|
+
blobs/check, blobs/<sha256>."""
|
|
87
|
+
|
|
88
|
+
def __init__(self, base: str, token: str, timeout: float = 120.0):
|
|
89
|
+
self.base = base.rstrip("/")
|
|
90
|
+
self._token = token
|
|
91
|
+
self.timeout = timeout
|
|
92
|
+
|
|
93
|
+
def request(self, method: str, rel: str, *, json_body: Any = None, data: bytes | None = None,
|
|
94
|
+
params: dict | None = None, headers: dict | None = None) -> Response:
|
|
95
|
+
url = f"{self.base}/{rel}"
|
|
96
|
+
if params:
|
|
97
|
+
url += "?" + urllib.parse.urlencode(params)
|
|
98
|
+
h = {"Authorization": f"Mirror {self._token}", **(headers or {})}
|
|
99
|
+
body = data
|
|
100
|
+
if json_body is not None:
|
|
101
|
+
body = json.dumps(json_body).encode("utf-8")
|
|
102
|
+
h["Content-Type"] = "application/json"
|
|
103
|
+
elif data is not None:
|
|
104
|
+
h["Content-Type"] = "application/octet-stream"
|
|
105
|
+
req = urllib.request.Request(url, data=body, method=method, headers=h)
|
|
106
|
+
try:
|
|
107
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as r:
|
|
108
|
+
raw = r.read()
|
|
109
|
+
return r.status, _json(raw), {k.lower(): v for k, v in r.headers.items()}
|
|
110
|
+
except urllib.error.HTTPError as e:
|
|
111
|
+
raw = e.read() if hasattr(e, "read") else b""
|
|
112
|
+
return e.code, _json(raw), {k.lower(): v for k, v in (e.headers.items() if e.headers else [])}
|
|
113
|
+
except (urllib.error.URLError, OSError, TimeoutError) as e:
|
|
114
|
+
raise MirrorError("retry", f"network ({type(e).__name__})") from None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _json(raw: bytes) -> dict:
|
|
118
|
+
try:
|
|
119
|
+
v = json.loads(raw.decode("utf-8") or "{}")
|
|
120
|
+
return v if isinstance(v, dict) else {}
|
|
121
|
+
except Exception:
|
|
122
|
+
return {}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# ── ledger ────────────────────────────────────────────────────────────────
|
|
126
|
+
|
|
127
|
+
_SCHEMA = """
|
|
128
|
+
CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT);
|
|
129
|
+
CREATE TABLE IF NOT EXISTS files(path TEXT PRIMARY KEY, sha256 TEXT NOT NULL, size INTEGER NOT NULL);
|
|
130
|
+
CREATE TABLE IF NOT EXISTS outbox(seq INTEGER PRIMARY KEY, op TEXT NOT NULL, path TEXT NOT NULL,
|
|
131
|
+
sha256 TEXT, size INTEGER, mtime_ms INTEGER);
|
|
132
|
+
CREATE INDEX IF NOT EXISTS outbox_path ON outbox(path);
|
|
133
|
+
CREATE TABLE IF NOT EXISTS cfg(path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER, sha256 TEXT);
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class Ledger:
|
|
138
|
+
def __init__(self, path: str):
|
|
139
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
140
|
+
self.con = sqlite3.connect(path, isolation_level=None, check_same_thread=False, timeout=30)
|
|
141
|
+
self.con.execute("PRAGMA journal_mode=WAL")
|
|
142
|
+
self.con.execute("PRAGMA synchronous=FULL") # tombstones must survive a crash
|
|
143
|
+
self.con.executescript(_SCHEMA)
|
|
144
|
+
try:
|
|
145
|
+
os.chmod(path, 0o600)
|
|
146
|
+
except OSError:
|
|
147
|
+
pass
|
|
148
|
+
|
|
149
|
+
def close(self) -> None:
|
|
150
|
+
try:
|
|
151
|
+
self.con.close()
|
|
152
|
+
except sqlite3.Error:
|
|
153
|
+
pass
|
|
154
|
+
|
|
155
|
+
def get(self, key: str, default: str = "") -> str:
|
|
156
|
+
row = self.con.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone()
|
|
157
|
+
return row[0] if row else default
|
|
158
|
+
|
|
159
|
+
def set(self, key: str, value: Any) -> None:
|
|
160
|
+
self.con.execute("INSERT INTO meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
161
|
+
(key, str(value)))
|
|
162
|
+
|
|
163
|
+
def pending(self) -> int:
|
|
164
|
+
return int(self.con.execute("SELECT COUNT(*) FROM outbox").fetchone()[0])
|
|
165
|
+
|
|
166
|
+
def enqueue(self, ops: Iterable[tuple[str, str, str | None, int | None, int | None]]) -> int:
|
|
167
|
+
"""Append ops in order. An op supersedes every queued op of the same
|
|
168
|
+
path (each op states the whole file), so the queue stays one op per
|
|
169
|
+
path, and the sequence order of what remains is preserved."""
|
|
170
|
+
n = 0
|
|
171
|
+
self.con.execute("BEGIN IMMEDIATE")
|
|
172
|
+
try:
|
|
173
|
+
seq = int(self.get("next_seq", "1") or 1)
|
|
174
|
+
for op, path, sha, size, mtime_ms in ops:
|
|
175
|
+
self.con.execute("DELETE FROM outbox WHERE path=?", (path,))
|
|
176
|
+
self.con.execute("INSERT INTO outbox VALUES(?,?,?,?,?,?)", (seq, op, path, sha, size, mtime_ms))
|
|
177
|
+
if op == "delete":
|
|
178
|
+
self.con.execute("DELETE FROM files WHERE path=?", (path,))
|
|
179
|
+
else:
|
|
180
|
+
self.con.execute("INSERT OR REPLACE INTO files VALUES(?,?,?)", (path, sha, size))
|
|
181
|
+
seq += 1
|
|
182
|
+
n += 1
|
|
183
|
+
self.set("next_seq", seq)
|
|
184
|
+
self.con.execute("COMMIT")
|
|
185
|
+
except BaseException:
|
|
186
|
+
self.con.execute("ROLLBACK")
|
|
187
|
+
raise
|
|
188
|
+
return n
|
|
189
|
+
|
|
190
|
+
def ack(self, published_seq: int) -> None:
|
|
191
|
+
self.con.execute("DELETE FROM outbox WHERE seq<=?", (int(published_seq),))
|
|
192
|
+
|
|
193
|
+
def reset(self, replica_id: str) -> None:
|
|
194
|
+
"""The copy starts over (new copy, or it lost track of us): forget what
|
|
195
|
+
it was believed to hold; the next plan queues every file again."""
|
|
196
|
+
self.con.execute("BEGIN IMMEDIATE")
|
|
197
|
+
try:
|
|
198
|
+
self.con.execute("DELETE FROM files")
|
|
199
|
+
self.con.execute("DELETE FROM outbox")
|
|
200
|
+
self.con.execute("DELETE FROM meta WHERE key='journal_seq'")
|
|
201
|
+
self.set("replica_id", replica_id)
|
|
202
|
+
self.con.execute("COMMIT")
|
|
203
|
+
except BaseException:
|
|
204
|
+
self.con.execute("ROLLBACK")
|
|
205
|
+
raise
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# ── mirror ────────────────────────────────────────────────────────────────
|
|
209
|
+
|
|
210
|
+
class Mirror:
|
|
211
|
+
def __init__(self, vault_id: str, policy: Any, index_path: str, ledger_path: str, *,
|
|
212
|
+
max_file_bytes: int = MAX_FILE_BYTES, log: Callable[[str], None] | None = None):
|
|
213
|
+
self.vault_id = vault_id
|
|
214
|
+
self.policy = policy # engine PathPolicy: the S1 path rules
|
|
215
|
+
self.index_path = index_path
|
|
216
|
+
self.ledger = Ledger(ledger_path)
|
|
217
|
+
self.max_file_bytes = max_file_bytes
|
|
218
|
+
self.log = log or (lambda m: None)
|
|
219
|
+
self.skipped = 0
|
|
220
|
+
|
|
221
|
+
def close(self) -> None:
|
|
222
|
+
self.ledger.close()
|
|
223
|
+
|
|
224
|
+
# ── what the copy should hold ──
|
|
225
|
+
def _index(self) -> sqlite3.Connection:
|
|
226
|
+
uri = Path(self.index_path).resolve().as_uri() + "?mode=ro"
|
|
227
|
+
return sqlite3.connect(uri, uri=True, timeout=30)
|
|
228
|
+
|
|
229
|
+
def _visible(self, path: str) -> bool:
|
|
230
|
+
"""Defence in depth: the index already applies the path policy; a
|
|
231
|
+
stale row still never names an excluded path."""
|
|
232
|
+
try:
|
|
233
|
+
from_policy = self.policy.is_excluded(path)
|
|
234
|
+
except Exception:
|
|
235
|
+
return False
|
|
236
|
+
if from_policy:
|
|
237
|
+
return False
|
|
238
|
+
segs = path.split("/")
|
|
239
|
+
return not any(s in ("", ".", "..") or s.startswith(".") for s in segs)
|
|
240
|
+
|
|
241
|
+
def _config_files(self) -> dict[str, tuple[str, int, int]]:
|
|
242
|
+
"""Allow-listed settings files of the config folder (read-only on the
|
|
243
|
+
copy), hashed only when their stat moved."""
|
|
244
|
+
out: dict[str, tuple[str, int, int]] = {}
|
|
245
|
+
cfg_dir = self.policy.config_dir
|
|
246
|
+
for name in CONFIG_FILES:
|
|
247
|
+
absolute = self.policy.config_file(name)
|
|
248
|
+
if not absolute:
|
|
249
|
+
continue
|
|
250
|
+
try:
|
|
251
|
+
st = os.lstat(absolute)
|
|
252
|
+
except OSError:
|
|
253
|
+
continue
|
|
254
|
+
if not _stat.S_ISREG(st.st_mode) or st.st_size > 2 * 1024 * 1024:
|
|
255
|
+
continue
|
|
256
|
+
rel = f"{cfg_dir}/{name}"
|
|
257
|
+
row = self.ledger.con.execute("SELECT size, mtime_ns, sha256 FROM cfg WHERE path=?", (rel,)).fetchone()
|
|
258
|
+
if row and row[0] == st.st_size and row[1] == st.st_mtime_ns:
|
|
259
|
+
sha = row[2]
|
|
260
|
+
else:
|
|
261
|
+
try:
|
|
262
|
+
with open(absolute, "rb") as f:
|
|
263
|
+
sha = hashlib.sha256(f.read()).hexdigest()
|
|
264
|
+
except OSError:
|
|
265
|
+
continue
|
|
266
|
+
self.ledger.con.execute("INSERT OR REPLACE INTO cfg VALUES(?,?,?,?)", (rel, st.st_size, st.st_mtime_ns, sha))
|
|
267
|
+
out[rel] = (sha, st.st_size, st.st_mtime_ns // 1_000_000)
|
|
268
|
+
return out
|
|
269
|
+
|
|
270
|
+
def plan(self) -> dict:
|
|
271
|
+
"""Queue the ops that take the copy from the ledger's state to the
|
|
272
|
+
index's: deletes first (a file may replace a folder of the same name),
|
|
273
|
+
then puts. Metadata only; no file is read except changed settings."""
|
|
274
|
+
desired: dict[str, tuple[str, int, int]] = {}
|
|
275
|
+
skipped = 0
|
|
276
|
+
con = self._index()
|
|
277
|
+
try:
|
|
278
|
+
for path, size, mtime_ns, sha in con.execute("SELECT path, size, mtime_ns, sha256 FROM manifest"):
|
|
279
|
+
if not sha or not isinstance(path, str) or not self._visible(path):
|
|
280
|
+
continue
|
|
281
|
+
if int(size or 0) > self.max_file_bytes:
|
|
282
|
+
skipped += 1
|
|
283
|
+
continue
|
|
284
|
+
desired[path] = (sha, int(size or 0), int(mtime_ns or 0) // 1_000_000)
|
|
285
|
+
finally:
|
|
286
|
+
con.close()
|
|
287
|
+
desired.update(self._config_files())
|
|
288
|
+
known = {p: (s, z) for p, s, z in self.ledger.con.execute("SELECT path, sha256, size FROM files")}
|
|
289
|
+
deletes = sorted(p for p in known if p not in desired)
|
|
290
|
+
puts = sorted(p for p, v in desired.items() if known.get(p) != (v[0], v[1]))
|
|
291
|
+
ops = [("delete", p, None, None, None) for p in deletes]
|
|
292
|
+
ops += [("put", p, desired[p][0], desired[p][1], desired[p][2]) for p in puts]
|
|
293
|
+
n = self.ledger.enqueue(ops) if ops else 0
|
|
294
|
+
self.skipped = skipped
|
|
295
|
+
return {"enqueued": n, "pending": self.ledger.pending(), "skipped": skipped}
|
|
296
|
+
|
|
297
|
+
def _device_state(self) -> tuple[int, int, int]:
|
|
298
|
+
"""(index_rev, journal head, journal floor) of the device index."""
|
|
299
|
+
con = self._index()
|
|
300
|
+
try:
|
|
301
|
+
meta = dict(con.execute("SELECT key, value FROM meta WHERE key IN ('index_rev','journal_floor')").fetchall())
|
|
302
|
+
head = con.execute("SELECT MAX(seq) FROM journal").fetchone()[0] or 0
|
|
303
|
+
return int(meta.get("index_rev") or 0), int(head), int(meta.get("journal_floor") or 0)
|
|
304
|
+
finally:
|
|
305
|
+
con.close()
|
|
306
|
+
|
|
307
|
+
def _journal(self, since: int) -> list[dict]:
|
|
308
|
+
con = self._index()
|
|
309
|
+
try:
|
|
310
|
+
rows = con.execute("SELECT seq, event_id, op, path, old_path, revision, origin, write_id, ts FROM journal "
|
|
311
|
+
"WHERE seq > ? ORDER BY seq LIMIT ?", (since, JOURNAL_BATCH)).fetchall()
|
|
312
|
+
finally:
|
|
313
|
+
con.close()
|
|
314
|
+
keys = ("seq", "event_id", "op", "path", "old_path", "revision", "origin", "write_id", "ts")
|
|
315
|
+
return [dict(zip(keys, r)) for r in rows]
|
|
316
|
+
|
|
317
|
+
# ── one sync round ──
|
|
318
|
+
def sync(self, t: Any, *, budget_s: float = 600.0, sleep: Callable[[float], None] = time.sleep) -> dict:
|
|
319
|
+
deadline = time.monotonic() + max(5.0, budget_s)
|
|
320
|
+
try:
|
|
321
|
+
return self._sync(t, deadline, sleep)
|
|
322
|
+
except MirrorError as e:
|
|
323
|
+
return {"state": e.code, "error": e.message[:300], "pending": self.ledger.pending()}
|
|
324
|
+
except FileChanged as e:
|
|
325
|
+
return {"state": "retry", "rescan": [e.path], "pending": self.ledger.pending()}
|
|
326
|
+
|
|
327
|
+
def _call(self, t: Any, method: str, rel: str, sleep: Callable[[float], None], **kw: Any) -> Response:
|
|
328
|
+
"""One request with backpressure: 429 / 503 wait for Retry-After."""
|
|
329
|
+
for attempt in range(MAX_BUSY_RETRIES):
|
|
330
|
+
status, body, headers = t.request(method, rel, **kw)
|
|
331
|
+
if status in (429, 503):
|
|
332
|
+
try:
|
|
333
|
+
wait = float(headers.get("retry-after") or 2)
|
|
334
|
+
except ValueError:
|
|
335
|
+
wait = 2.0
|
|
336
|
+
sleep(min(30.0, max(0.5, wait)) * (1 + attempt * 0.5))
|
|
337
|
+
continue
|
|
338
|
+
if status in (401, 403):
|
|
339
|
+
raise MirrorError("auth", str(body.get("message") or body.get("error") or status))
|
|
340
|
+
if status in (507, 413) and body.get("error") in ("quota", "too_large"):
|
|
341
|
+
raise MirrorError("quota", str(body.get("message") or "quota"))
|
|
342
|
+
if status >= 500:
|
|
343
|
+
raise MirrorError("retry", f"server {status}")
|
|
344
|
+
return status, body, headers
|
|
345
|
+
raise MirrorError("retry", "the Vault API is busy")
|
|
346
|
+
|
|
347
|
+
def _sync(self, t: Any, deadline: float, sleep: Callable[[float], None]) -> dict:
|
|
348
|
+
self.plan()
|
|
349
|
+
index_rev, head, floor = self._device_state()
|
|
350
|
+
journal_seq = int(self.ledger.get("journal_seq", "-1") or -1)
|
|
351
|
+
replica_id = self.ledger.get("replica_id")
|
|
352
|
+
if replica_id and journal_seq >= 0 and not self.ledger.pending() and journal_seq >= head \
|
|
353
|
+
and self.ledger.get("published_index_rev") == str(index_rev):
|
|
354
|
+
return {"state": "ready", "pending": 0, "noop": True, "skipped": self.skipped}
|
|
355
|
+
|
|
356
|
+
status, b, _ = self._call(t, "POST", "begin", sleep, json_body={"replica_id": replica_id or None})
|
|
357
|
+
if status != 200:
|
|
358
|
+
raise MirrorError("error", f"begin {status}")
|
|
359
|
+
if b.get("reset") or b.get("replica_id") != replica_id or int(b.get("staged_seq") or 0) >= int(self.ledger.get("next_seq", "1") or 1):
|
|
360
|
+
if not b.get("reset"):
|
|
361
|
+
status, b, _ = self._call(t, "POST", "begin", sleep, json_body={"replica_id": None, "reset": True})
|
|
362
|
+
if status != 200:
|
|
363
|
+
raise MirrorError("error", f"begin {status}")
|
|
364
|
+
self.ledger.reset(str(b.get("replica_id") or ""))
|
|
365
|
+
self.plan()
|
|
366
|
+
journal_seq = -1
|
|
367
|
+
if journal_seq < 0: # a new copy: its journal starts with recent history only
|
|
368
|
+
journal_seq = max(floor, head - FRESH_JOURNAL_BACKLOG, 0)
|
|
369
|
+
journal_seq = max(journal_seq, floor)
|
|
370
|
+
staged = int(b.get("staged_seq") or 0)
|
|
371
|
+
chunk = max(64 * 1024, min(int(b.get("chunk_size") or DEFAULT_CHUNK), int(b.get("max_chunk") or DEFAULT_CHUNK)))
|
|
372
|
+
|
|
373
|
+
while True:
|
|
374
|
+
rows = self.ledger.con.execute(
|
|
375
|
+
"SELECT seq, op, path, sha256, size, mtime_ms FROM outbox WHERE seq > ? ORDER BY seq LIMIT ?",
|
|
376
|
+
(staged, OPS_BATCH)).fetchall()
|
|
377
|
+
if not rows:
|
|
378
|
+
break
|
|
379
|
+
ops = [{"seq": r[0], "op": r[1], "path": r[2], **({"sha256": r[3], "size": r[4], "mtime_ms": r[5]}
|
|
380
|
+
if r[1] == "put" else {})} for r in rows]
|
|
381
|
+
self._upload_missing(t, ops, chunk, sleep)
|
|
382
|
+
for attempt in range(3):
|
|
383
|
+
status, body, _ = self._call(t, "POST", "ops", sleep, json_body={"ops": ops})
|
|
384
|
+
if status == 409 and body.get("error") == "missing_blobs":
|
|
385
|
+
self._upload_missing(t, ops, chunk, sleep, only=set(body.get("missing") or []))
|
|
386
|
+
continue
|
|
387
|
+
break
|
|
388
|
+
if status != 200:
|
|
389
|
+
raise MirrorError("error", f"ops {status} {body.get('error') or ''}".strip())
|
|
390
|
+
staged = int(body.get("staged_seq") or staged)
|
|
391
|
+
if time.monotonic() > deadline:
|
|
392
|
+
return {"state": "partial", "pending": self.ledger.pending(), "stagedSeq": staged}
|
|
393
|
+
|
|
394
|
+
events = self._journal(journal_seq)
|
|
395
|
+
status, p, _ = self._call(t, "POST", "publish", sleep, json_body={
|
|
396
|
+
"upto_seq": staged, "device_index_rev": index_rev,
|
|
397
|
+
"journal": [{k: v for k, v in e.items() if k != "seq"} for e in events],
|
|
398
|
+
})
|
|
399
|
+
if status == 409 and p.get("error") == "staged_seq":
|
|
400
|
+
return {"state": "retry", "pending": self.ledger.pending()}
|
|
401
|
+
if status != 200:
|
|
402
|
+
raise MirrorError("error", f"publish {status}")
|
|
403
|
+
self.ledger.ack(int(p.get("published_seq") or staged))
|
|
404
|
+
if events:
|
|
405
|
+
self.ledger.set("journal_seq", events[-1]["seq"])
|
|
406
|
+
else:
|
|
407
|
+
self.ledger.set("journal_seq", journal_seq)
|
|
408
|
+
self.ledger.set("published_index_rev", index_rev)
|
|
409
|
+
more = len(events) >= JOURNAL_BATCH or self.ledger.pending() > 0
|
|
410
|
+
return {
|
|
411
|
+
"state": "partial" if more else "ready",
|
|
412
|
+
"pending": self.ledger.pending(),
|
|
413
|
+
"publishedSeq": int(p.get("published_seq") or staged),
|
|
414
|
+
"replicaRev": int(p.get("replica_rev") or 0),
|
|
415
|
+
"notes": int(p.get("notes") or 0),
|
|
416
|
+
"bytes": int(p.get("bytes") or 0),
|
|
417
|
+
"skipped": self.skipped,
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
# ── contents ──
|
|
421
|
+
def _upload_missing(self, t: Any, ops: list[dict], chunk: int, sleep: Callable[[float], None],
|
|
422
|
+
only: set[str] | None = None) -> None:
|
|
423
|
+
puts: dict[str, dict] = {}
|
|
424
|
+
for o in ops:
|
|
425
|
+
if o["op"] == "put" and (only is None or o["sha256"] in only):
|
|
426
|
+
puts.setdefault(o["sha256"], o)
|
|
427
|
+
if not puts:
|
|
428
|
+
return
|
|
429
|
+
shas = list(puts)
|
|
430
|
+
missing: list[str] = []
|
|
431
|
+
partial: dict[str, int] = {}
|
|
432
|
+
for i in range(0, len(shas), 1000):
|
|
433
|
+
status, body, _ = self._call(t, "POST", "blobs/check", sleep, json_body={"sha256": shas[i:i + 1000]})
|
|
434
|
+
if status != 200:
|
|
435
|
+
raise MirrorError("error", f"check {status}")
|
|
436
|
+
missing += [s for s in body.get("missing") or [] if s in puts]
|
|
437
|
+
partial.update({k: int(v) for k, v in (body.get("partial") or {}).items() if k in puts})
|
|
438
|
+
for sha in missing:
|
|
439
|
+
self._upload(t, puts[sha], int(partial.get(sha, 0)), chunk, sleep)
|
|
440
|
+
|
|
441
|
+
def _open(self, path: str):
|
|
442
|
+
"""Open through the path policy: no links, no excluded path, and the
|
|
443
|
+
handle must be the file lstat saw (a swap between check and open fails)."""
|
|
444
|
+
segs = path.split("/")
|
|
445
|
+
if len(segs) == 2 and segs[0] == self.policy.config_dir and segs[1] in CONFIG_FILES:
|
|
446
|
+
absolute = self.policy.config_file(segs[1])
|
|
447
|
+
if not absolute:
|
|
448
|
+
raise FileChanged(path)
|
|
449
|
+
else:
|
|
450
|
+
if not self._visible(path):
|
|
451
|
+
raise FileChanged(path)
|
|
452
|
+
try:
|
|
453
|
+
_norm, absolute = self.policy.check(path, allow_missing=False)
|
|
454
|
+
except Exception:
|
|
455
|
+
raise FileChanged(path) from None
|
|
456
|
+
try:
|
|
457
|
+
st = os.lstat(absolute)
|
|
458
|
+
if not _stat.S_ISREG(st.st_mode):
|
|
459
|
+
raise FileChanged(path)
|
|
460
|
+
fd = os.open(absolute, os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0))
|
|
461
|
+
except OSError:
|
|
462
|
+
raise FileChanged(path) from None
|
|
463
|
+
fst = os.fstat(fd)
|
|
464
|
+
if (fst.st_ino, fst.st_dev) != (st.st_ino, st.st_dev) and fst.st_ino and st.st_ino:
|
|
465
|
+
os.close(fd)
|
|
466
|
+
raise FileChanged(path)
|
|
467
|
+
return os.fdopen(fd, "rb")
|
|
468
|
+
|
|
469
|
+
def _upload(self, t: Any, op: dict, offset: int, chunk: int, sleep: Callable[[float], None], depth: int = 0) -> None:
|
|
470
|
+
if depth > 3:
|
|
471
|
+
raise MirrorError("retry", "upload did not converge")
|
|
472
|
+
sha, size = op["sha256"], int(op["size"])
|
|
473
|
+
with self._open(op["path"]) as f:
|
|
474
|
+
h = hashlib.sha256()
|
|
475
|
+
pos = 0
|
|
476
|
+
while pos < offset: # resume: the prefix is hashed locally, not resent
|
|
477
|
+
block = f.read(min(1 << 20, offset - pos))
|
|
478
|
+
if not block:
|
|
479
|
+
raise FileChanged(op["path"])
|
|
480
|
+
h.update(block)
|
|
481
|
+
pos += len(block)
|
|
482
|
+
while True:
|
|
483
|
+
data = f.read(min(chunk, size - pos)) if pos < size else b""
|
|
484
|
+
if pos < size and not data:
|
|
485
|
+
raise FileChanged(op["path"])
|
|
486
|
+
h.update(data)
|
|
487
|
+
last = pos + len(data) >= size
|
|
488
|
+
if last and (h.hexdigest() != sha or f.read(1)):
|
|
489
|
+
raise FileChanged(op["path"]) # never send content that is not the queued one
|
|
490
|
+
for attempt in range(2):
|
|
491
|
+
status, body, _ = self._call(t, "PUT", f"blobs/{sha}", sleep, data=data,
|
|
492
|
+
params={"offset": pos, "total": size},
|
|
493
|
+
headers={"x-mel-chunk-sha256": hashlib.sha256(data).hexdigest()})
|
|
494
|
+
if status == 422 and body.get("error") == "chunk_checksum" and attempt == 0:
|
|
495
|
+
continue
|
|
496
|
+
break
|
|
497
|
+
if status == 409 and body.get("error") == "offset":
|
|
498
|
+
return self._upload(t, op, int(body.get("offset") or 0), chunk, sleep, depth + 1)
|
|
499
|
+
if status == 422 and body.get("error") == "checksum_mismatch":
|
|
500
|
+
raise FileChanged(op["path"])
|
|
501
|
+
if status != 200:
|
|
502
|
+
raise MirrorError("error", f"upload {status} {body.get('error') or ''}".strip())
|
|
503
|
+
if body.get("complete"):
|
|
504
|
+
return None
|
|
505
|
+
pos += len(data)
|
|
506
|
+
if last:
|
|
507
|
+
raise MirrorError("error", "upload incomplete")
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@melaya/runner",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.62",
|
|
4
4
|
"description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Melaya Labs LLC",
|
|
@@ -18,12 +18,14 @@
|
|
|
18
18
|
"localRagRetrieve.py",
|
|
19
19
|
"localDbProbe.py",
|
|
20
20
|
"localBacklinkFetch.py",
|
|
21
|
+
"localVault.py",
|
|
22
|
+
"localVaultMirror.py",
|
|
21
23
|
"nltk_data/**",
|
|
22
24
|
"README.md",
|
|
23
25
|
"LICENSE"
|
|
24
26
|
],
|
|
25
27
|
"scripts": {
|
|
26
|
-
"build": "tsc && node -e \"const fs=require('fs'); fs.copyFileSync('localRagIngest.py','dist/localRagIngest.py'); fs.copyFileSync('localRagRetrieve.py','dist/localRagRetrieve.py'); fs.copyFileSync('localDbProbe.py','dist/localDbProbe.py'); fs.copyFileSync('localBacklinkFetch.py','dist/localBacklinkFetch.py'); fs.copyFileSync('src/assistantHost.py','dist/assistantHost.py')\"",
|
|
28
|
+
"build": "tsc && node -e \"const fs=require('fs'); fs.copyFileSync('localRagIngest.py','dist/localRagIngest.py'); fs.copyFileSync('localRagRetrieve.py','dist/localRagRetrieve.py'); fs.copyFileSync('localDbProbe.py','dist/localDbProbe.py'); fs.copyFileSync('localBacklinkFetch.py','dist/localBacklinkFetch.py'); fs.copyFileSync('localVault.py','dist/localVault.py'); fs.copyFileSync('localVaultMirror.py','dist/localVaultMirror.py'); fs.copyFileSync('src/assistantHost.py','dist/assistantHost.py')\"",
|
|
27
29
|
"test": "node --test --import tsx \"src/**/*.test.ts\"",
|
|
28
30
|
"prepublishOnly": "npm run build"
|
|
29
31
|
},
|