@r1z4x/tezgah 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1268 @@
1
+ #!/usr/bin/env python3
2
+ """SQLite runtime stores (ADR 021): the connection, the schema, and the import
3
+ of the JSON and JSONL files a store replaces. No caller opens a store file
4
+ itself.
5
+
6
+ It holds the taste stores, the evidence ledger and the cache stores:
7
+ - `<repo>/.tezgah/taste/taste.db`: the capture signals, the typed decisions,
8
+ the defects, the calibration labels, the injection rows, the benefit gate's
9
+ state, the repository's learnings with their meta, and the learnings each
10
+ session's write notes already showed.
11
+ - `~/.config/tezgah/taste/taste.db`: the user-scope learnings with their meta.
12
+ - `<cache>/tezgah.db`: the evidence ledger, one `evidence` row per ledger line
13
+ of every session (`tezgah_integrity` writes and reads it, through the
14
+ functions under "the cache database" below, and so does opencode's plugin,
15
+ through node:sqlite or the `evidence` CLI in `main`), and the small stores
16
+ the hooks kept as files beside it before (CACHE_LEGACY): the used-tool
17
+ marks, the turn stamps, the switch baselines, the status marks, the judge's
18
+ last use and down marks, the skill hints, the update check, the lesson-taint
19
+ index, Cursor's marks, the replay and snapshot indexes and the taste-learn
20
+ stamps (the functions under "the cache stores").
21
+
22
+ A row keeps its JSON payload in `row`, so its fields stay what the old files
23
+ held; the columns beside it are the fields a query filters on. WAL and a busy
24
+ timeout let a hook append while the CLI reads. The directory is made 0700 and
25
+ the database file is created 0600 before SQLite opens it; SQLite gives its
26
+ `-wal` and `-shm` files the database's mode.
27
+ """
28
+ import contextlib
29
+ import hashlib
30
+ import json
31
+ import os
32
+ import sqlite3
33
+ import subprocess
34
+ import sys
35
+ import time
36
+
37
+ try:
38
+ import fcntl
39
+ except ImportError: # Windows: the old writers took no lock there either
40
+ fcntl = None
41
+
42
+ BUSY_MS = 5000
43
+ TASTE_DB = "taste.db"
44
+ ERRORS = (OSError, sqlite3.Error)
45
+ # how often the WAL switch is retried while another connection holds the file
46
+ # (`_wal`)
47
+ LOCK_POLL = 0.02
48
+
49
+ # Each append-only taste table and the columns a query reads beside its JSON
50
+ # row; `n` is the append order.
51
+ TASTE_ROWS = {
52
+ "signals": ("session", "kind", "ts"),
53
+ "decisions": ("id", "session", "provider", "day", "at"),
54
+ "defects": ("id", "session", "day"),
55
+ "labels": ("id", "provider", "day"),
56
+ "injected": ("session", "day"),
57
+ }
58
+ IMPORT_SCHEMA = """
59
+ CREATE TABLE IF NOT EXISTS imported (name TEXT NOT NULL, dev INTEGER NOT NULL,
60
+ ino INTEGER NOT NULL, bytes INTEGER NOT NULL,
61
+ PRIMARY KEY (name, dev, ino));
62
+ CREATE TABLE IF NOT EXISTS import_failed (name TEXT PRIMARY KEY, mtime_ns INTEGER NOT NULL,
63
+ size INTEGER NOT NULL);
64
+ """
65
+ USER_SCHEMA = """
66
+ CREATE TABLE IF NOT EXISTS learnings (id TEXT PRIMARY KEY, row TEXT NOT NULL);
67
+ CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
68
+ """ + IMPORT_SCHEMA
69
+ REPO_SCHEMA = USER_SCHEMA + "".join(
70
+ "CREATE TABLE IF NOT EXISTS %s (n INTEGER PRIMARY KEY, %s, row TEXT NOT NULL);\n"
71
+ % (table, ", ".join(cols)) for table, cols in TASTE_ROWS.items()) + """
72
+ CREATE TABLE IF NOT EXISTS gate (one INTEGER PRIMARY KEY CHECK (one = 1),
73
+ stopped INTEGER NOT NULL, report TEXT NOT NULL);
74
+ CREATE TABLE IF NOT EXISTS notes_seen (session TEXT NOT NULL, learning TEXT NOT NULL,
75
+ PRIMARY KEY (session, learning));
76
+ """
77
+ # The files each taste store replaces. A JSON document imports only into the
78
+ # tables it names while they are empty; a row file (`<table>.jsonl`) always.
79
+ DOC_TABLES = {"ledger.json": ("learnings", "meta"), "gate.json": ("gate",)}
80
+ REPO_LEGACY = ("ledger.json", "gate.json") + tuple(t + ".jsonl" for t in TASTE_ROWS)
81
+ USER_LEGACY = ("ledger.json",)
82
+
83
+
84
+ def connect(path, schema, version=0):
85
+ """An autocommit connection to the database at `path` with `schema`
86
+ applied (idempotent). With `version`, the schema runs only while the file's
87
+ `user_version` is below it, so a hook's open skips the DDL and an older
88
+ install leaves a newer schema alone. Raises OSError or
89
+ sqlite3.Error."""
90
+ os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
91
+ os.close(os.open(path, os.O_RDWR | os.O_CREAT, 0o600))
92
+ conn = sqlite3.connect(path, isolation_level=None)
93
+ try:
94
+ conn.execute("PRAGMA busy_timeout = %d" % BUSY_MS)
95
+ _wal(conn)
96
+ if not version or conn.execute("PRAGMA user_version").fetchone()[0] < version:
97
+ # the write lock first, so a second opener waits on the busy
98
+ # timeout instead of failing on SQLite's lock-upgrade deadlock
99
+ conn.executescript("BEGIN IMMEDIATE;\n%s\nPRAGMA user_version = %d;\nCOMMIT;"
100
+ % (schema, version))
101
+ except sqlite3.Error:
102
+ if conn.in_transaction:
103
+ conn.execute("ROLLBACK")
104
+ conn.close()
105
+ raise
106
+ return conn
107
+
108
+
109
+ def _wal(conn):
110
+ """Switch `conn`'s file to WAL. The switch needs the file to itself and
111
+ SQLite answers "locked" at once instead of waiting, so it is retried for as
112
+ long as the busy timeout would have waited."""
113
+ deadline = time.monotonic() + BUSY_MS / 1000.0
114
+ while True:
115
+ try:
116
+ conn.execute("PRAGMA journal_mode = WAL")
117
+ return
118
+ except sqlite3.OperationalError:
119
+ if time.monotonic() >= deadline:
120
+ raise
121
+ time.sleep(LOCK_POLL)
122
+
123
+
124
+ @contextlib.contextmanager
125
+ def transaction(conn):
126
+ """One write transaction, taken at its start so two writers queue on the
127
+ busy timeout instead of failing at commit; rolled back on any failure,
128
+ the commit's own included."""
129
+ conn.execute("BEGIN IMMEDIATE")
130
+ try:
131
+ yield conn
132
+ conn.execute("COMMIT")
133
+ except BaseException:
134
+ if conn.in_transaction:
135
+ conn.execute("ROLLBACK")
136
+ raise
137
+
138
+
139
+ @contextlib.contextmanager
140
+ def taste(dirpath, user=False, create=True):
141
+ """The taste database in `dirpath` (the user store when `user`), the legacy
142
+ files there imported first (`_import`). With `create` False it yields None
143
+ instead of creating a database where neither it nor a legacy file exists,
144
+ so a read leaves no file behind."""
145
+ path = os.path.join(dirpath, TASTE_DB)
146
+ legacy = USER_LEGACY if user else REPO_LEGACY
147
+ if not create and not os.path.exists(path) and not any(
148
+ os.path.exists(os.path.join(dirpath, name)) for name in legacy):
149
+ yield None
150
+ return
151
+ conn = connect(path, USER_SCHEMA if user else REPO_SCHEMA)
152
+ try:
153
+ _import(conn, dirpath, legacy)
154
+ yield conn
155
+ finally:
156
+ conn.close()
157
+
158
+
159
+ def _import(conn, dirpath, names):
160
+ """Bring the legacy files present in `dirpath` into their tables. With no
161
+ legacy file, an open costs one stat per name.
162
+
163
+ A row file (`<table>.jsonl`) imports on every open that finds it, under
164
+ its old writer's flock. The `imported` table records how many bytes of
165
+ that file (by device and inode) went in, in the same transaction as the
166
+ rows, so the next open takes only what was appended since; a torn last
167
+ line waits for its newline. A JSON document imports only while its tables
168
+ are empty. The rename aside (`_aside`) comes after the commit and is
169
+ tidiness: a process killed before it re-imports nothing. A file that
170
+ cannot be read or parsed is recorded in `import_failed` and skipped,
171
+ before any transaction, until its mtime or size changes."""
172
+ for name in names:
173
+ src = os.path.join(dirpath, name)
174
+ if not os.path.exists(src):
175
+ continue
176
+ try:
177
+ st = os.stat(src)
178
+ except OSError:
179
+ continue
180
+ if conn.execute("SELECT 1 FROM import_failed WHERE name = ? AND mtime_ns = ? "
181
+ "AND size = ?", (name, st.st_mtime_ns, st.st_size)).fetchone():
182
+ continue
183
+ if name in DOC_TABLES:
184
+ _import_doc(conn, name, src, st)
185
+ else:
186
+ _import_rows(conn, name, src, st)
187
+
188
+
189
+ def _failed(conn, name, st):
190
+ with transaction(conn):
191
+ conn.execute("INSERT OR REPLACE INTO import_failed (name, mtime_ns, size) "
192
+ "VALUES (?, ?, ?)", (name, st.st_mtime_ns, st.st_size))
193
+
194
+
195
+ def _wanted(conn, name):
196
+ """False for a JSON document whose tables already hold rows."""
197
+ return all(conn.execute("SELECT 1 FROM %s LIMIT 1" % table).fetchone() is None
198
+ for table in DOC_TABLES.get(name, ()))
199
+
200
+
201
+ def _aside(src, read):
202
+ """Rename `src` to `<src>.imported`, or `.imported.N` beside an earlier one,
203
+ when it is still the file the import read (`read`, its fstat): one another
204
+ writer created at that path since was never read and stays. Never over a
205
+ file, so no import's source is ever lost. A failed rename is left for the
206
+ next open, which finds nothing new in the file."""
207
+ try:
208
+ if not os.path.samestat(os.stat(src), read):
209
+ return
210
+ except OSError:
211
+ return
212
+ dst, n = src + ".imported", 0
213
+ while os.path.exists(dst):
214
+ n += 1
215
+ dst = "%s.imported.%d" % (src, n)
216
+ try:
217
+ os.rename(src, dst)
218
+ except OSError:
219
+ pass
220
+
221
+
222
+ def _import_doc(conn, name, src, st):
223
+ if not _wanted(conn, name):
224
+ return
225
+ try:
226
+ with open(src, "rb") as fh:
227
+ read = os.fstat(fh.fileno())
228
+ data = json.loads(fh.read().decode("utf-8"))
229
+ except FileNotFoundError:
230
+ return # another process imported it since the stat
231
+ except (OSError, ValueError): # UnicodeDecodeError is a ValueError
232
+ data = None
233
+ if not isinstance(data, dict) or (
234
+ name == "ledger.json" and not isinstance(data.get("learnings"), dict)):
235
+ _failed(conn, name, st)
236
+ return
237
+ with transaction(conn):
238
+ if not _wanted(conn, name):
239
+ return # another process imported it first
240
+ if name == "ledger.json":
241
+ _write_ledger(conn, data)
242
+ else:
243
+ set_gate(conn, data.get("stopped"), data.get("report") or {})
244
+ _aside(src, read)
245
+
246
+
247
+ def _locked(fh):
248
+ """An exclusive flock on `fh` if one is free now, True where flock is
249
+ missing. Never waits: the caller holds the database's write lock."""
250
+ if fcntl is None:
251
+ return True
252
+ try:
253
+ fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
254
+ return True
255
+ except OSError:
256
+ return False
257
+
258
+
259
+ def _taste_put(table):
260
+ """The import of one taste row line: a line that is not a UTF-8 JSON object
261
+ is skipped, as the old reader skipped it."""
262
+ def put(conn, line):
263
+ try:
264
+ row = json.loads(line.decode("utf-8"))
265
+ except ValueError: # UnicodeDecodeError is one
266
+ return
267
+ if isinstance(row, dict):
268
+ append(conn, table, row)
269
+ return put
270
+
271
+
272
+ def _offset(conn, name, fh):
273
+ """(the file's key, the byte offset its import resumes at, whether that
274
+ differs from the recorded one). A file cut below what went in (rewritten in
275
+ place) resumes at its end, so the rows appended after the cut still
276
+ import."""
277
+ st = os.fstat(fh.fileno())
278
+ key = (name, st.st_dev, st.st_ino)
279
+ found = conn.execute("SELECT bytes FROM imported WHERE name = ? AND dev = ? "
280
+ "AND ino = ?", key).fetchone()
281
+ start = min(found[0], st.st_size) if found else 0
282
+ return key, start, bool(found and found[0] != start)
283
+
284
+
285
+ def _import_rows(conn, name, src, st, put=None, keep=False):
286
+ """The complete lines of `src` past the recorded offset, each handed to
287
+ `put` (a taste table's row by default). With `keep` the file is never
288
+ renamed aside: its writer may still append to it.
289
+
290
+ A peek without a transaction comes first: a file with no complete line
291
+ past the offset (nothing new, or a torn fragment its writer has not
292
+ finished) costs no write lock, so a read under a busy writer still answers.
293
+ Then the database's write lock, and the file's flock only if it is free: an
294
+ import that held the flock while it waited on a busy database would hold
295
+ the file's writer for that wait. The offset is read again inside the
296
+ transaction, so two importers cannot both take the same lines, with or
297
+ without a flock."""
298
+ try:
299
+ fh = open(src, "rb")
300
+ except FileNotFoundError:
301
+ return # another process imported it since the stat
302
+ except OSError:
303
+ _failed(conn, name, st)
304
+ return
305
+ with fh: # closing it releases the flock
306
+ key, start, cut = _offset(conn, name, fh)
307
+ fh.seek(start)
308
+ raw = fh.read()
309
+ if b"\n" in raw or cut:
310
+ with transaction(conn):
311
+ if not _locked(fh):
312
+ return # its writer holds it: the next open takes it
313
+ key, start, cut = _offset(conn, name, fh)
314
+ fh.seek(start)
315
+ raw = fh.read()
316
+ whole = raw[:raw.rfind(b"\n") + 1]
317
+ if whole or cut:
318
+ put = put or _taste_put(name[:-len(".jsonl")])
319
+ for line in whole.split(b"\n"):
320
+ put(conn, line)
321
+ conn.execute("INSERT OR REPLACE INTO imported (name, dev, ino, bytes) "
322
+ "VALUES (?, ?, ?, ?)", key + (start + len(whole),))
323
+ # aside only when every byte went in (no fragment past the last newline)
324
+ # and no writer appended past the read (the old writer appends unlocked
325
+ # once its own wait runs out), under the flock so no writer is inside
326
+ # an append
327
+ if not keep and not raw[raw.rfind(b"\n") + 1:] and _locked(fh):
328
+ now = os.fstat(fh.fileno())
329
+ if now.st_size == start + len(raw):
330
+ _aside(src, now)
331
+
332
+
333
+ # --- the taste tables ---------------------------------------------------------------
334
+
335
+ def _column(value):
336
+ return value if value is None or isinstance(value, (str, int, float)) else None
337
+
338
+
339
+ def append(conn, table, row):
340
+ """One row into an append-only taste table; raises when it is not written."""
341
+ cols = TASTE_ROWS[table]
342
+ conn.execute("INSERT INTO %s (%s, row) VALUES (%s)" % (
343
+ table, ", ".join(cols), ", ".join("?" * (len(cols) + 1))),
344
+ [_column(row.get(c)) for c in cols] + [json.dumps(row, ensure_ascii=False)])
345
+
346
+
347
+ def rows(conn, table, where="", args=()):
348
+ """The rows of one append-only taste table in append order; `where` is a
349
+ clause over its columns."""
350
+ return [json.loads(text) for (text,) in conn.execute(
351
+ "SELECT row FROM %s %s ORDER BY n" % (table, where), args)]
352
+
353
+
354
+ def last(conn, table):
355
+ """The append number of the table's newest row, 0 when it has none."""
356
+ return conn.execute("SELECT coalesce(max(n), 0) FROM %s" % table).fetchone()[0]
357
+
358
+
359
+ def load_ledger(conn):
360
+ """{"v": 1, "learnings": {id: learning}, "meta": {...}}, ids in order."""
361
+ return {"v": 1,
362
+ "learnings": {lid: json.loads(text) for lid, text in conn.execute(
363
+ "SELECT id, row FROM learnings ORDER BY id")},
364
+ "meta": {key: json.loads(value) for key, value in conn.execute(
365
+ "SELECT key, value FROM meta")}}
366
+
367
+
368
+ def _write_ledger(conn, ledger):
369
+ conn.execute("DELETE FROM learnings")
370
+ conn.executemany("INSERT INTO learnings (id, row) VALUES (?, ?)", [
371
+ (lid, json.dumps(learning, ensure_ascii=False, sort_keys=True))
372
+ for lid, learning in ledger["learnings"].items()])
373
+ conn.execute("DELETE FROM meta")
374
+ conn.executemany("INSERT INTO meta (key, value) VALUES (?, ?)", [
375
+ (key, json.dumps(value)) for key, value in (ledger.get("meta") or {}).items()])
376
+
377
+
378
+ def save_ledger(conn, ledger):
379
+ """Replace the learnings and their meta with `ledger`'s, in one transaction:
380
+ a forgotten learning leaves the table with it."""
381
+ with transaction(conn):
382
+ _write_ledger(conn, ledger)
383
+
384
+
385
+ def gate_stopped(conn):
386
+ """True when the benefit gate stopped injection."""
387
+ found = conn.execute("SELECT stopped FROM gate").fetchone()
388
+ return bool(found and found[0])
389
+
390
+
391
+ def set_gate(conn, stopped, report):
392
+ conn.execute("INSERT OR REPLACE INTO gate (one, stopped, report) VALUES (1, ?, ?)",
393
+ (1 if stopped else 0, json.dumps(report)))
394
+
395
+
396
+ def seen(conn, session):
397
+ """The learning ids a write note already showed in `session`."""
398
+ return {lid for (lid,) in conn.execute(
399
+ "SELECT learning FROM notes_seen WHERE session = ?", (session,))}
400
+
401
+
402
+ def mark_seen(conn, session, ids):
403
+ with transaction(conn):
404
+ conn.executemany("INSERT OR IGNORE INTO notes_seen (session, learning) VALUES (?, ?)",
405
+ [(session, lid) for lid in ids])
406
+
407
+
408
+ # --- the cache database -------------------------------------------------------------
409
+ # One database per cache dir, `<cache>/tezgah.db`: the evidence ledger and the
410
+ # small cache stores below it (CACHE_SCHEMA).
411
+ #
412
+ # The evidence ledger: a caller names a session's ledger by the path its JSONL
413
+ # file had, `<cache>/evidence/<session>.jsonl` (`tezgah_integrity._path`): the
414
+ # cache is two directories up and the session is the file's stem. A JSONL file
415
+ # still at that path is the session's legacy file, written before this database
416
+ # (or by an opencode process still running a plugin from before it), imported
417
+ # past the bytes already in before every read or write of that session and
418
+ # never renamed aside, because that writer may still append to it.
419
+ CACHE_DB = "tezgah.db"
420
+ CACHE_VERSION = 2
421
+ EVIDENCE_SCHEMA = IMPORT_SCHEMA + """
422
+ CREATE TABLE IF NOT EXISTS evidence (n INTEGER PRIMARY KEY, session TEXT NOT NULL,
423
+ kind TEXT, ts INTEGER, row TEXT NOT NULL);
424
+ CREATE INDEX IF NOT EXISTS evidence_session ON evidence (session, n);
425
+ CREATE INDEX IF NOT EXISTS evidence_ts ON evidence (ts);
426
+ """
427
+ # The cache stores, each the table of one kind of file the hooks wrote before
428
+ # (CACHE_LEGACY). A document store keeps one JSON `doc` per `key`, a mark store
429
+ # only the key; `at` is the epoch second of the last write, the mtime the file
430
+ # had. The key is the name the file had, so a legacy file imports under it.
431
+ DOC_STORES = ("turns", "switches", "harness_drift", "import_crash", "judge_last",
432
+ "skill_pick", "section_hint", "update_check", "cursor_answer",
433
+ "taste_learn")
434
+ MARK_STORES = ("gate_inactive", "nudged", "cursor_reinforced")
435
+ CACHE_SCHEMA = EVIDENCE_SCHEMA + "".join(
436
+ "CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, at REAL NOT NULL, doc TEXT NOT NULL);\n"
437
+ % table for table in DOC_STORES) + "".join(
438
+ "CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, at REAL NOT NULL);\n"
439
+ % table for table in MARK_STORES) + """
440
+ CREATE TABLE IF NOT EXISTS used (session TEXT NOT NULL, kind TEXT NOT NULL, at REAL NOT NULL,
441
+ PRIMARY KEY (session, kind));
442
+ CREATE TABLE IF NOT EXISTS judge_down (key TEXT PRIMARY KEY, until REAL NOT NULL);
443
+ CREATE TABLE IF NOT EXISTS lesson_taint (root TEXT NOT NULL, key TEXT NOT NULL,
444
+ source TEXT, ts INTEGER, PRIMARY KEY (root, key));
445
+ CREATE TABLE IF NOT EXISTS snapshots (n INTEGER PRIMARY KEY, id TEXT NOT NULL UNIQUE,
446
+ meta TEXT NOT NULL);
447
+ CREATE TABLE IF NOT EXISTS replay_runs (n INTEGER PRIMARY KEY, run TEXT NOT NULL UNIQUE,
448
+ summary TEXT NOT NULL);
449
+ """
450
+ # Where each cache store's files were, under the cache dir (`_import_cache`).
451
+ CACHE_LEGACY = {
452
+ "used": "sessions", "turns": "turns", "switches": "switches",
453
+ "gate_inactive": "gate-inactive", "nudged": "nudged", "harness_drift": "harness-drift",
454
+ "import_crash": "import-crash", "judge_down": "judge-down",
455
+ "judge_last": "judge-last.json", "skill_pick": "skill-pick",
456
+ "section_hint": "skill-section-hint", "update_check": "update.json",
457
+ "lesson_taint": "lessons", "cursor_reinforced": "reinforced", "cursor_answer": "answer",
458
+ "replay_runs": "replay", "snapshots": "snapshots", "taste_learn": "taste-learn",
459
+ }
460
+ # tezgah_judge.DOWN_FOR when the judge-down markers were files: a marker's
461
+ # mtime plus this is the expiry it imports with
462
+ LEGACY_DOWN_FOR = 300
463
+ # the bulk import's stamp in the cache dir, and how often session start may
464
+ # launch it (`import_later`)
465
+ IMPORT_STAMP = "evidence-import.stamp"
466
+ IMPORT_EVERY = 86400
467
+ # One connection per database per process: db path -> (connection, the file's
468
+ # (dev, ino), {legacy path: its (dev, ino, size) once fully imported}). A
469
+ # database removed or replaced under the process (doctor, uninstall, a test's
470
+ # fresh cache) is reopened on the next call.
471
+ _HELD = {}
472
+
473
+
474
+ def _ledger(path):
475
+ """(cache dir, session) for a ledger path."""
476
+ folder, name = os.path.split(path)
477
+ return os.path.dirname(folder), name[:-len(".jsonl")] if name.endswith(".jsonl") else name
478
+
479
+
480
+ def _cache_db(cache, create=True):
481
+ """The held database of `cache`, the cache stores' legacy files imported
482
+ on its open (`_import_cache`); None when it does not exist and not
483
+ `create`."""
484
+ path = os.path.join(cache, CACHE_DB)
485
+ try:
486
+ st = os.stat(path)
487
+ here = (st.st_dev, st.st_ino)
488
+ except OSError:
489
+ here = None
490
+ held = _HELD.get(path)
491
+ if held and held[1] == here:
492
+ return held
493
+ if held:
494
+ del _HELD[path]
495
+ held[0].close()
496
+ if here is None and not create:
497
+ return None
498
+ conn = connect(path, CACHE_SCHEMA, CACHE_VERSION)
499
+ # no fsync per commit under WAL: a crash can lose the newest rows but never
500
+ # corrupts the file, the durability the JSONL append had
501
+ conn.execute("PRAGMA synchronous = NORMAL")
502
+ st = os.stat(path)
503
+ held = _HELD[path] = (conn, (st.st_dev, st.st_ino), {})
504
+ _import_cache(conn, cache)
505
+ return held
506
+
507
+
508
+ def _insert(conn, session, text):
509
+ """One evidence row: `text` verbatim, its `kind` and `ts` beside it when it
510
+ is a JSON object that carries them."""
511
+ try:
512
+ row = json.loads(text)
513
+ except ValueError:
514
+ row = None
515
+ row = row if isinstance(row, dict) else {}
516
+ kind, ts = row.get("kind"), row.get("ts")
517
+ try:
518
+ ts = int(ts) if isinstance(ts, (int, float)) and not isinstance(ts, bool) else None
519
+ except (ValueError, OverflowError): # NaN, infinity
520
+ ts = None
521
+ conn.execute("INSERT INTO evidence (session, kind, ts, row) VALUES (?, ?, ?, ?)",
522
+ (session, kind if isinstance(kind, str) else None,
523
+ ts if ts is not None and -2 ** 63 <= ts < 2 ** 63 else None, text))
524
+
525
+
526
+ def _evidence_put(session):
527
+ """The import of one legacy ledger line: kept as it was written, so a line
528
+ that does not parse stays a line the reader names as damage."""
529
+ def put(conn, line):
530
+ if not line.strip():
531
+ return
532
+ try:
533
+ text = line.decode("utf-8")
534
+ except UnicodeDecodeError:
535
+ # the marker `tezgah_integrity._parse` names as damage, hashed
536
+ # over the line with its newline as the file reader hashed it, so
537
+ # a damage row it wrote still names the same line
538
+ text = "\x00not utf-8 %s" % hashlib.sha1(line + b"\n").hexdigest()[:12]
539
+ _insert(conn, session, text)
540
+ return put
541
+
542
+
543
+ def _sync(held, path):
544
+ """Import the legacy JSONL at `path` past the bytes already in. Without a
545
+ file, or with nothing new in it, this is one stat (and one lookup the first
546
+ time a process sees it)."""
547
+ try:
548
+ st = os.stat(path)
549
+ except OSError:
550
+ return
551
+ conn, _here, seen = held
552
+ mark = (st.st_dev, st.st_ino, st.st_size)
553
+ if seen.get(path) == mark:
554
+ return
555
+ name = os.path.basename(path)
556
+ found = conn.execute("SELECT bytes FROM imported WHERE name = ? AND dev = ? AND ino = ?",
557
+ (name, st.st_dev, st.st_ino)).fetchone()
558
+ if found and found[0] == st.st_size:
559
+ seen[path] = mark
560
+ return
561
+ if conn.execute("SELECT 1 FROM import_failed WHERE name = ? AND mtime_ns = ? "
562
+ "AND size = ?", (name, st.st_mtime_ns, st.st_size)).fetchone():
563
+ return
564
+ _import_rows(conn, name, path, st, _evidence_put(_ledger(path)[1]), keep=True)
565
+
566
+
567
+ def _session(path, create=False):
568
+ """(held database, session) for the ledger at `path`, its legacy file
569
+ imported first; the database is None when neither it nor a legacy file
570
+ exists and not `create`, so a read leaves no file behind."""
571
+ cache, session = _ledger(path)
572
+ held = _cache_db(cache, create or os.path.exists(path))
573
+ if held:
574
+ _sync(held, path)
575
+ return held, session
576
+
577
+
578
+ def import_session(path):
579
+ """The legacy JSONL at `path` into its cache's database."""
580
+ if os.path.exists(path):
581
+ _session(path)
582
+
583
+
584
+ def append_evidence(path, text):
585
+ """One row (`text`, its JSON) into the ledger at `path`, after its legacy
586
+ file's rows: one INSERT in autocommit, so a row is whole or absent. Raises
587
+ OSError or sqlite3.Error."""
588
+ held, session = _session(path, create=True)
589
+ _insert(held[0], session, text)
590
+
591
+
592
+ def evidence_rows(path, tail=None, kind=None):
593
+ """The row texts of the ledger at `path`, oldest first: the last `tail`
594
+ with one, only the rows of `kind` with that."""
595
+ held, session = _session(path)
596
+ if held is None:
597
+ return []
598
+ conn = held[0]
599
+ if kind:
600
+ found = conn.execute("SELECT row FROM evidence WHERE session = ? AND kind = ? "
601
+ "ORDER BY n", (session, kind))
602
+ elif tail:
603
+ return [text for (text,) in reversed(conn.execute(
604
+ "SELECT row FROM evidence WHERE session = ? ORDER BY n DESC LIMIT ?",
605
+ (session, tail)).fetchall())]
606
+ else:
607
+ found = conn.execute("SELECT row FROM evidence WHERE session = ? ORDER BY n",
608
+ (session,))
609
+ return [text for (text,) in found]
610
+
611
+
612
+ def evidence_first(path):
613
+ """The first row text of the ledger at `path`, or None."""
614
+ held, session = _session(path)
615
+ found = held and held[0].execute(
616
+ "SELECT row FROM evidence WHERE session = ? ORDER BY n LIMIT 1", (session,)).fetchone()
617
+ return found[0] if found else None
618
+
619
+
620
+ def has_evidence(path):
621
+ """True when the ledger at `path` has a legacy file or a row; creates
622
+ nothing."""
623
+ if os.path.exists(path):
624
+ return True
625
+ cache, session = _ledger(path)
626
+ held = _cache_db(cache, create=False)
627
+ return bool(held and held[0].execute(
628
+ "SELECT 1 FROM evidence WHERE session = ? LIMIT 1", (session,)).fetchone())
629
+
630
+
631
+ def import_evidence(cache):
632
+ """Every legacy session file under `<cache>/evidence` into the cache's
633
+ database, the least recently written first; the count of files seen.
634
+ Idempotent: a file imports only past the bytes the `imported` table
635
+ already counts for it."""
636
+ found = []
637
+ try:
638
+ for entry in os.scandir(os.path.join(cache, "evidence")):
639
+ if entry.name.endswith(".jsonl"):
640
+ try:
641
+ found.append((entry.stat().st_mtime, entry.path))
642
+ except OSError:
643
+ continue
644
+ except OSError:
645
+ return 0
646
+ if found:
647
+ held = _cache_db(cache)
648
+ for _mtime, path in sorted(found):
649
+ _sync(held, path)
650
+ return len(found)
651
+
652
+
653
+ def sessions(cache):
654
+ """[(session, its newest ts)] of every ledger in `cache`, the one written
655
+ last first, after the bulk import (`import_evidence`)."""
656
+ import_evidence(cache)
657
+ held = _cache_db(cache, create=False)
658
+ return held[0].execute(
659
+ "SELECT session, max(ts) FROM evidence GROUP BY session "
660
+ "ORDER BY max(ts) DESC, max(n) DESC").fetchall() if held else []
661
+
662
+
663
+ def recent_rows(cache, since, kinds, but):
664
+ """(session, ts, row text) of every row of `kinds` stamped at or after `since`
665
+ in a ledger other than session `but`, oldest first. The legacy files
666
+ written since are imported first; the older ones cannot hold such a row."""
667
+ fresh = []
668
+ try:
669
+ for entry in os.scandir(os.path.join(cache, "evidence")):
670
+ try:
671
+ if entry.name.endswith(".jsonl") and entry.stat().st_mtime >= since:
672
+ fresh.append(entry.path)
673
+ except OSError:
674
+ continue
675
+ except OSError:
676
+ pass
677
+ held = _cache_db(cache, create=bool(fresh))
678
+ if held is None:
679
+ return []
680
+ for path in fresh:
681
+ _sync(held, path)
682
+ return held[0].execute(
683
+ "SELECT session, ts, row FROM evidence WHERE ts >= ? AND kind IN (%s) AND session != ? "
684
+ "ORDER BY n" % ", ".join("?" * len(kinds)), (since,) + tuple(kinds) + (but,)).fetchall()
685
+
686
+
687
+ def _forget(conn, cache, session):
688
+ """Delete one session's rows, its import offsets and its legacy file (inside
689
+ the caller's transaction); the count of rows deleted. The offsets go with the
690
+ file: a later file at that name may reuse its inode."""
691
+ rows = conn.execute("DELETE FROM evidence WHERE session = ?", (session,)).rowcount
692
+ conn.execute("DELETE FROM imported WHERE name = ?", (session + ".jsonl",))
693
+ try:
694
+ os.remove(os.path.join(cache, "evidence", session + ".jsonl"))
695
+ except OSError:
696
+ pass
697
+ return rows
698
+
699
+
700
+ def forget_session(path):
701
+ """Delete the ledger at `path`: its rows and its legacy file; the count of
702
+ rows deleted."""
703
+ cache, session = _ledger(path)
704
+ held = _cache_db(cache, create=False)
705
+ if held is None:
706
+ try:
707
+ os.remove(path)
708
+ except OSError:
709
+ pass
710
+ return 0
711
+ with transaction(held[0]) as conn:
712
+ return _forget(conn, cache, session)
713
+
714
+
715
+ def sweep_evidence(cache, cutoff, keep=None):
716
+ """Delete every ledger in `cache` whose newest row is older than `cutoff`
717
+ (epoch seconds), all but session `keep`: (sessions, rows) deleted. A
718
+ session is the unit, as the file was: a ledger cut short at its start would
719
+ lose the first row the switch latch reads."""
720
+ import_evidence(cache)
721
+ held = _cache_db(cache, create=False)
722
+ if held is None:
723
+ return 0, 0
724
+ with transaction(held[0]) as conn:
725
+ idle = [s for (s,) in conn.execute(
726
+ "SELECT session FROM evidence GROUP BY session HAVING max(coalesce(ts, 0)) < ?",
727
+ (cutoff,)).fetchall() if s != keep]
728
+ return len(idle), sum(_forget(conn, cache, s) for s in idle)
729
+
730
+
731
+ def import_later(cache):
732
+ """Start `import-evidence` for `cache` detached, while a legacy file exists
733
+ and at most once per IMPORT_EVERY (the stamp's mtime, written before the
734
+ start, so a failed start waits a day). Total."""
735
+ try:
736
+ with os.scandir(os.path.join(cache, "evidence")) as found:
737
+ if not any(entry.name.endswith(".jsonl") for entry in found):
738
+ return
739
+ stamp = os.path.join(cache, IMPORT_STAMP)
740
+ try:
741
+ if time.time() - os.path.getmtime(stamp) < IMPORT_EVERY:
742
+ return
743
+ except OSError:
744
+ pass
745
+ with open(stamp, "wb"):
746
+ pass
747
+ subprocess.Popen([sys.executable, os.path.abspath(__file__), "import-evidence", cache],
748
+ stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
749
+ stderr=subprocess.DEVNULL, start_new_session=True)
750
+ except (OSError, ValueError):
751
+ pass
752
+
753
+
754
+ # --- the cache stores ---------------------------------------------------------------
755
+ # Each function takes the cache dir as `cache`, `tezgah_paths.cache_dir()` when
756
+ # it is None. A read never creates the database: with neither it nor the
757
+ # store's legacy files there, it answers as an empty store.
758
+
759
+ def _cache_dir():
760
+ import tezgah_paths
761
+ return tezgah_paths.cache_dir()
762
+
763
+
764
+ def _store(cache, table, create):
765
+ """The connection of the database holding `table`, or None when it does
766
+ not exist, the store has no legacy file to import, and not `create`."""
767
+ cache = cache or _cache_dir()
768
+ held = _cache_db(cache, create=False) or (
769
+ (create or os.path.exists(os.path.join(cache, CACHE_LEGACY[table])))
770
+ and _cache_db(cache))
771
+ return held[0] if held else None
772
+
773
+
774
+ def doc(table, key, cache=None):
775
+ """The JSON value a document store holds under `key`, or None: none
776
+ stored, or the database cannot be read."""
777
+ try:
778
+ conn = _store(cache, table, False)
779
+ found = conn and conn.execute("SELECT doc FROM %s WHERE key = ?" % table,
780
+ (key,)).fetchone()
781
+ return json.loads(found[0]) if found else None
782
+ except ERRORS + (ValueError,):
783
+ return None
784
+
785
+
786
+ def put_doc(table, key, value, cache=None):
787
+ """`value` (JSON) under `key` in a document store; False when it was not
788
+ written."""
789
+ try:
790
+ _store(cache, table, True).execute(
791
+ "INSERT OR REPLACE INTO %s (key, at, doc) VALUES (?, ?, ?)" % table,
792
+ (key, time.time(), json.dumps(value)))
793
+ return True
794
+ except ERRORS + (TypeError, ValueError):
795
+ return False
796
+
797
+
798
+ def drop(table, key, cache=None):
799
+ """`key` out of a document or mark store. Total."""
800
+ try:
801
+ conn = _store(cache, table, False)
802
+ if conn:
803
+ conn.execute("DELETE FROM %s WHERE key = ?" % table, (key,))
804
+ except ERRORS:
805
+ pass
806
+
807
+
808
+ def mark(table, key, cache=None):
809
+ """Set `key` in a mark store: True when this call set it, False when it
810
+ was set already. One INSERT, so of two callers exactly one gets True.
811
+ Raises ERRORS."""
812
+ return _store(cache, table, True).execute(
813
+ "INSERT OR IGNORE INTO %s (key, at) VALUES (?, ?)" % table,
814
+ (key, time.time())).rowcount == 1
815
+
816
+
817
+ def marked(table, key, cache=None):
818
+ """True when a mark store holds `key`. Total: an unreadable store is
819
+ unmarked."""
820
+ try:
821
+ conn = _store(cache, table, False)
822
+ return bool(conn and conn.execute("SELECT 1 FROM %s WHERE key = ?" % table,
823
+ (key,)).fetchone())
824
+ except ERRORS:
825
+ return False
826
+
827
+
828
+ def use(session, kind, cache=None):
829
+ """Record that `session` used `kind`, stamped now. Raises ERRORS."""
830
+ _store(cache, "used", True).execute(
831
+ "INSERT OR REPLACE INTO used (session, kind, at) VALUES (?, ?, ?)",
832
+ (session, kind, time.time()))
833
+
834
+
835
+ def used(session, cache=None):
836
+ """The kinds `session` used, a set. Total."""
837
+ try:
838
+ conn = _store(cache, "used", False)
839
+ return {kind for (kind,) in conn.execute(
840
+ "SELECT kind FROM used WHERE session = ?", (session,))} if conn else set()
841
+ except ERRORS:
842
+ return set()
843
+
844
+
845
+ def used_sessions(window, cache=None):
846
+ """(the kind sets of the `window` sessions that recorded last, newest
847
+ first; the count of sessions recorded). Total."""
848
+ try:
849
+ conn = _store(cache, "used", False)
850
+ if not conn:
851
+ return [], 0
852
+ newest = [s for (s,) in conn.execute(
853
+ "SELECT session FROM used GROUP BY session ORDER BY max(at) DESC LIMIT ?",
854
+ (window,))]
855
+ total = conn.execute("SELECT count(DISTINCT session) FROM used").fetchone()[0]
856
+ return [used(s, cache) for s in newest], total
857
+ except ERRORS:
858
+ return [], 0
859
+
860
+
861
+ def down_until(key, cache=None):
862
+ """The epoch second the judge-down mark `key` expires at, or None. Total."""
863
+ try:
864
+ conn = _store(cache, "judge_down", False)
865
+ found = conn and conn.execute("SELECT until FROM judge_down WHERE key = ?",
866
+ (key,)).fetchone()
867
+ return found[0] if found else None
868
+ except ERRORS:
869
+ return None
870
+
871
+
872
+ def set_down(key, until, cache=None):
873
+ """Mark `key` down until the epoch second `until`. Total."""
874
+ try:
875
+ _store(cache, "judge_down", True).execute(
876
+ "INSERT OR REPLACE INTO judge_down (key, until) VALUES (?, ?)", (key, until))
877
+ except ERRORS:
878
+ pass
879
+
880
+
881
+ def taint(root, cache=None):
882
+ """{lesson key: source} of the lesson-taint rows of `root` (a key the
883
+ caller derives from the repository). Total."""
884
+ try:
885
+ conn = _store(cache, "lesson_taint", False)
886
+ return {key: source or "" for key, source in conn.execute(
887
+ "SELECT key, source FROM lesson_taint WHERE root = ?", (root,))} if conn else {}
888
+ except ERRORS:
889
+ return {}
890
+
891
+
892
+ def write_taint(root, keep, rows, cache=None):
893
+ """In one transaction: drop the rows of `root` whose key is not in `keep`,
894
+ then store `rows` ({key, source, ts}), a later row over an earlier one of
895
+ its key. Total."""
896
+ try:
897
+ conn = _store(cache, "lesson_taint", True)
898
+ with transaction(conn):
899
+ gone = [(root, key) for (key,) in conn.execute(
900
+ "SELECT key FROM lesson_taint WHERE root = ?", (root,)) if key not in keep]
901
+ conn.executemany("DELETE FROM lesson_taint WHERE root = ? AND key = ?", gone)
902
+ conn.executemany(
903
+ "INSERT OR REPLACE INTO lesson_taint (root, key, source, ts) VALUES (?, ?, ?, ?)",
904
+ [(root, str(r["key"]), _column(r.get("source")), _column(r.get("ts")))
905
+ for r in rows])
906
+ except ERRORS:
907
+ pass
908
+
909
+
910
+ def add_snapshot(sid, meta, cap, cache=None):
911
+ """Index snapshot `sid` (its manifest `meta`) and drop the oldest rows past
912
+ `cap`, in one transaction; the ids dropped, whose blobs the caller removes.
913
+ Raises ERRORS."""
914
+ conn = _store(cache, "snapshots", True)
915
+ with transaction(conn):
916
+ conn.execute("INSERT INTO snapshots (id, meta) VALUES (?, ?)", (sid, json.dumps(meta)))
917
+ old = [i for (i,) in conn.execute(
918
+ "SELECT id FROM snapshots ORDER BY n DESC LIMIT -1 OFFSET ?", (cap,))]
919
+ conn.executemany("DELETE FROM snapshots WHERE id = ?", [(i,) for i in old])
920
+ return old
921
+
922
+
923
+ def snapshots(sid=None, cache=None):
924
+ """The snapshot manifests, oldest first, or the one of `sid`. Total."""
925
+ try:
926
+ conn = _store(cache, "snapshots", False)
927
+ found = [] if conn is None else conn.execute(
928
+ "SELECT meta FROM snapshots %s ORDER BY n" % ("WHERE id = ?" if sid else ""),
929
+ (sid,) if sid else ())
930
+ return [json.loads(meta) for (meta,) in found]
931
+ except ERRORS + (ValueError,):
932
+ return []
933
+
934
+
935
+ def add_replay(run, summary, cache=None):
936
+ """Index replay run `run` (its directory) with its summary; the newest
937
+ indexed run is the latest. Raises ERRORS."""
938
+ _store(cache, "replay_runs", True).execute(
939
+ "INSERT OR REPLACE INTO replay_runs (run, summary) VALUES (?, ?)",
940
+ (run, json.dumps(summary)))
941
+
942
+
943
+ def replay_latest(cache=None):
944
+ """The directory of the latest replay run, or None. Total."""
945
+ try:
946
+ conn = _store(cache, "replay_runs", False)
947
+ found = conn and conn.execute(
948
+ "SELECT run FROM replay_runs ORDER BY n DESC LIMIT 1").fetchone()
949
+ return found[0] if found else None
950
+ except ERRORS:
951
+ return None
952
+
953
+
954
+ def replay_summary(run, cache=None):
955
+ """The summary replay run `run` was indexed with, or None. Total."""
956
+ try:
957
+ conn = _store(cache, "replay_runs", False)
958
+ found = conn and conn.execute("SELECT summary FROM replay_runs WHERE run = ?",
959
+ (run,)).fetchone()
960
+ return json.loads(found[0]) if found else None
961
+ except ERRORS + (ValueError,):
962
+ return None
963
+
964
+
965
+ def sweep_cache(cache, cutoff, keep=None):
966
+ """Delete the turn stamps and the used-kind sessions of `cache` last
967
+ written before `cutoff` (epoch seconds), all but session `keep` (its
968
+ `tezgah_context.slug`), after importing their legacy files; the count of
969
+ rows deleted. The two stores the doctor swept as files. Raises ERRORS."""
970
+ conn = _store(cache, "turns", False) or _store(cache, "used", False)
971
+ if conn is None:
972
+ return 0
973
+ with transaction(conn):
974
+ rows = conn.execute("DELETE FROM turns WHERE at < ? AND key IS NOT ?",
975
+ (cutoff, keep)).rowcount
976
+ return rows + conn.execute(
977
+ "DELETE FROM used WHERE session IN (SELECT session FROM used GROUP BY session "
978
+ "HAVING max(at) < ?) AND session IS NOT ?", (cutoff, keep)).rowcount
979
+
980
+
981
+ def _files(folder):
982
+ """[(name, path, mtime)] of the regular files directly in `folder`."""
983
+ out = []
984
+ try:
985
+ with os.scandir(folder) as found:
986
+ for entry in found:
987
+ try:
988
+ if entry.is_file(follow_symlinks=False):
989
+ out.append((entry.name, entry.path, entry.stat().st_mtime))
990
+ except OSError:
991
+ continue
992
+ except OSError:
993
+ pass
994
+ return out
995
+
996
+
997
+ def _dirs(folder):
998
+ """[(mtime_ns, name, path)] of the directories directly in `folder`, oldest
999
+ first."""
1000
+ out = []
1001
+ try:
1002
+ with os.scandir(folder) as found:
1003
+ for entry in found:
1004
+ try:
1005
+ if entry.is_dir(follow_symlinks=False):
1006
+ out.append((entry.stat().st_mtime_ns, entry.name, entry.path))
1007
+ except OSError:
1008
+ continue
1009
+ except OSError:
1010
+ pass
1011
+ return sorted(out)
1012
+
1013
+
1014
+ def _read_text(path):
1015
+ try:
1016
+ with open(path, encoding="utf-8") as fh:
1017
+ return fh.read()
1018
+ except (OSError, ValueError): # UnicodeDecodeError is a ValueError
1019
+ return None
1020
+
1021
+
1022
+ def _read_json(path):
1023
+ try:
1024
+ return json.loads(_read_text(path) or "")
1025
+ except ValueError:
1026
+ return None
1027
+
1028
+
1029
+ DOC_SQL = "INSERT OR IGNORE INTO %s (key, at, doc) VALUES (?, ?, ?)"
1030
+ MARK_SQL = "INSERT OR IGNORE INTO %s (key, at) VALUES (?, ?)"
1031
+
1032
+
1033
+ def _legacy(cache, table):
1034
+ """(the INSERTs, the files) of one cache store's legacy files under
1035
+ `cache`. A file that cannot be read or parsed brings no row, as its old
1036
+ reader read nothing from it, and goes with the rest."""
1037
+ src = os.path.join(cache, CACHE_LEGACY[table])
1038
+ put, gone = [], []
1039
+ if table in ("judge_last", "update_check"):
1040
+ if os.path.isfile(src):
1041
+ value = _read_json(src)
1042
+ if isinstance(value, dict):
1043
+ put.append((DOC_SQL % table, ("", os.path.getmtime(src), json.dumps(value))))
1044
+ gone.append(src)
1045
+ elif table == "snapshots":
1046
+ # the capture order is each directory's mtime; the blob stays
1047
+ for _ns, name, path in _dirs(src):
1048
+ meta = os.path.join(path, "meta.json")
1049
+ value = _read_json(meta)
1050
+ if isinstance(value, dict):
1051
+ put.append(("INSERT OR IGNORE INTO snapshots (id, meta) VALUES (?, ?)",
1052
+ (name, json.dumps(value))))
1053
+ if os.path.exists(meta):
1054
+ gone.append(meta)
1055
+ elif table == "replay_runs":
1056
+ latest = (_read_text(os.path.join(src, "latest")) or "").strip()
1057
+ latest = latest and os.path.realpath(latest)
1058
+ # by name (the run's start time), the one `latest` named last; the key
1059
+ # is the run directory's real path (tezgah_replay._run_key)
1060
+ for _ns, _name, path in sorted(_dirs(src), key=lambda d: (
1061
+ os.path.realpath(d[2]) == latest, d[1])):
1062
+ summary = os.path.join(path, "summary.json")
1063
+ value = _read_json(summary)
1064
+ if isinstance(value, dict):
1065
+ put.append(("INSERT OR IGNORE INTO replay_runs (run, summary) VALUES (?, ?)",
1066
+ (os.path.realpath(path), json.dumps(value))))
1067
+ if os.path.exists(summary):
1068
+ gone.append(summary)
1069
+ if os.path.exists(os.path.join(src, "latest")):
1070
+ gone.append(os.path.join(src, "latest"))
1071
+ elif table == "cursor_reinforced":
1072
+ for _ns, session, folder in _dirs(src):
1073
+ for name, path, at in _files(folder):
1074
+ put.append((MARK_SQL % table, (session + "/" + name, at)))
1075
+ gone.append(path)
1076
+ else:
1077
+ for name, path, at in _files(src):
1078
+ if table == "used" and name.endswith(".jsonl"):
1079
+ for line in (_read_text(path) or "").splitlines():
1080
+ try:
1081
+ kind = json.loads(line)["kind"]
1082
+ except (ValueError, KeyError, TypeError):
1083
+ continue
1084
+ if isinstance(kind, str):
1085
+ put.append(("INSERT OR IGNORE INTO used (session, kind, at) "
1086
+ "VALUES (?, ?, ?)", (name[:-len(".jsonl")], kind, at)))
1087
+ elif table == "lesson_taint" and name.endswith(".jsonl"):
1088
+ for line in (_read_text(path) or "").splitlines():
1089
+ try:
1090
+ row = json.loads(line)
1091
+ except ValueError:
1092
+ continue
1093
+ if isinstance(row, dict) and row.get("key"):
1094
+ put.append(("INSERT OR REPLACE INTO lesson_taint (root, key, source, "
1095
+ "ts) VALUES (?, ?, ?, ?)",
1096
+ (name[:-len(".jsonl")], str(row["key"]),
1097
+ _column(row.get("source")), _column(row.get("ts")))))
1098
+ elif table == "judge_down":
1099
+ put.append(("INSERT OR IGNORE INTO judge_down (key, until) VALUES (?, ?)",
1100
+ (name, at + LEGACY_DOWN_FOR)))
1101
+ elif table in MARK_STORES:
1102
+ put.append((MARK_SQL % table, (name, at)))
1103
+ elif table in ("harness_drift", "import_crash", "cursor_answer"):
1104
+ text = _read_text(path)
1105
+ if text is not None:
1106
+ put.append((DOC_SQL % table, (name, at, json.dumps(text))))
1107
+ elif table in DOC_STORES and name.endswith(".json"): # taste-learn's logs stay
1108
+ value = _read_json(path)
1109
+ if value is not None:
1110
+ put.append((DOC_SQL % table, (name[:-len(".json")], at, json.dumps(value))))
1111
+ else:
1112
+ continue
1113
+ gone.append(path)
1114
+ return put, gone
1115
+
1116
+
1117
+ def _legacy_done(conn):
1118
+ return {name[len("legacy:"):] for (name,) in conn.execute(
1119
+ "SELECT name FROM imported WHERE name >= 'legacy:' AND name < 'legacy;'")}
1120
+
1121
+
1122
+ def _import_cache(conn, cache):
1123
+ """Bring each cache store's legacy files (CACHE_LEGACY) in, once per
1124
+ database: a store is done when `imported` holds its `legacy:<table>` row,
1125
+ written in the transaction that takes its rows, one transaction per store.
1126
+ The files are read before that transaction and removed after it commits. A
1127
+ file of a done store is never read again, and any open that finds one
1128
+ removes it: one a process killed between the commit and the removal left,
1129
+ or one a writer from before this release added (an opencode plugin started
1130
+ before the upgrade, until it restarts). With every store done and its files
1131
+ gone this is one indexed lookup and a stat per store per open. ponytail:
1132
+ the snapshot manifests are not looked for once done, because that is a stat
1133
+ per snapshot per open; a leftover one goes with its blob at eviction.
1134
+ Total: a store that fails, whatever the error, is left for the next open
1135
+ and costs the others nothing."""
1136
+ try:
1137
+ done = _legacy_done(conn)
1138
+ except ERRORS:
1139
+ return
1140
+ for table, name in CACHE_LEGACY.items():
1141
+ folder = os.path.join(cache, name)
1142
+ try:
1143
+ if table in done:
1144
+ if table == "snapshots" or not os.path.exists(folder):
1145
+ continue
1146
+ gone = _legacy(cache, table)[1]
1147
+ else:
1148
+ put, gone = _legacy(cache, table)
1149
+ with transaction(conn):
1150
+ if table not in _legacy_done(conn):
1151
+ for sql, args in put:
1152
+ conn.execute(sql, args)
1153
+ conn.execute("INSERT OR IGNORE INTO imported (name, dev, ino, bytes) "
1154
+ "VALUES (?, 0, 0, 0)", ("legacy:" + table,))
1155
+ except Exception:
1156
+ continue
1157
+ for path in gone:
1158
+ try:
1159
+ os.remove(path)
1160
+ except OSError:
1161
+ pass
1162
+ for _ns, _name, path in _dirs(folder) if table == "cursor_reinforced" else ():
1163
+ try:
1164
+ os.rmdir(path)
1165
+ except OSError:
1166
+ pass
1167
+ try:
1168
+ os.rmdir(folder) # only an emptied one: blobs, runs and logs stay
1169
+ except OSError:
1170
+ pass
1171
+
1172
+
1173
+
1174
+ # `evidence VERB LEDGER [ARG]`: the ledger reads and writes for a caller that
1175
+ # cannot open the database itself (opencode's plugin on a runtime without
1176
+ # node:sqlite), each verb with the count of arguments it takes after LEDGER.
1177
+ EVIDENCE_VERBS = {"append": 1, "tail": 1, "first": 0, "kind": 1, "import": 0}
1178
+ EVIDENCE_USAGE = ("usage: tezgah_store.py evidence append LEDGER JSON | tail LEDGER N"
1179
+ " | first LEDGER | kind LEDGER KIND | import LEDGER")
1180
+
1181
+
1182
+ def _evidence_cli(args):
1183
+ """One verb on the ledger at args[1]: `append` stores JSON as a row,
1184
+ `import` brings its legacy file in, and a read prints row texts one per
1185
+ line, oldest first - the last N (`tail`), the first (`first`), or the rows
1186
+ of KIND (`kind`)."""
1187
+ verb = args[0] if args else None
1188
+ if EVIDENCE_VERBS.get(verb) != len(args) - 2:
1189
+ print(EVIDENCE_USAGE, file=sys.stderr)
1190
+ return 2
1191
+ path, rest = args[1], args[2:]
1192
+ try:
1193
+ if verb == "append":
1194
+ append_evidence(path, rest[0])
1195
+ return 0
1196
+ if verb == "import":
1197
+ import_session(path)
1198
+ return 0
1199
+ if verb == "first":
1200
+ found = [evidence_first(path)]
1201
+ elif verb == "kind":
1202
+ found = evidence_rows(path, kind=rest[0])
1203
+ else:
1204
+ try:
1205
+ tail = int(rest[0])
1206
+ except ValueError:
1207
+ print(EVIDENCE_USAGE, file=sys.stderr)
1208
+ return 2
1209
+ found = evidence_rows(path, tail=max(tail, 1))
1210
+ except ERRORS as exc:
1211
+ print("tezgah_store: %s" % exc, file=sys.stderr)
1212
+ return 1
1213
+ sys.stdout.buffer.write("".join(text + "\n" for text in found if text is not None)
1214
+ .encode("utf-8", "surrogatepass"))
1215
+ return 0
1216
+
1217
+
1218
+ # `mark CACHE TABLE KEY` and `use CACHE SESSION KIND`: the two cache-store
1219
+ # writes opencode's plugin makes (the once-per-session nudge and the used-kind
1220
+ # marks), for the same caller as `evidence`. `mark` prints 1 when it set the
1221
+ # mark and 0 when it was set already.
1222
+ STORE_USAGE = "usage: tezgah_store.py mark CACHE TABLE KEY | use CACHE SESSION KIND"
1223
+
1224
+
1225
+ def _store_cli(args):
1226
+ if len(args) != 4 or args[0] not in ("mark", "use") or (
1227
+ args[0] == "mark" and args[2] not in MARK_STORES):
1228
+ print(STORE_USAGE, file=sys.stderr)
1229
+ return 2
1230
+ verb, cache, name, value = args
1231
+ try:
1232
+ if verb == "use":
1233
+ use(name, value, cache)
1234
+ else:
1235
+ print(1 if mark(name, value, cache) else 0)
1236
+ except ERRORS as exc:
1237
+ print("tezgah_store: %s" % exc, file=sys.stderr)
1238
+ return 1
1239
+ return 0
1240
+
1241
+
1242
+ def main(argv):
1243
+ """`import-evidence [CACHE...]`: the bulk import, for the given cache dirs
1244
+ or for the cache and its temp fallback. `evidence ...`: `_evidence_cli`;
1245
+ `mark ...` and `use ...`: `_store_cli`."""
1246
+ if argv[1:2] == ["evidence"]:
1247
+ return _evidence_cli(argv[2:])
1248
+ if argv[1:2] in (["mark"], ["use"]):
1249
+ return _store_cli(argv[1:])
1250
+ if argv[1:2] != ["import-evidence"]:
1251
+ print("usage: tezgah_store.py import-evidence [CACHE_DIR...]\n" + EVIDENCE_USAGE
1252
+ + "\n" + STORE_USAGE, file=sys.stderr)
1253
+ return 2
1254
+ caches = argv[2:]
1255
+ if not caches:
1256
+ import tezgah_paths as tp
1257
+ caches = list(dict.fromkeys((tp.CACHE, tp.fallback_cache())))
1258
+ for cache in caches:
1259
+ try:
1260
+ print("%s: %d legacy ledger file(s) imported" % (cache, import_evidence(cache)))
1261
+ except ERRORS as exc:
1262
+ print("%s: import failed: %s" % (cache, exc), file=sys.stderr)
1263
+ return 1
1264
+ return 0
1265
+
1266
+
1267
+ if __name__ == "__main__":
1268
+ sys.exit(main(sys.argv))