@r1z4x/tezgah 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +103 -1
- package/MANIFEST +3 -0
- package/agents/tezgah-reviewer.md +19 -10
- package/bin/tezgah-docs +3 -3
- package/bin/tezgah-doctor +75 -18
- package/bin/tezgah-route +5 -2
- package/bin/tezgah-setup +30 -18
- package/bin/tezgah-status +3 -1
- package/bin/tezgah-task +130 -37
- package/bin/tezgah-taste +74 -55
- package/bin/tezgah-triage +12 -7
- package/docs/architecture.md +4 -4
- package/docs/clarity-baseline.json +9 -9
- package/docs/contract.md +22 -12
- package/docs/environment.md +4 -1
- package/docs/evidence.md +71 -34
- package/docs/gate.md +21 -21
- package/docs/glossary.md +5 -5
- package/docs/hosts.md +8 -8
- package/docs/judge.md +93 -27
- package/docs/operations.md +69 -30
- package/docs/skills.md +2 -2
- package/docs/status-line.md +7 -7
- package/docs/vision.md +10 -5
- package/hooks/tezgah_agents.py +22 -11
- package/hooks/tezgah_attest.py +15 -22
- package/hooks/tezgah_context.py +84 -124
- package/hooks/tezgah_gate.py +26 -23
- package/hooks/tezgah_guard.py +9 -11
- package/hooks/tezgah_integrity.py +77 -178
- package/hooks/tezgah_judge.py +270 -122
- package/hooks/tezgah_lessons.py +18 -63
- package/hooks/tezgah_paths.py +183 -55
- package/hooks/tezgah_policy.py +35 -8
- package/hooks/tezgah_replay.py +57 -33
- package/hooks/tezgah_shapes.py +4 -4
- package/hooks/tezgah_skill_pick.py +6 -35
- package/hooks/tezgah_snapshot.py +25 -46
- package/hooks/tezgah_store.py +1268 -0
- package/hooks/tezgah_task.py +146 -40
- package/hooks/tezgah_taste.py +21 -18
- package/hooks/tezgah_taste_ledger.py +63 -88
- package/hooks/tezgah_update.py +6 -26
- package/hosts/codex/hook.py +5 -0
- package/hosts/cursor/hook.py +19 -31
- package/hosts/opencode/plugins/tezgah.js +309 -108
- package/package.json +1 -1
- package/skills/harness/SKILL.md +4 -0
- package/skills/plan-add/SKILL.md +23 -7
- package/skills/plan-add/render_table.py +3 -2
- package/skills/plan-status/SKILL.md +12 -4
- package/skills/plan-sync/SKILL.md +19 -9
- package/skills/tezgah-contract/SKILL.md +25 -12
|
@@ -0,0 +1,1268 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""SQLite runtime stores (ADR 021): the connection, the schema, and the import
|
|
3
|
+
of the JSON and JSONL files a store replaces. No caller opens a store file
|
|
4
|
+
itself.
|
|
5
|
+
|
|
6
|
+
It holds the taste stores, the evidence ledger and the cache stores:
|
|
7
|
+
- `<repo>/.tezgah/taste/taste.db`: the capture signals, the typed decisions,
|
|
8
|
+
the defects, the calibration labels, the injection rows, the benefit gate's
|
|
9
|
+
state, the repository's learnings with their meta, and the learnings each
|
|
10
|
+
session's write notes already showed.
|
|
11
|
+
- `~/.config/tezgah/taste/taste.db`: the user-scope learnings with their meta.
|
|
12
|
+
- `<cache>/tezgah.db`: the evidence ledger, one `evidence` row per ledger line
|
|
13
|
+
of every session (`tezgah_integrity` writes and reads it, through the
|
|
14
|
+
functions under "the cache database" below, and so does opencode's plugin,
|
|
15
|
+
through node:sqlite or the `evidence` CLI in `main`), and the small stores
|
|
16
|
+
the hooks kept as files beside it before (CACHE_LEGACY): the used-tool
|
|
17
|
+
marks, the turn stamps, the switch baselines, the status marks, the judge's
|
|
18
|
+
last use and down marks, the skill hints, the update check, the lesson-taint
|
|
19
|
+
index, Cursor's marks, the replay and snapshot indexes and the taste-learn
|
|
20
|
+
stamps (the functions under "the cache stores").
|
|
21
|
+
|
|
22
|
+
A row keeps its JSON payload in `row`, so its fields stay what the old files
|
|
23
|
+
held; the columns beside it are the fields a query filters on. WAL and a busy
|
|
24
|
+
timeout let a hook append while the CLI reads. The directory is made 0700 and
|
|
25
|
+
the database file is created 0600 before SQLite opens it; SQLite gives its
|
|
26
|
+
`-wal` and `-shm` files the database's mode.
|
|
27
|
+
"""
|
|
28
|
+
import contextlib
|
|
29
|
+
import hashlib
|
|
30
|
+
import json
|
|
31
|
+
import os
|
|
32
|
+
import sqlite3
|
|
33
|
+
import subprocess
|
|
34
|
+
import sys
|
|
35
|
+
import time
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
import fcntl
|
|
39
|
+
except ImportError: # Windows: the old writers took no lock there either
|
|
40
|
+
fcntl = None
|
|
41
|
+
|
|
42
|
+
BUSY_MS = 5000
|
|
43
|
+
TASTE_DB = "taste.db"
|
|
44
|
+
ERRORS = (OSError, sqlite3.Error)
|
|
45
|
+
# how often the WAL switch is retried while another connection holds the file
|
|
46
|
+
# (`_wal`)
|
|
47
|
+
LOCK_POLL = 0.02
|
|
48
|
+
|
|
49
|
+
# Each append-only taste table and the columns a query reads beside its JSON
|
|
50
|
+
# row; `n` is the append order.
|
|
51
|
+
TASTE_ROWS = {
|
|
52
|
+
"signals": ("session", "kind", "ts"),
|
|
53
|
+
"decisions": ("id", "session", "provider", "day", "at"),
|
|
54
|
+
"defects": ("id", "session", "day"),
|
|
55
|
+
"labels": ("id", "provider", "day"),
|
|
56
|
+
"injected": ("session", "day"),
|
|
57
|
+
}
|
|
58
|
+
IMPORT_SCHEMA = """
|
|
59
|
+
CREATE TABLE IF NOT EXISTS imported (name TEXT NOT NULL, dev INTEGER NOT NULL,
|
|
60
|
+
ino INTEGER NOT NULL, bytes INTEGER NOT NULL,
|
|
61
|
+
PRIMARY KEY (name, dev, ino));
|
|
62
|
+
CREATE TABLE IF NOT EXISTS import_failed (name TEXT PRIMARY KEY, mtime_ns INTEGER NOT NULL,
|
|
63
|
+
size INTEGER NOT NULL);
|
|
64
|
+
"""
|
|
65
|
+
USER_SCHEMA = """
|
|
66
|
+
CREATE TABLE IF NOT EXISTS learnings (id TEXT PRIMARY KEY, row TEXT NOT NULL);
|
|
67
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
68
|
+
""" + IMPORT_SCHEMA
|
|
69
|
+
REPO_SCHEMA = USER_SCHEMA + "".join(
|
|
70
|
+
"CREATE TABLE IF NOT EXISTS %s (n INTEGER PRIMARY KEY, %s, row TEXT NOT NULL);\n"
|
|
71
|
+
% (table, ", ".join(cols)) for table, cols in TASTE_ROWS.items()) + """
|
|
72
|
+
CREATE TABLE IF NOT EXISTS gate (one INTEGER PRIMARY KEY CHECK (one = 1),
|
|
73
|
+
stopped INTEGER NOT NULL, report TEXT NOT NULL);
|
|
74
|
+
CREATE TABLE IF NOT EXISTS notes_seen (session TEXT NOT NULL, learning TEXT NOT NULL,
|
|
75
|
+
PRIMARY KEY (session, learning));
|
|
76
|
+
"""
|
|
77
|
+
# The files each taste store replaces. A JSON document imports only into the
|
|
78
|
+
# tables it names while they are empty; a row file (`<table>.jsonl`) always.
|
|
79
|
+
DOC_TABLES = {"ledger.json": ("learnings", "meta"), "gate.json": ("gate",)}
|
|
80
|
+
REPO_LEGACY = ("ledger.json", "gate.json") + tuple(t + ".jsonl" for t in TASTE_ROWS)
|
|
81
|
+
USER_LEGACY = ("ledger.json",)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def connect(path, schema, version=0):
|
|
85
|
+
"""An autocommit connection to the database at `path` with `schema`
|
|
86
|
+
applied (idempotent). With `version`, the schema runs only while the file's
|
|
87
|
+
`user_version` is below it, so a hook's open skips the DDL and an older
|
|
88
|
+
install leaves a newer schema alone. Raises OSError or
|
|
89
|
+
sqlite3.Error."""
|
|
90
|
+
os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
|
|
91
|
+
os.close(os.open(path, os.O_RDWR | os.O_CREAT, 0o600))
|
|
92
|
+
conn = sqlite3.connect(path, isolation_level=None)
|
|
93
|
+
try:
|
|
94
|
+
conn.execute("PRAGMA busy_timeout = %d" % BUSY_MS)
|
|
95
|
+
_wal(conn)
|
|
96
|
+
if not version or conn.execute("PRAGMA user_version").fetchone()[0] < version:
|
|
97
|
+
# the write lock first, so a second opener waits on the busy
|
|
98
|
+
# timeout instead of failing on SQLite's lock-upgrade deadlock
|
|
99
|
+
conn.executescript("BEGIN IMMEDIATE;\n%s\nPRAGMA user_version = %d;\nCOMMIT;"
|
|
100
|
+
% (schema, version))
|
|
101
|
+
except sqlite3.Error:
|
|
102
|
+
if conn.in_transaction:
|
|
103
|
+
conn.execute("ROLLBACK")
|
|
104
|
+
conn.close()
|
|
105
|
+
raise
|
|
106
|
+
return conn
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _wal(conn):
|
|
110
|
+
"""Switch `conn`'s file to WAL. The switch needs the file to itself and
|
|
111
|
+
SQLite answers "locked" at once instead of waiting, so it is retried for as
|
|
112
|
+
long as the busy timeout would have waited."""
|
|
113
|
+
deadline = time.monotonic() + BUSY_MS / 1000.0
|
|
114
|
+
while True:
|
|
115
|
+
try:
|
|
116
|
+
conn.execute("PRAGMA journal_mode = WAL")
|
|
117
|
+
return
|
|
118
|
+
except sqlite3.OperationalError:
|
|
119
|
+
if time.monotonic() >= deadline:
|
|
120
|
+
raise
|
|
121
|
+
time.sleep(LOCK_POLL)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@contextlib.contextmanager
|
|
125
|
+
def transaction(conn):
|
|
126
|
+
"""One write transaction, taken at its start so two writers queue on the
|
|
127
|
+
busy timeout instead of failing at commit; rolled back on any failure,
|
|
128
|
+
the commit's own included."""
|
|
129
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
130
|
+
try:
|
|
131
|
+
yield conn
|
|
132
|
+
conn.execute("COMMIT")
|
|
133
|
+
except BaseException:
|
|
134
|
+
if conn.in_transaction:
|
|
135
|
+
conn.execute("ROLLBACK")
|
|
136
|
+
raise
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@contextlib.contextmanager
|
|
140
|
+
def taste(dirpath, user=False, create=True):
|
|
141
|
+
"""The taste database in `dirpath` (the user store when `user`), the legacy
|
|
142
|
+
files there imported first (`_import`). With `create` False it yields None
|
|
143
|
+
instead of creating a database where neither it nor a legacy file exists,
|
|
144
|
+
so a read leaves no file behind."""
|
|
145
|
+
path = os.path.join(dirpath, TASTE_DB)
|
|
146
|
+
legacy = USER_LEGACY if user else REPO_LEGACY
|
|
147
|
+
if not create and not os.path.exists(path) and not any(
|
|
148
|
+
os.path.exists(os.path.join(dirpath, name)) for name in legacy):
|
|
149
|
+
yield None
|
|
150
|
+
return
|
|
151
|
+
conn = connect(path, USER_SCHEMA if user else REPO_SCHEMA)
|
|
152
|
+
try:
|
|
153
|
+
_import(conn, dirpath, legacy)
|
|
154
|
+
yield conn
|
|
155
|
+
finally:
|
|
156
|
+
conn.close()
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _import(conn, dirpath, names):
|
|
160
|
+
"""Bring the legacy files present in `dirpath` into their tables. With no
|
|
161
|
+
legacy file, an open costs one stat per name.
|
|
162
|
+
|
|
163
|
+
A row file (`<table>.jsonl`) imports on every open that finds it, under
|
|
164
|
+
its old writer's flock. The `imported` table records how many bytes of
|
|
165
|
+
that file (by device and inode) went in, in the same transaction as the
|
|
166
|
+
rows, so the next open takes only what was appended since; a torn last
|
|
167
|
+
line waits for its newline. A JSON document imports only while its tables
|
|
168
|
+
are empty. The rename aside (`_aside`) comes after the commit and is
|
|
169
|
+
tidiness: a process killed before it re-imports nothing. A file that
|
|
170
|
+
cannot be read or parsed is recorded in `import_failed` and skipped,
|
|
171
|
+
before any transaction, until its mtime or size changes."""
|
|
172
|
+
for name in names:
|
|
173
|
+
src = os.path.join(dirpath, name)
|
|
174
|
+
if not os.path.exists(src):
|
|
175
|
+
continue
|
|
176
|
+
try:
|
|
177
|
+
st = os.stat(src)
|
|
178
|
+
except OSError:
|
|
179
|
+
continue
|
|
180
|
+
if conn.execute("SELECT 1 FROM import_failed WHERE name = ? AND mtime_ns = ? "
|
|
181
|
+
"AND size = ?", (name, st.st_mtime_ns, st.st_size)).fetchone():
|
|
182
|
+
continue
|
|
183
|
+
if name in DOC_TABLES:
|
|
184
|
+
_import_doc(conn, name, src, st)
|
|
185
|
+
else:
|
|
186
|
+
_import_rows(conn, name, src, st)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _failed(conn, name, st):
|
|
190
|
+
with transaction(conn):
|
|
191
|
+
conn.execute("INSERT OR REPLACE INTO import_failed (name, mtime_ns, size) "
|
|
192
|
+
"VALUES (?, ?, ?)", (name, st.st_mtime_ns, st.st_size))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _wanted(conn, name):
|
|
196
|
+
"""False for a JSON document whose tables already hold rows."""
|
|
197
|
+
return all(conn.execute("SELECT 1 FROM %s LIMIT 1" % table).fetchone() is None
|
|
198
|
+
for table in DOC_TABLES.get(name, ()))
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _aside(src, read):
|
|
202
|
+
"""Rename `src` to `<src>.imported`, or `.imported.N` beside an earlier one,
|
|
203
|
+
when it is still the file the import read (`read`, its fstat): one another
|
|
204
|
+
writer created at that path since was never read and stays. Never over a
|
|
205
|
+
file, so no import's source is ever lost. A failed rename is left for the
|
|
206
|
+
next open, which finds nothing new in the file."""
|
|
207
|
+
try:
|
|
208
|
+
if not os.path.samestat(os.stat(src), read):
|
|
209
|
+
return
|
|
210
|
+
except OSError:
|
|
211
|
+
return
|
|
212
|
+
dst, n = src + ".imported", 0
|
|
213
|
+
while os.path.exists(dst):
|
|
214
|
+
n += 1
|
|
215
|
+
dst = "%s.imported.%d" % (src, n)
|
|
216
|
+
try:
|
|
217
|
+
os.rename(src, dst)
|
|
218
|
+
except OSError:
|
|
219
|
+
pass
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _import_doc(conn, name, src, st):
|
|
223
|
+
if not _wanted(conn, name):
|
|
224
|
+
return
|
|
225
|
+
try:
|
|
226
|
+
with open(src, "rb") as fh:
|
|
227
|
+
read = os.fstat(fh.fileno())
|
|
228
|
+
data = json.loads(fh.read().decode("utf-8"))
|
|
229
|
+
except FileNotFoundError:
|
|
230
|
+
return # another process imported it since the stat
|
|
231
|
+
except (OSError, ValueError): # UnicodeDecodeError is a ValueError
|
|
232
|
+
data = None
|
|
233
|
+
if not isinstance(data, dict) or (
|
|
234
|
+
name == "ledger.json" and not isinstance(data.get("learnings"), dict)):
|
|
235
|
+
_failed(conn, name, st)
|
|
236
|
+
return
|
|
237
|
+
with transaction(conn):
|
|
238
|
+
if not _wanted(conn, name):
|
|
239
|
+
return # another process imported it first
|
|
240
|
+
if name == "ledger.json":
|
|
241
|
+
_write_ledger(conn, data)
|
|
242
|
+
else:
|
|
243
|
+
set_gate(conn, data.get("stopped"), data.get("report") or {})
|
|
244
|
+
_aside(src, read)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _locked(fh):
|
|
248
|
+
"""An exclusive flock on `fh` if one is free now, True where flock is
|
|
249
|
+
missing. Never waits: the caller holds the database's write lock."""
|
|
250
|
+
if fcntl is None:
|
|
251
|
+
return True
|
|
252
|
+
try:
|
|
253
|
+
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
254
|
+
return True
|
|
255
|
+
except OSError:
|
|
256
|
+
return False
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _taste_put(table):
|
|
260
|
+
"""The import of one taste row line: a line that is not a UTF-8 JSON object
|
|
261
|
+
is skipped, as the old reader skipped it."""
|
|
262
|
+
def put(conn, line):
|
|
263
|
+
try:
|
|
264
|
+
row = json.loads(line.decode("utf-8"))
|
|
265
|
+
except ValueError: # UnicodeDecodeError is one
|
|
266
|
+
return
|
|
267
|
+
if isinstance(row, dict):
|
|
268
|
+
append(conn, table, row)
|
|
269
|
+
return put
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _offset(conn, name, fh):
|
|
273
|
+
"""(the file's key, the byte offset its import resumes at, whether that
|
|
274
|
+
differs from the recorded one). A file cut below what went in (rewritten in
|
|
275
|
+
place) resumes at its end, so the rows appended after the cut still
|
|
276
|
+
import."""
|
|
277
|
+
st = os.fstat(fh.fileno())
|
|
278
|
+
key = (name, st.st_dev, st.st_ino)
|
|
279
|
+
found = conn.execute("SELECT bytes FROM imported WHERE name = ? AND dev = ? "
|
|
280
|
+
"AND ino = ?", key).fetchone()
|
|
281
|
+
start = min(found[0], st.st_size) if found else 0
|
|
282
|
+
return key, start, bool(found and found[0] != start)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _import_rows(conn, name, src, st, put=None, keep=False):
|
|
286
|
+
"""The complete lines of `src` past the recorded offset, each handed to
|
|
287
|
+
`put` (a taste table's row by default). With `keep` the file is never
|
|
288
|
+
renamed aside: its writer may still append to it.
|
|
289
|
+
|
|
290
|
+
A peek without a transaction comes first: a file with no complete line
|
|
291
|
+
past the offset (nothing new, or a torn fragment its writer has not
|
|
292
|
+
finished) costs no write lock, so a read under a busy writer still answers.
|
|
293
|
+
Then the database's write lock, and the file's flock only if it is free: an
|
|
294
|
+
import that held the flock while it waited on a busy database would hold
|
|
295
|
+
the file's writer for that wait. The offset is read again inside the
|
|
296
|
+
transaction, so two importers cannot both take the same lines, with or
|
|
297
|
+
without a flock."""
|
|
298
|
+
try:
|
|
299
|
+
fh = open(src, "rb")
|
|
300
|
+
except FileNotFoundError:
|
|
301
|
+
return # another process imported it since the stat
|
|
302
|
+
except OSError:
|
|
303
|
+
_failed(conn, name, st)
|
|
304
|
+
return
|
|
305
|
+
with fh: # closing it releases the flock
|
|
306
|
+
key, start, cut = _offset(conn, name, fh)
|
|
307
|
+
fh.seek(start)
|
|
308
|
+
raw = fh.read()
|
|
309
|
+
if b"\n" in raw or cut:
|
|
310
|
+
with transaction(conn):
|
|
311
|
+
if not _locked(fh):
|
|
312
|
+
return # its writer holds it: the next open takes it
|
|
313
|
+
key, start, cut = _offset(conn, name, fh)
|
|
314
|
+
fh.seek(start)
|
|
315
|
+
raw = fh.read()
|
|
316
|
+
whole = raw[:raw.rfind(b"\n") + 1]
|
|
317
|
+
if whole or cut:
|
|
318
|
+
put = put or _taste_put(name[:-len(".jsonl")])
|
|
319
|
+
for line in whole.split(b"\n"):
|
|
320
|
+
put(conn, line)
|
|
321
|
+
conn.execute("INSERT OR REPLACE INTO imported (name, dev, ino, bytes) "
|
|
322
|
+
"VALUES (?, ?, ?, ?)", key + (start + len(whole),))
|
|
323
|
+
# aside only when every byte went in (no fragment past the last newline)
|
|
324
|
+
# and no writer appended past the read (the old writer appends unlocked
|
|
325
|
+
# once its own wait runs out), under the flock so no writer is inside
|
|
326
|
+
# an append
|
|
327
|
+
if not keep and not raw[raw.rfind(b"\n") + 1:] and _locked(fh):
|
|
328
|
+
now = os.fstat(fh.fileno())
|
|
329
|
+
if now.st_size == start + len(raw):
|
|
330
|
+
_aside(src, now)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
# --- the taste tables ---------------------------------------------------------------
|
|
334
|
+
|
|
335
|
+
def _column(value):
|
|
336
|
+
return value if value is None or isinstance(value, (str, int, float)) else None
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def append(conn, table, row):
|
|
340
|
+
"""One row into an append-only taste table; raises when it is not written."""
|
|
341
|
+
cols = TASTE_ROWS[table]
|
|
342
|
+
conn.execute("INSERT INTO %s (%s, row) VALUES (%s)" % (
|
|
343
|
+
table, ", ".join(cols), ", ".join("?" * (len(cols) + 1))),
|
|
344
|
+
[_column(row.get(c)) for c in cols] + [json.dumps(row, ensure_ascii=False)])
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def rows(conn, table, where="", args=()):
|
|
348
|
+
"""The rows of one append-only taste table in append order; `where` is a
|
|
349
|
+
clause over its columns."""
|
|
350
|
+
return [json.loads(text) for (text,) in conn.execute(
|
|
351
|
+
"SELECT row FROM %s %s ORDER BY n" % (table, where), args)]
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def last(conn, table):
|
|
355
|
+
"""The append number of the table's newest row, 0 when it has none."""
|
|
356
|
+
return conn.execute("SELECT coalesce(max(n), 0) FROM %s" % table).fetchone()[0]
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def load_ledger(conn):
|
|
360
|
+
"""{"v": 1, "learnings": {id: learning}, "meta": {...}}, ids in order."""
|
|
361
|
+
return {"v": 1,
|
|
362
|
+
"learnings": {lid: json.loads(text) for lid, text in conn.execute(
|
|
363
|
+
"SELECT id, row FROM learnings ORDER BY id")},
|
|
364
|
+
"meta": {key: json.loads(value) for key, value in conn.execute(
|
|
365
|
+
"SELECT key, value FROM meta")}}
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _write_ledger(conn, ledger):
|
|
369
|
+
conn.execute("DELETE FROM learnings")
|
|
370
|
+
conn.executemany("INSERT INTO learnings (id, row) VALUES (?, ?)", [
|
|
371
|
+
(lid, json.dumps(learning, ensure_ascii=False, sort_keys=True))
|
|
372
|
+
for lid, learning in ledger["learnings"].items()])
|
|
373
|
+
conn.execute("DELETE FROM meta")
|
|
374
|
+
conn.executemany("INSERT INTO meta (key, value) VALUES (?, ?)", [
|
|
375
|
+
(key, json.dumps(value)) for key, value in (ledger.get("meta") or {}).items()])
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def save_ledger(conn, ledger):
|
|
379
|
+
"""Replace the learnings and their meta with `ledger`'s, in one transaction:
|
|
380
|
+
a forgotten learning leaves the table with it."""
|
|
381
|
+
with transaction(conn):
|
|
382
|
+
_write_ledger(conn, ledger)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def gate_stopped(conn):
|
|
386
|
+
"""True when the benefit gate stopped injection."""
|
|
387
|
+
found = conn.execute("SELECT stopped FROM gate").fetchone()
|
|
388
|
+
return bool(found and found[0])
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def set_gate(conn, stopped, report):
|
|
392
|
+
conn.execute("INSERT OR REPLACE INTO gate (one, stopped, report) VALUES (1, ?, ?)",
|
|
393
|
+
(1 if stopped else 0, json.dumps(report)))
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def seen(conn, session):
|
|
397
|
+
"""The learning ids a write note already showed in `session`."""
|
|
398
|
+
return {lid for (lid,) in conn.execute(
|
|
399
|
+
"SELECT learning FROM notes_seen WHERE session = ?", (session,))}
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def mark_seen(conn, session, ids):
|
|
403
|
+
with transaction(conn):
|
|
404
|
+
conn.executemany("INSERT OR IGNORE INTO notes_seen (session, learning) VALUES (?, ?)",
|
|
405
|
+
[(session, lid) for lid in ids])
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
# --- the cache database -------------------------------------------------------------
|
|
409
|
+
# One database per cache dir, `<cache>/tezgah.db`: the evidence ledger and the
|
|
410
|
+
# small cache stores below it (CACHE_SCHEMA).
|
|
411
|
+
#
|
|
412
|
+
# The evidence ledger: a caller names a session's ledger by the path its JSONL
|
|
413
|
+
# file had, `<cache>/evidence/<session>.jsonl` (`tezgah_integrity._path`): the
|
|
414
|
+
# cache is two directories up and the session is the file's stem. A JSONL file
|
|
415
|
+
# still at that path is the session's legacy file, written before this database
|
|
416
|
+
# (or by an opencode process still running a plugin from before it), imported
|
|
417
|
+
# past the bytes already in before every read or write of that session and
|
|
418
|
+
# never renamed aside, because that writer may still append to it.
|
|
419
|
+
CACHE_DB = "tezgah.db"
|
|
420
|
+
CACHE_VERSION = 2
|
|
421
|
+
EVIDENCE_SCHEMA = IMPORT_SCHEMA + """
|
|
422
|
+
CREATE TABLE IF NOT EXISTS evidence (n INTEGER PRIMARY KEY, session TEXT NOT NULL,
|
|
423
|
+
kind TEXT, ts INTEGER, row TEXT NOT NULL);
|
|
424
|
+
CREATE INDEX IF NOT EXISTS evidence_session ON evidence (session, n);
|
|
425
|
+
CREATE INDEX IF NOT EXISTS evidence_ts ON evidence (ts);
|
|
426
|
+
"""
|
|
427
|
+
# The cache stores, each the table of one kind of file the hooks wrote before
|
|
428
|
+
# (CACHE_LEGACY). A document store keeps one JSON `doc` per `key`, a mark store
|
|
429
|
+
# only the key; `at` is the epoch second of the last write, the mtime the file
|
|
430
|
+
# had. The key is the name the file had, so a legacy file imports under it.
|
|
431
|
+
DOC_STORES = ("turns", "switches", "harness_drift", "import_crash", "judge_last",
|
|
432
|
+
"skill_pick", "section_hint", "update_check", "cursor_answer",
|
|
433
|
+
"taste_learn")
|
|
434
|
+
MARK_STORES = ("gate_inactive", "nudged", "cursor_reinforced")
|
|
435
|
+
CACHE_SCHEMA = EVIDENCE_SCHEMA + "".join(
|
|
436
|
+
"CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, at REAL NOT NULL, doc TEXT NOT NULL);\n"
|
|
437
|
+
% table for table in DOC_STORES) + "".join(
|
|
438
|
+
"CREATE TABLE IF NOT EXISTS %s (key TEXT PRIMARY KEY, at REAL NOT NULL);\n"
|
|
439
|
+
% table for table in MARK_STORES) + """
|
|
440
|
+
CREATE TABLE IF NOT EXISTS used (session TEXT NOT NULL, kind TEXT NOT NULL, at REAL NOT NULL,
|
|
441
|
+
PRIMARY KEY (session, kind));
|
|
442
|
+
CREATE TABLE IF NOT EXISTS judge_down (key TEXT PRIMARY KEY, until REAL NOT NULL);
|
|
443
|
+
CREATE TABLE IF NOT EXISTS lesson_taint (root TEXT NOT NULL, key TEXT NOT NULL,
|
|
444
|
+
source TEXT, ts INTEGER, PRIMARY KEY (root, key));
|
|
445
|
+
CREATE TABLE IF NOT EXISTS snapshots (n INTEGER PRIMARY KEY, id TEXT NOT NULL UNIQUE,
|
|
446
|
+
meta TEXT NOT NULL);
|
|
447
|
+
CREATE TABLE IF NOT EXISTS replay_runs (n INTEGER PRIMARY KEY, run TEXT NOT NULL UNIQUE,
|
|
448
|
+
summary TEXT NOT NULL);
|
|
449
|
+
"""
|
|
450
|
+
# Where each cache store's files were, under the cache dir (`_import_cache`).
|
|
451
|
+
CACHE_LEGACY = {
|
|
452
|
+
"used": "sessions", "turns": "turns", "switches": "switches",
|
|
453
|
+
"gate_inactive": "gate-inactive", "nudged": "nudged", "harness_drift": "harness-drift",
|
|
454
|
+
"import_crash": "import-crash", "judge_down": "judge-down",
|
|
455
|
+
"judge_last": "judge-last.json", "skill_pick": "skill-pick",
|
|
456
|
+
"section_hint": "skill-section-hint", "update_check": "update.json",
|
|
457
|
+
"lesson_taint": "lessons", "cursor_reinforced": "reinforced", "cursor_answer": "answer",
|
|
458
|
+
"replay_runs": "replay", "snapshots": "snapshots", "taste_learn": "taste-learn",
|
|
459
|
+
}
|
|
460
|
+
# tezgah_judge.DOWN_FOR when the judge-down markers were files: a marker's
|
|
461
|
+
# mtime plus this is the expiry it imports with
|
|
462
|
+
LEGACY_DOWN_FOR = 300
|
|
463
|
+
# the bulk import's stamp in the cache dir, and how often session start may
|
|
464
|
+
# launch it (`import_later`)
|
|
465
|
+
IMPORT_STAMP = "evidence-import.stamp"
|
|
466
|
+
IMPORT_EVERY = 86400
|
|
467
|
+
# One connection per database per process: db path -> (connection, the file's
|
|
468
|
+
# (dev, ino), {legacy path: its (dev, ino, size) once fully imported}). A
|
|
469
|
+
# database removed or replaced under the process (doctor, uninstall, a test's
|
|
470
|
+
# fresh cache) is reopened on the next call.
|
|
471
|
+
_HELD = {}
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _ledger(path):
|
|
475
|
+
"""(cache dir, session) for a ledger path."""
|
|
476
|
+
folder, name = os.path.split(path)
|
|
477
|
+
return os.path.dirname(folder), name[:-len(".jsonl")] if name.endswith(".jsonl") else name
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _cache_db(cache, create=True):
|
|
481
|
+
"""The held database of `cache`, the cache stores' legacy files imported
|
|
482
|
+
on its open (`_import_cache`); None when it does not exist and not
|
|
483
|
+
`create`."""
|
|
484
|
+
path = os.path.join(cache, CACHE_DB)
|
|
485
|
+
try:
|
|
486
|
+
st = os.stat(path)
|
|
487
|
+
here = (st.st_dev, st.st_ino)
|
|
488
|
+
except OSError:
|
|
489
|
+
here = None
|
|
490
|
+
held = _HELD.get(path)
|
|
491
|
+
if held and held[1] == here:
|
|
492
|
+
return held
|
|
493
|
+
if held:
|
|
494
|
+
del _HELD[path]
|
|
495
|
+
held[0].close()
|
|
496
|
+
if here is None and not create:
|
|
497
|
+
return None
|
|
498
|
+
conn = connect(path, CACHE_SCHEMA, CACHE_VERSION)
|
|
499
|
+
# no fsync per commit under WAL: a crash can lose the newest rows but never
|
|
500
|
+
# corrupts the file, the durability the JSONL append had
|
|
501
|
+
conn.execute("PRAGMA synchronous = NORMAL")
|
|
502
|
+
st = os.stat(path)
|
|
503
|
+
held = _HELD[path] = (conn, (st.st_dev, st.st_ino), {})
|
|
504
|
+
_import_cache(conn, cache)
|
|
505
|
+
return held
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _insert(conn, session, text):
|
|
509
|
+
"""One evidence row: `text` verbatim, its `kind` and `ts` beside it when it
|
|
510
|
+
is a JSON object that carries them."""
|
|
511
|
+
try:
|
|
512
|
+
row = json.loads(text)
|
|
513
|
+
except ValueError:
|
|
514
|
+
row = None
|
|
515
|
+
row = row if isinstance(row, dict) else {}
|
|
516
|
+
kind, ts = row.get("kind"), row.get("ts")
|
|
517
|
+
try:
|
|
518
|
+
ts = int(ts) if isinstance(ts, (int, float)) and not isinstance(ts, bool) else None
|
|
519
|
+
except (ValueError, OverflowError): # NaN, infinity
|
|
520
|
+
ts = None
|
|
521
|
+
conn.execute("INSERT INTO evidence (session, kind, ts, row) VALUES (?, ?, ?, ?)",
|
|
522
|
+
(session, kind if isinstance(kind, str) else None,
|
|
523
|
+
ts if ts is not None and -2 ** 63 <= ts < 2 ** 63 else None, text))
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def _evidence_put(session):
|
|
527
|
+
"""The import of one legacy ledger line: kept as it was written, so a line
|
|
528
|
+
that does not parse stays a line the reader names as damage."""
|
|
529
|
+
def put(conn, line):
|
|
530
|
+
if not line.strip():
|
|
531
|
+
return
|
|
532
|
+
try:
|
|
533
|
+
text = line.decode("utf-8")
|
|
534
|
+
except UnicodeDecodeError:
|
|
535
|
+
# the marker `tezgah_integrity._parse` names as damage, hashed
|
|
536
|
+
# over the line with its newline as the file reader hashed it, so
|
|
537
|
+
# a damage row it wrote still names the same line
|
|
538
|
+
text = "\x00not utf-8 %s" % hashlib.sha1(line + b"\n").hexdigest()[:12]
|
|
539
|
+
_insert(conn, session, text)
|
|
540
|
+
return put
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _sync(held, path):
|
|
544
|
+
"""Import the legacy JSONL at `path` past the bytes already in. Without a
|
|
545
|
+
file, or with nothing new in it, this is one stat (and one lookup the first
|
|
546
|
+
time a process sees it)."""
|
|
547
|
+
try:
|
|
548
|
+
st = os.stat(path)
|
|
549
|
+
except OSError:
|
|
550
|
+
return
|
|
551
|
+
conn, _here, seen = held
|
|
552
|
+
mark = (st.st_dev, st.st_ino, st.st_size)
|
|
553
|
+
if seen.get(path) == mark:
|
|
554
|
+
return
|
|
555
|
+
name = os.path.basename(path)
|
|
556
|
+
found = conn.execute("SELECT bytes FROM imported WHERE name = ? AND dev = ? AND ino = ?",
|
|
557
|
+
(name, st.st_dev, st.st_ino)).fetchone()
|
|
558
|
+
if found and found[0] == st.st_size:
|
|
559
|
+
seen[path] = mark
|
|
560
|
+
return
|
|
561
|
+
if conn.execute("SELECT 1 FROM import_failed WHERE name = ? AND mtime_ns = ? "
|
|
562
|
+
"AND size = ?", (name, st.st_mtime_ns, st.st_size)).fetchone():
|
|
563
|
+
return
|
|
564
|
+
_import_rows(conn, name, path, st, _evidence_put(_ledger(path)[1]), keep=True)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def _session(path, create=False):
|
|
568
|
+
"""(held database, session) for the ledger at `path`, its legacy file
|
|
569
|
+
imported first; the database is None when neither it nor a legacy file
|
|
570
|
+
exists and not `create`, so a read leaves no file behind."""
|
|
571
|
+
cache, session = _ledger(path)
|
|
572
|
+
held = _cache_db(cache, create or os.path.exists(path))
|
|
573
|
+
if held:
|
|
574
|
+
_sync(held, path)
|
|
575
|
+
return held, session
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def import_session(path):
|
|
579
|
+
"""The legacy JSONL at `path` into its cache's database."""
|
|
580
|
+
if os.path.exists(path):
|
|
581
|
+
_session(path)
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def append_evidence(path, text):
|
|
585
|
+
"""One row (`text`, its JSON) into the ledger at `path`, after its legacy
|
|
586
|
+
file's rows: one INSERT in autocommit, so a row is whole or absent. Raises
|
|
587
|
+
OSError or sqlite3.Error."""
|
|
588
|
+
held, session = _session(path, create=True)
|
|
589
|
+
_insert(held[0], session, text)
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def evidence_rows(path, tail=None, kind=None):
|
|
593
|
+
"""The row texts of the ledger at `path`, oldest first: the last `tail`
|
|
594
|
+
with one, only the rows of `kind` with that."""
|
|
595
|
+
held, session = _session(path)
|
|
596
|
+
if held is None:
|
|
597
|
+
return []
|
|
598
|
+
conn = held[0]
|
|
599
|
+
if kind:
|
|
600
|
+
found = conn.execute("SELECT row FROM evidence WHERE session = ? AND kind = ? "
|
|
601
|
+
"ORDER BY n", (session, kind))
|
|
602
|
+
elif tail:
|
|
603
|
+
return [text for (text,) in reversed(conn.execute(
|
|
604
|
+
"SELECT row FROM evidence WHERE session = ? ORDER BY n DESC LIMIT ?",
|
|
605
|
+
(session, tail)).fetchall())]
|
|
606
|
+
else:
|
|
607
|
+
found = conn.execute("SELECT row FROM evidence WHERE session = ? ORDER BY n",
|
|
608
|
+
(session,))
|
|
609
|
+
return [text for (text,) in found]
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def evidence_first(path):
|
|
613
|
+
"""The first row text of the ledger at `path`, or None."""
|
|
614
|
+
held, session = _session(path)
|
|
615
|
+
found = held and held[0].execute(
|
|
616
|
+
"SELECT row FROM evidence WHERE session = ? ORDER BY n LIMIT 1", (session,)).fetchone()
|
|
617
|
+
return found[0] if found else None
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def has_evidence(path):
|
|
621
|
+
"""True when the ledger at `path` has a legacy file or a row; creates
|
|
622
|
+
nothing."""
|
|
623
|
+
if os.path.exists(path):
|
|
624
|
+
return True
|
|
625
|
+
cache, session = _ledger(path)
|
|
626
|
+
held = _cache_db(cache, create=False)
|
|
627
|
+
return bool(held and held[0].execute(
|
|
628
|
+
"SELECT 1 FROM evidence WHERE session = ? LIMIT 1", (session,)).fetchone())
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def import_evidence(cache):
|
|
632
|
+
"""Every legacy session file under `<cache>/evidence` into the cache's
|
|
633
|
+
database, the least recently written first; the count of files seen.
|
|
634
|
+
Idempotent: a file imports only past the bytes the `imported` table
|
|
635
|
+
already counts for it."""
|
|
636
|
+
found = []
|
|
637
|
+
try:
|
|
638
|
+
for entry in os.scandir(os.path.join(cache, "evidence")):
|
|
639
|
+
if entry.name.endswith(".jsonl"):
|
|
640
|
+
try:
|
|
641
|
+
found.append((entry.stat().st_mtime, entry.path))
|
|
642
|
+
except OSError:
|
|
643
|
+
continue
|
|
644
|
+
except OSError:
|
|
645
|
+
return 0
|
|
646
|
+
if found:
|
|
647
|
+
held = _cache_db(cache)
|
|
648
|
+
for _mtime, path in sorted(found):
|
|
649
|
+
_sync(held, path)
|
|
650
|
+
return len(found)
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def sessions(cache):
|
|
654
|
+
"""[(session, its newest ts)] of every ledger in `cache`, the one written
|
|
655
|
+
last first, after the bulk import (`import_evidence`)."""
|
|
656
|
+
import_evidence(cache)
|
|
657
|
+
held = _cache_db(cache, create=False)
|
|
658
|
+
return held[0].execute(
|
|
659
|
+
"SELECT session, max(ts) FROM evidence GROUP BY session "
|
|
660
|
+
"ORDER BY max(ts) DESC, max(n) DESC").fetchall() if held else []
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def recent_rows(cache, since, kinds, but):
|
|
664
|
+
"""(session, ts, row text) of every row of `kinds` stamped at or after `since`
|
|
665
|
+
in a ledger other than session `but`, oldest first. The legacy files
|
|
666
|
+
written since are imported first; the older ones cannot hold such a row."""
|
|
667
|
+
fresh = []
|
|
668
|
+
try:
|
|
669
|
+
for entry in os.scandir(os.path.join(cache, "evidence")):
|
|
670
|
+
try:
|
|
671
|
+
if entry.name.endswith(".jsonl") and entry.stat().st_mtime >= since:
|
|
672
|
+
fresh.append(entry.path)
|
|
673
|
+
except OSError:
|
|
674
|
+
continue
|
|
675
|
+
except OSError:
|
|
676
|
+
pass
|
|
677
|
+
held = _cache_db(cache, create=bool(fresh))
|
|
678
|
+
if held is None:
|
|
679
|
+
return []
|
|
680
|
+
for path in fresh:
|
|
681
|
+
_sync(held, path)
|
|
682
|
+
return held[0].execute(
|
|
683
|
+
"SELECT session, ts, row FROM evidence WHERE ts >= ? AND kind IN (%s) AND session != ? "
|
|
684
|
+
"ORDER BY n" % ", ".join("?" * len(kinds)), (since,) + tuple(kinds) + (but,)).fetchall()
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _forget(conn, cache, session):
|
|
688
|
+
"""Delete one session's rows, its import offsets and its legacy file (inside
|
|
689
|
+
the caller's transaction); the count of rows deleted. The offsets go with the
|
|
690
|
+
file: a later file at that name may reuse its inode."""
|
|
691
|
+
rows = conn.execute("DELETE FROM evidence WHERE session = ?", (session,)).rowcount
|
|
692
|
+
conn.execute("DELETE FROM imported WHERE name = ?", (session + ".jsonl",))
|
|
693
|
+
try:
|
|
694
|
+
os.remove(os.path.join(cache, "evidence", session + ".jsonl"))
|
|
695
|
+
except OSError:
|
|
696
|
+
pass
|
|
697
|
+
return rows
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def forget_session(path):
|
|
701
|
+
"""Delete the ledger at `path`: its rows and its legacy file; the count of
|
|
702
|
+
rows deleted."""
|
|
703
|
+
cache, session = _ledger(path)
|
|
704
|
+
held = _cache_db(cache, create=False)
|
|
705
|
+
if held is None:
|
|
706
|
+
try:
|
|
707
|
+
os.remove(path)
|
|
708
|
+
except OSError:
|
|
709
|
+
pass
|
|
710
|
+
return 0
|
|
711
|
+
with transaction(held[0]) as conn:
|
|
712
|
+
return _forget(conn, cache, session)
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def sweep_evidence(cache, cutoff, keep=None):
|
|
716
|
+
"""Delete every ledger in `cache` whose newest row is older than `cutoff`
|
|
717
|
+
(epoch seconds), all but session `keep`: (sessions, rows) deleted. A
|
|
718
|
+
session is the unit, as the file was: a ledger cut short at its start would
|
|
719
|
+
lose the first row the switch latch reads."""
|
|
720
|
+
import_evidence(cache)
|
|
721
|
+
held = _cache_db(cache, create=False)
|
|
722
|
+
if held is None:
|
|
723
|
+
return 0, 0
|
|
724
|
+
with transaction(held[0]) as conn:
|
|
725
|
+
idle = [s for (s,) in conn.execute(
|
|
726
|
+
"SELECT session FROM evidence GROUP BY session HAVING max(coalesce(ts, 0)) < ?",
|
|
727
|
+
(cutoff,)).fetchall() if s != keep]
|
|
728
|
+
return len(idle), sum(_forget(conn, cache, s) for s in idle)
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def import_later(cache):
|
|
732
|
+
"""Start `import-evidence` for `cache` detached, while a legacy file exists
|
|
733
|
+
and at most once per IMPORT_EVERY (the stamp's mtime, written before the
|
|
734
|
+
start, so a failed start waits a day). Total."""
|
|
735
|
+
try:
|
|
736
|
+
with os.scandir(os.path.join(cache, "evidence")) as found:
|
|
737
|
+
if not any(entry.name.endswith(".jsonl") for entry in found):
|
|
738
|
+
return
|
|
739
|
+
stamp = os.path.join(cache, IMPORT_STAMP)
|
|
740
|
+
try:
|
|
741
|
+
if time.time() - os.path.getmtime(stamp) < IMPORT_EVERY:
|
|
742
|
+
return
|
|
743
|
+
except OSError:
|
|
744
|
+
pass
|
|
745
|
+
with open(stamp, "wb"):
|
|
746
|
+
pass
|
|
747
|
+
subprocess.Popen([sys.executable, os.path.abspath(__file__), "import-evidence", cache],
|
|
748
|
+
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
|
749
|
+
stderr=subprocess.DEVNULL, start_new_session=True)
|
|
750
|
+
except (OSError, ValueError):
|
|
751
|
+
pass
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
# --- the cache stores ---------------------------------------------------------------
|
|
755
|
+
# Each function takes the cache dir as `cache`, `tezgah_paths.cache_dir()` when
|
|
756
|
+
# it is None. A read never creates the database: with neither it nor the
|
|
757
|
+
# store's legacy files there, it answers as an empty store.
|
|
758
|
+
|
|
759
|
+
def _cache_dir():
|
|
760
|
+
import tezgah_paths
|
|
761
|
+
return tezgah_paths.cache_dir()
|
|
762
|
+
|
|
763
|
+
|
|
764
|
+
def _store(cache, table, create):
|
|
765
|
+
"""The connection of the database holding `table`, or None when it does
|
|
766
|
+
not exist, the store has no legacy file to import, and not `create`."""
|
|
767
|
+
cache = cache or _cache_dir()
|
|
768
|
+
held = _cache_db(cache, create=False) or (
|
|
769
|
+
(create or os.path.exists(os.path.join(cache, CACHE_LEGACY[table])))
|
|
770
|
+
and _cache_db(cache))
|
|
771
|
+
return held[0] if held else None
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def doc(table, key, cache=None):
|
|
775
|
+
"""The JSON value a document store holds under `key`, or None: none
|
|
776
|
+
stored, or the database cannot be read."""
|
|
777
|
+
try:
|
|
778
|
+
conn = _store(cache, table, False)
|
|
779
|
+
found = conn and conn.execute("SELECT doc FROM %s WHERE key = ?" % table,
|
|
780
|
+
(key,)).fetchone()
|
|
781
|
+
return json.loads(found[0]) if found else None
|
|
782
|
+
except ERRORS + (ValueError,):
|
|
783
|
+
return None
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def put_doc(table, key, value, cache=None):
|
|
787
|
+
"""`value` (JSON) under `key` in a document store; False when it was not
|
|
788
|
+
written."""
|
|
789
|
+
try:
|
|
790
|
+
_store(cache, table, True).execute(
|
|
791
|
+
"INSERT OR REPLACE INTO %s (key, at, doc) VALUES (?, ?, ?)" % table,
|
|
792
|
+
(key, time.time(), json.dumps(value)))
|
|
793
|
+
return True
|
|
794
|
+
except ERRORS + (TypeError, ValueError):
|
|
795
|
+
return False
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def drop(table, key, cache=None):
|
|
799
|
+
"""`key` out of a document or mark store. Total."""
|
|
800
|
+
try:
|
|
801
|
+
conn = _store(cache, table, False)
|
|
802
|
+
if conn:
|
|
803
|
+
conn.execute("DELETE FROM %s WHERE key = ?" % table, (key,))
|
|
804
|
+
except ERRORS:
|
|
805
|
+
pass
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def mark(table, key, cache=None):
|
|
809
|
+
"""Set `key` in a mark store: True when this call set it, False when it
|
|
810
|
+
was set already. One INSERT, so of two callers exactly one gets True.
|
|
811
|
+
Raises ERRORS."""
|
|
812
|
+
return _store(cache, table, True).execute(
|
|
813
|
+
"INSERT OR IGNORE INTO %s (key, at) VALUES (?, ?)" % table,
|
|
814
|
+
(key, time.time())).rowcount == 1
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def marked(table, key, cache=None):
|
|
818
|
+
"""True when a mark store holds `key`. Total: an unreadable store is
|
|
819
|
+
unmarked."""
|
|
820
|
+
try:
|
|
821
|
+
conn = _store(cache, table, False)
|
|
822
|
+
return bool(conn and conn.execute("SELECT 1 FROM %s WHERE key = ?" % table,
|
|
823
|
+
(key,)).fetchone())
|
|
824
|
+
except ERRORS:
|
|
825
|
+
return False
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
def use(session, kind, cache=None):
|
|
829
|
+
"""Record that `session` used `kind`, stamped now. Raises ERRORS."""
|
|
830
|
+
_store(cache, "used", True).execute(
|
|
831
|
+
"INSERT OR REPLACE INTO used (session, kind, at) VALUES (?, ?, ?)",
|
|
832
|
+
(session, kind, time.time()))
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def used(session, cache=None):
|
|
836
|
+
"""The kinds `session` used, a set. Total."""
|
|
837
|
+
try:
|
|
838
|
+
conn = _store(cache, "used", False)
|
|
839
|
+
return {kind for (kind,) in conn.execute(
|
|
840
|
+
"SELECT kind FROM used WHERE session = ?", (session,))} if conn else set()
|
|
841
|
+
except ERRORS:
|
|
842
|
+
return set()
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def used_sessions(window, cache=None):
|
|
846
|
+
"""(the kind sets of the `window` sessions that recorded last, newest
|
|
847
|
+
first; the count of sessions recorded). Total."""
|
|
848
|
+
try:
|
|
849
|
+
conn = _store(cache, "used", False)
|
|
850
|
+
if not conn:
|
|
851
|
+
return [], 0
|
|
852
|
+
newest = [s for (s,) in conn.execute(
|
|
853
|
+
"SELECT session FROM used GROUP BY session ORDER BY max(at) DESC LIMIT ?",
|
|
854
|
+
(window,))]
|
|
855
|
+
total = conn.execute("SELECT count(DISTINCT session) FROM used").fetchone()[0]
|
|
856
|
+
return [used(s, cache) for s in newest], total
|
|
857
|
+
except ERRORS:
|
|
858
|
+
return [], 0
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
def down_until(key, cache=None):
|
|
862
|
+
"""The epoch second the judge-down mark `key` expires at, or None. Total."""
|
|
863
|
+
try:
|
|
864
|
+
conn = _store(cache, "judge_down", False)
|
|
865
|
+
found = conn and conn.execute("SELECT until FROM judge_down WHERE key = ?",
|
|
866
|
+
(key,)).fetchone()
|
|
867
|
+
return found[0] if found else None
|
|
868
|
+
except ERRORS:
|
|
869
|
+
return None
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def set_down(key, until, cache=None):
|
|
873
|
+
"""Mark `key` down until the epoch second `until`. Total."""
|
|
874
|
+
try:
|
|
875
|
+
_store(cache, "judge_down", True).execute(
|
|
876
|
+
"INSERT OR REPLACE INTO judge_down (key, until) VALUES (?, ?)", (key, until))
|
|
877
|
+
except ERRORS:
|
|
878
|
+
pass
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def taint(root, cache=None):
|
|
882
|
+
"""{lesson key: source} of the lesson-taint rows of `root` (a key the
|
|
883
|
+
caller derives from the repository). Total."""
|
|
884
|
+
try:
|
|
885
|
+
conn = _store(cache, "lesson_taint", False)
|
|
886
|
+
return {key: source or "" for key, source in conn.execute(
|
|
887
|
+
"SELECT key, source FROM lesson_taint WHERE root = ?", (root,))} if conn else {}
|
|
888
|
+
except ERRORS:
|
|
889
|
+
return {}
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def write_taint(root, keep, rows, cache=None):
|
|
893
|
+
"""In one transaction: drop the rows of `root` whose key is not in `keep`,
|
|
894
|
+
then store `rows` ({key, source, ts}), a later row over an earlier one of
|
|
895
|
+
its key. Total."""
|
|
896
|
+
try:
|
|
897
|
+
conn = _store(cache, "lesson_taint", True)
|
|
898
|
+
with transaction(conn):
|
|
899
|
+
gone = [(root, key) for (key,) in conn.execute(
|
|
900
|
+
"SELECT key FROM lesson_taint WHERE root = ?", (root,)) if key not in keep]
|
|
901
|
+
conn.executemany("DELETE FROM lesson_taint WHERE root = ? AND key = ?", gone)
|
|
902
|
+
conn.executemany(
|
|
903
|
+
"INSERT OR REPLACE INTO lesson_taint (root, key, source, ts) VALUES (?, ?, ?, ?)",
|
|
904
|
+
[(root, str(r["key"]), _column(r.get("source")), _column(r.get("ts")))
|
|
905
|
+
for r in rows])
|
|
906
|
+
except ERRORS:
|
|
907
|
+
pass
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
def add_snapshot(sid, meta, cap, cache=None):
|
|
911
|
+
"""Index snapshot `sid` (its manifest `meta`) and drop the oldest rows past
|
|
912
|
+
`cap`, in one transaction; the ids dropped, whose blobs the caller removes.
|
|
913
|
+
Raises ERRORS."""
|
|
914
|
+
conn = _store(cache, "snapshots", True)
|
|
915
|
+
with transaction(conn):
|
|
916
|
+
conn.execute("INSERT INTO snapshots (id, meta) VALUES (?, ?)", (sid, json.dumps(meta)))
|
|
917
|
+
old = [i for (i,) in conn.execute(
|
|
918
|
+
"SELECT id FROM snapshots ORDER BY n DESC LIMIT -1 OFFSET ?", (cap,))]
|
|
919
|
+
conn.executemany("DELETE FROM snapshots WHERE id = ?", [(i,) for i in old])
|
|
920
|
+
return old
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def snapshots(sid=None, cache=None):
|
|
924
|
+
"""The snapshot manifests, oldest first, or the one of `sid`. Total."""
|
|
925
|
+
try:
|
|
926
|
+
conn = _store(cache, "snapshots", False)
|
|
927
|
+
found = [] if conn is None else conn.execute(
|
|
928
|
+
"SELECT meta FROM snapshots %s ORDER BY n" % ("WHERE id = ?" if sid else ""),
|
|
929
|
+
(sid,) if sid else ())
|
|
930
|
+
return [json.loads(meta) for (meta,) in found]
|
|
931
|
+
except ERRORS + (ValueError,):
|
|
932
|
+
return []
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def add_replay(run, summary, cache=None):
|
|
936
|
+
"""Index replay run `run` (its directory) with its summary; the newest
|
|
937
|
+
indexed run is the latest. Raises ERRORS."""
|
|
938
|
+
_store(cache, "replay_runs", True).execute(
|
|
939
|
+
"INSERT OR REPLACE INTO replay_runs (run, summary) VALUES (?, ?)",
|
|
940
|
+
(run, json.dumps(summary)))
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def replay_latest(cache=None):
|
|
944
|
+
"""The directory of the latest replay run, or None. Total."""
|
|
945
|
+
try:
|
|
946
|
+
conn = _store(cache, "replay_runs", False)
|
|
947
|
+
found = conn and conn.execute(
|
|
948
|
+
"SELECT run FROM replay_runs ORDER BY n DESC LIMIT 1").fetchone()
|
|
949
|
+
return found[0] if found else None
|
|
950
|
+
except ERRORS:
|
|
951
|
+
return None
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
def replay_summary(run, cache=None):
|
|
955
|
+
"""The summary replay run `run` was indexed with, or None. Total."""
|
|
956
|
+
try:
|
|
957
|
+
conn = _store(cache, "replay_runs", False)
|
|
958
|
+
found = conn and conn.execute("SELECT summary FROM replay_runs WHERE run = ?",
|
|
959
|
+
(run,)).fetchone()
|
|
960
|
+
return json.loads(found[0]) if found else None
|
|
961
|
+
except ERRORS + (ValueError,):
|
|
962
|
+
return None
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def sweep_cache(cache, cutoff, keep=None):
|
|
966
|
+
"""Delete the turn stamps and the used-kind sessions of `cache` last
|
|
967
|
+
written before `cutoff` (epoch seconds), all but session `keep` (its
|
|
968
|
+
`tezgah_context.slug`), after importing their legacy files; the count of
|
|
969
|
+
rows deleted. The two stores the doctor swept as files. Raises ERRORS."""
|
|
970
|
+
conn = _store(cache, "turns", False) or _store(cache, "used", False)
|
|
971
|
+
if conn is None:
|
|
972
|
+
return 0
|
|
973
|
+
with transaction(conn):
|
|
974
|
+
rows = conn.execute("DELETE FROM turns WHERE at < ? AND key IS NOT ?",
|
|
975
|
+
(cutoff, keep)).rowcount
|
|
976
|
+
return rows + conn.execute(
|
|
977
|
+
"DELETE FROM used WHERE session IN (SELECT session FROM used GROUP BY session "
|
|
978
|
+
"HAVING max(at) < ?) AND session IS NOT ?", (cutoff, keep)).rowcount
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def _files(folder):
|
|
982
|
+
"""[(name, path, mtime)] of the regular files directly in `folder`."""
|
|
983
|
+
out = []
|
|
984
|
+
try:
|
|
985
|
+
with os.scandir(folder) as found:
|
|
986
|
+
for entry in found:
|
|
987
|
+
try:
|
|
988
|
+
if entry.is_file(follow_symlinks=False):
|
|
989
|
+
out.append((entry.name, entry.path, entry.stat().st_mtime))
|
|
990
|
+
except OSError:
|
|
991
|
+
continue
|
|
992
|
+
except OSError:
|
|
993
|
+
pass
|
|
994
|
+
return out
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def _dirs(folder):
|
|
998
|
+
"""[(mtime_ns, name, path)] of the directories directly in `folder`, oldest
|
|
999
|
+
first."""
|
|
1000
|
+
out = []
|
|
1001
|
+
try:
|
|
1002
|
+
with os.scandir(folder) as found:
|
|
1003
|
+
for entry in found:
|
|
1004
|
+
try:
|
|
1005
|
+
if entry.is_dir(follow_symlinks=False):
|
|
1006
|
+
out.append((entry.stat().st_mtime_ns, entry.name, entry.path))
|
|
1007
|
+
except OSError:
|
|
1008
|
+
continue
|
|
1009
|
+
except OSError:
|
|
1010
|
+
pass
|
|
1011
|
+
return sorted(out)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def _read_text(path):
|
|
1015
|
+
try:
|
|
1016
|
+
with open(path, encoding="utf-8") as fh:
|
|
1017
|
+
return fh.read()
|
|
1018
|
+
except (OSError, ValueError): # UnicodeDecodeError is a ValueError
|
|
1019
|
+
return None
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def _read_json(path):
|
|
1023
|
+
try:
|
|
1024
|
+
return json.loads(_read_text(path) or "")
|
|
1025
|
+
except ValueError:
|
|
1026
|
+
return None
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
DOC_SQL = "INSERT OR IGNORE INTO %s (key, at, doc) VALUES (?, ?, ?)"
|
|
1030
|
+
MARK_SQL = "INSERT OR IGNORE INTO %s (key, at) VALUES (?, ?)"
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def _legacy(cache, table):
|
|
1034
|
+
"""(the INSERTs, the files) of one cache store's legacy files under
|
|
1035
|
+
`cache`. A file that cannot be read or parsed brings no row, as its old
|
|
1036
|
+
reader read nothing from it, and goes with the rest."""
|
|
1037
|
+
src = os.path.join(cache, CACHE_LEGACY[table])
|
|
1038
|
+
put, gone = [], []
|
|
1039
|
+
if table in ("judge_last", "update_check"):
|
|
1040
|
+
if os.path.isfile(src):
|
|
1041
|
+
value = _read_json(src)
|
|
1042
|
+
if isinstance(value, dict):
|
|
1043
|
+
put.append((DOC_SQL % table, ("", os.path.getmtime(src), json.dumps(value))))
|
|
1044
|
+
gone.append(src)
|
|
1045
|
+
elif table == "snapshots":
|
|
1046
|
+
# the capture order is each directory's mtime; the blob stays
|
|
1047
|
+
for _ns, name, path in _dirs(src):
|
|
1048
|
+
meta = os.path.join(path, "meta.json")
|
|
1049
|
+
value = _read_json(meta)
|
|
1050
|
+
if isinstance(value, dict):
|
|
1051
|
+
put.append(("INSERT OR IGNORE INTO snapshots (id, meta) VALUES (?, ?)",
|
|
1052
|
+
(name, json.dumps(value))))
|
|
1053
|
+
if os.path.exists(meta):
|
|
1054
|
+
gone.append(meta)
|
|
1055
|
+
elif table == "replay_runs":
|
|
1056
|
+
latest = (_read_text(os.path.join(src, "latest")) or "").strip()
|
|
1057
|
+
latest = latest and os.path.realpath(latest)
|
|
1058
|
+
# by name (the run's start time), the one `latest` named last; the key
|
|
1059
|
+
# is the run directory's real path (tezgah_replay._run_key)
|
|
1060
|
+
for _ns, _name, path in sorted(_dirs(src), key=lambda d: (
|
|
1061
|
+
os.path.realpath(d[2]) == latest, d[1])):
|
|
1062
|
+
summary = os.path.join(path, "summary.json")
|
|
1063
|
+
value = _read_json(summary)
|
|
1064
|
+
if isinstance(value, dict):
|
|
1065
|
+
put.append(("INSERT OR IGNORE INTO replay_runs (run, summary) VALUES (?, ?)",
|
|
1066
|
+
(os.path.realpath(path), json.dumps(value))))
|
|
1067
|
+
if os.path.exists(summary):
|
|
1068
|
+
gone.append(summary)
|
|
1069
|
+
if os.path.exists(os.path.join(src, "latest")):
|
|
1070
|
+
gone.append(os.path.join(src, "latest"))
|
|
1071
|
+
elif table == "cursor_reinforced":
|
|
1072
|
+
for _ns, session, folder in _dirs(src):
|
|
1073
|
+
for name, path, at in _files(folder):
|
|
1074
|
+
put.append((MARK_SQL % table, (session + "/" + name, at)))
|
|
1075
|
+
gone.append(path)
|
|
1076
|
+
else:
|
|
1077
|
+
for name, path, at in _files(src):
|
|
1078
|
+
if table == "used" and name.endswith(".jsonl"):
|
|
1079
|
+
for line in (_read_text(path) or "").splitlines():
|
|
1080
|
+
try:
|
|
1081
|
+
kind = json.loads(line)["kind"]
|
|
1082
|
+
except (ValueError, KeyError, TypeError):
|
|
1083
|
+
continue
|
|
1084
|
+
if isinstance(kind, str):
|
|
1085
|
+
put.append(("INSERT OR IGNORE INTO used (session, kind, at) "
|
|
1086
|
+
"VALUES (?, ?, ?)", (name[:-len(".jsonl")], kind, at)))
|
|
1087
|
+
elif table == "lesson_taint" and name.endswith(".jsonl"):
|
|
1088
|
+
for line in (_read_text(path) or "").splitlines():
|
|
1089
|
+
try:
|
|
1090
|
+
row = json.loads(line)
|
|
1091
|
+
except ValueError:
|
|
1092
|
+
continue
|
|
1093
|
+
if isinstance(row, dict) and row.get("key"):
|
|
1094
|
+
put.append(("INSERT OR REPLACE INTO lesson_taint (root, key, source, "
|
|
1095
|
+
"ts) VALUES (?, ?, ?, ?)",
|
|
1096
|
+
(name[:-len(".jsonl")], str(row["key"]),
|
|
1097
|
+
_column(row.get("source")), _column(row.get("ts")))))
|
|
1098
|
+
elif table == "judge_down":
|
|
1099
|
+
put.append(("INSERT OR IGNORE INTO judge_down (key, until) VALUES (?, ?)",
|
|
1100
|
+
(name, at + LEGACY_DOWN_FOR)))
|
|
1101
|
+
elif table in MARK_STORES:
|
|
1102
|
+
put.append((MARK_SQL % table, (name, at)))
|
|
1103
|
+
elif table in ("harness_drift", "import_crash", "cursor_answer"):
|
|
1104
|
+
text = _read_text(path)
|
|
1105
|
+
if text is not None:
|
|
1106
|
+
put.append((DOC_SQL % table, (name, at, json.dumps(text))))
|
|
1107
|
+
elif table in DOC_STORES and name.endswith(".json"): # taste-learn's logs stay
|
|
1108
|
+
value = _read_json(path)
|
|
1109
|
+
if value is not None:
|
|
1110
|
+
put.append((DOC_SQL % table, (name[:-len(".json")], at, json.dumps(value))))
|
|
1111
|
+
else:
|
|
1112
|
+
continue
|
|
1113
|
+
gone.append(path)
|
|
1114
|
+
return put, gone
|
|
1115
|
+
|
|
1116
|
+
|
|
1117
|
+
def _legacy_done(conn):
|
|
1118
|
+
return {name[len("legacy:"):] for (name,) in conn.execute(
|
|
1119
|
+
"SELECT name FROM imported WHERE name >= 'legacy:' AND name < 'legacy;'")}
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def _import_cache(conn, cache):
|
|
1123
|
+
"""Bring each cache store's legacy files (CACHE_LEGACY) in, once per
|
|
1124
|
+
database: a store is done when `imported` holds its `legacy:<table>` row,
|
|
1125
|
+
written in the transaction that takes its rows, one transaction per store.
|
|
1126
|
+
The files are read before that transaction and removed after it commits. A
|
|
1127
|
+
file of a done store is never read again, and any open that finds one
|
|
1128
|
+
removes it: one a process killed between the commit and the removal left,
|
|
1129
|
+
or one a writer from before this release added (an opencode plugin started
|
|
1130
|
+
before the upgrade, until it restarts). With every store done and its files
|
|
1131
|
+
gone this is one indexed lookup and a stat per store per open. ponytail:
|
|
1132
|
+
the snapshot manifests are not looked for once done, because that is a stat
|
|
1133
|
+
per snapshot per open; a leftover one goes with its blob at eviction.
|
|
1134
|
+
Total: a store that fails, whatever the error, is left for the next open
|
|
1135
|
+
and costs the others nothing."""
|
|
1136
|
+
try:
|
|
1137
|
+
done = _legacy_done(conn)
|
|
1138
|
+
except ERRORS:
|
|
1139
|
+
return
|
|
1140
|
+
for table, name in CACHE_LEGACY.items():
|
|
1141
|
+
folder = os.path.join(cache, name)
|
|
1142
|
+
try:
|
|
1143
|
+
if table in done:
|
|
1144
|
+
if table == "snapshots" or not os.path.exists(folder):
|
|
1145
|
+
continue
|
|
1146
|
+
gone = _legacy(cache, table)[1]
|
|
1147
|
+
else:
|
|
1148
|
+
put, gone = _legacy(cache, table)
|
|
1149
|
+
with transaction(conn):
|
|
1150
|
+
if table not in _legacy_done(conn):
|
|
1151
|
+
for sql, args in put:
|
|
1152
|
+
conn.execute(sql, args)
|
|
1153
|
+
conn.execute("INSERT OR IGNORE INTO imported (name, dev, ino, bytes) "
|
|
1154
|
+
"VALUES (?, 0, 0, 0)", ("legacy:" + table,))
|
|
1155
|
+
except Exception:
|
|
1156
|
+
continue
|
|
1157
|
+
for path in gone:
|
|
1158
|
+
try:
|
|
1159
|
+
os.remove(path)
|
|
1160
|
+
except OSError:
|
|
1161
|
+
pass
|
|
1162
|
+
for _ns, _name, path in _dirs(folder) if table == "cursor_reinforced" else ():
|
|
1163
|
+
try:
|
|
1164
|
+
os.rmdir(path)
|
|
1165
|
+
except OSError:
|
|
1166
|
+
pass
|
|
1167
|
+
try:
|
|
1168
|
+
os.rmdir(folder) # only an emptied one: blobs, runs and logs stay
|
|
1169
|
+
except OSError:
|
|
1170
|
+
pass
|
|
1171
|
+
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
# `evidence VERB LEDGER [ARG]`: the ledger reads and writes for a caller that
|
|
1175
|
+
# cannot open the database itself (opencode's plugin on a runtime without
|
|
1176
|
+
# node:sqlite), each verb with the count of arguments it takes after LEDGER.
|
|
1177
|
+
EVIDENCE_VERBS = {"append": 1, "tail": 1, "first": 0, "kind": 1, "import": 0}
|
|
1178
|
+
EVIDENCE_USAGE = ("usage: tezgah_store.py evidence append LEDGER JSON | tail LEDGER N"
|
|
1179
|
+
" | first LEDGER | kind LEDGER KIND | import LEDGER")
|
|
1180
|
+
|
|
1181
|
+
|
|
1182
|
+
def _evidence_cli(args):
|
|
1183
|
+
"""One verb on the ledger at args[1]: `append` stores JSON as a row,
|
|
1184
|
+
`import` brings its legacy file in, and a read prints row texts one per
|
|
1185
|
+
line, oldest first - the last N (`tail`), the first (`first`), or the rows
|
|
1186
|
+
of KIND (`kind`)."""
|
|
1187
|
+
verb = args[0] if args else None
|
|
1188
|
+
if EVIDENCE_VERBS.get(verb) != len(args) - 2:
|
|
1189
|
+
print(EVIDENCE_USAGE, file=sys.stderr)
|
|
1190
|
+
return 2
|
|
1191
|
+
path, rest = args[1], args[2:]
|
|
1192
|
+
try:
|
|
1193
|
+
if verb == "append":
|
|
1194
|
+
append_evidence(path, rest[0])
|
|
1195
|
+
return 0
|
|
1196
|
+
if verb == "import":
|
|
1197
|
+
import_session(path)
|
|
1198
|
+
return 0
|
|
1199
|
+
if verb == "first":
|
|
1200
|
+
found = [evidence_first(path)]
|
|
1201
|
+
elif verb == "kind":
|
|
1202
|
+
found = evidence_rows(path, kind=rest[0])
|
|
1203
|
+
else:
|
|
1204
|
+
try:
|
|
1205
|
+
tail = int(rest[0])
|
|
1206
|
+
except ValueError:
|
|
1207
|
+
print(EVIDENCE_USAGE, file=sys.stderr)
|
|
1208
|
+
return 2
|
|
1209
|
+
found = evidence_rows(path, tail=max(tail, 1))
|
|
1210
|
+
except ERRORS as exc:
|
|
1211
|
+
print("tezgah_store: %s" % exc, file=sys.stderr)
|
|
1212
|
+
return 1
|
|
1213
|
+
sys.stdout.buffer.write("".join(text + "\n" for text in found if text is not None)
|
|
1214
|
+
.encode("utf-8", "surrogatepass"))
|
|
1215
|
+
return 0
|
|
1216
|
+
|
|
1217
|
+
|
|
1218
|
+
# `mark CACHE TABLE KEY` and `use CACHE SESSION KIND`: the two cache-store
|
|
1219
|
+
# writes opencode's plugin makes (the once-per-session nudge and the used-kind
|
|
1220
|
+
# marks), for the same caller as `evidence`. `mark` prints 1 when it set the
|
|
1221
|
+
# mark and 0 when it was set already.
|
|
1222
|
+
STORE_USAGE = "usage: tezgah_store.py mark CACHE TABLE KEY | use CACHE SESSION KIND"
|
|
1223
|
+
|
|
1224
|
+
|
|
1225
|
+
def _store_cli(args):
|
|
1226
|
+
if len(args) != 4 or args[0] not in ("mark", "use") or (
|
|
1227
|
+
args[0] == "mark" and args[2] not in MARK_STORES):
|
|
1228
|
+
print(STORE_USAGE, file=sys.stderr)
|
|
1229
|
+
return 2
|
|
1230
|
+
verb, cache, name, value = args
|
|
1231
|
+
try:
|
|
1232
|
+
if verb == "use":
|
|
1233
|
+
use(name, value, cache)
|
|
1234
|
+
else:
|
|
1235
|
+
print(1 if mark(name, value, cache) else 0)
|
|
1236
|
+
except ERRORS as exc:
|
|
1237
|
+
print("tezgah_store: %s" % exc, file=sys.stderr)
|
|
1238
|
+
return 1
|
|
1239
|
+
return 0
|
|
1240
|
+
|
|
1241
|
+
|
|
1242
|
+
def main(argv):
|
|
1243
|
+
"""`import-evidence [CACHE...]`: the bulk import, for the given cache dirs
|
|
1244
|
+
or for the cache and its temp fallback. `evidence ...`: `_evidence_cli`;
|
|
1245
|
+
`mark ...` and `use ...`: `_store_cli`."""
|
|
1246
|
+
if argv[1:2] == ["evidence"]:
|
|
1247
|
+
return _evidence_cli(argv[2:])
|
|
1248
|
+
if argv[1:2] in (["mark"], ["use"]):
|
|
1249
|
+
return _store_cli(argv[1:])
|
|
1250
|
+
if argv[1:2] != ["import-evidence"]:
|
|
1251
|
+
print("usage: tezgah_store.py import-evidence [CACHE_DIR...]\n" + EVIDENCE_USAGE
|
|
1252
|
+
+ "\n" + STORE_USAGE, file=sys.stderr)
|
|
1253
|
+
return 2
|
|
1254
|
+
caches = argv[2:]
|
|
1255
|
+
if not caches:
|
|
1256
|
+
import tezgah_paths as tp
|
|
1257
|
+
caches = list(dict.fromkeys((tp.CACHE, tp.fallback_cache())))
|
|
1258
|
+
for cache in caches:
|
|
1259
|
+
try:
|
|
1260
|
+
print("%s: %d legacy ledger file(s) imported" % (cache, import_evidence(cache)))
|
|
1261
|
+
except ERRORS as exc:
|
|
1262
|
+
print("%s: import failed: %s" % (cache, exc), file=sys.stderr)
|
|
1263
|
+
return 1
|
|
1264
|
+
return 0
|
|
1265
|
+
|
|
1266
|
+
|
|
1267
|
+
if __name__ == "__main__":
|
|
1268
|
+
sys.exit(main(sys.argv))
|