@totalreclaw/totalreclaw 3.3.1-rc.8 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +268 -1
- package/SKILL.md +29 -23
- package/api-client.ts +18 -11
- package/claims-helper.ts +47 -1
- package/config.ts +108 -4
- package/confirm-indexed.ts +191 -0
- package/crypto.ts +10 -2
- package/dist/api-client.js +226 -0
- package/dist/billing-cache.js +100 -0
- package/dist/claims-helper.js +624 -0
- package/dist/config.js +297 -0
- package/dist/confirm-indexed.js +127 -0
- package/dist/consolidation.js +258 -0
- package/dist/contradiction-sync.js +1034 -0
- package/dist/crypto.js +138 -0
- package/dist/digest-sync.js +361 -0
- package/dist/download-ux.js +63 -0
- package/dist/embedder-cache.js +185 -0
- package/dist/embedder-loader.js +121 -0
- package/dist/embedder-network.js +301 -0
- package/dist/embedding.js +141 -0
- package/dist/extractor.js +1225 -0
- package/dist/first-run.js +103 -0
- package/dist/fs-helpers.js +725 -0
- package/dist/gateway-url.js +197 -0
- package/dist/generate-mnemonic.js +13 -0
- package/dist/hot-cache-wrapper.js +101 -0
- package/dist/import-adapters/base-adapter.js +64 -0
- package/dist/import-adapters/chatgpt-adapter.js +238 -0
- package/dist/import-adapters/claude-adapter.js +114 -0
- package/dist/import-adapters/gemini-adapter.js +201 -0
- package/dist/import-adapters/index.js +26 -0
- package/dist/import-adapters/mcp-memory-adapter.js +219 -0
- package/dist/import-adapters/mem0-adapter.js +158 -0
- package/dist/import-adapters/types.js +1 -0
- package/dist/index.js +5388 -0
- package/dist/llm-client.js +687 -0
- package/dist/llm-profile-reader.js +346 -0
- package/dist/lsh.js +62 -0
- package/dist/onboarding-cli.js +750 -0
- package/dist/pair-cli.js +344 -0
- package/dist/pair-crypto.js +359 -0
- package/dist/pair-http.js +404 -0
- package/dist/pair-page.js +826 -0
- package/dist/pair-qr.js +107 -0
- package/dist/pair-remote-client.js +410 -0
- package/dist/pair-session-store.js +566 -0
- package/dist/pin.js +556 -0
- package/dist/qa-bug-report.js +301 -0
- package/dist/relay-headers.js +44 -0
- package/dist/reranker.js +409 -0
- package/dist/retype-setscope.js +368 -0
- package/dist/semantic-dedup.js +75 -0
- package/dist/subgraph-search.js +289 -0
- package/dist/subgraph-store.js +694 -0
- package/dist/tool-gating.js +58 -0
- package/download-ux.ts +91 -0
- package/embedder-cache.ts +230 -0
- package/embedder-loader.ts +189 -0
- package/embedder-network.ts +350 -0
- package/embedding.ts +118 -27
- package/fs-helpers.ts +277 -0
- package/gateway-url.ts +57 -9
- package/index.ts +469 -250
- package/llm-client.ts +4 -3
- package/lsh.ts +7 -2
- package/onboarding-cli.ts +114 -1
- package/package.json +24 -5
- package/pair-cli.ts +76 -8
- package/pair-crypto.ts +34 -24
- package/pair-page.ts +28 -17
- package/pair-qr.ts +152 -0
- package/pair-remote-client.ts +540 -0
- package/pin.ts +31 -0
- package/qa-bug-report.ts +84 -2
- package/relay-headers.ts +50 -0
- package/reranker.ts +40 -0
- package/retype-setscope.ts +69 -8
- package/skill.json +1 -1
- package/subgraph-search.ts +4 -3
- package/subgraph-store.ts +15 -10
|
@@ -0,0 +1,1034 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TotalReclaw Plugin — contradiction detection + auto-resolution (Phase 2 Slice 2d).
|
|
3
|
+
*
|
|
4
|
+
* Runs after store-time dedup (>= 0.85 cosine) and before the canonical Claim
|
|
5
|
+
* is encrypted + written. For every entity on the new claim, fetches existing
|
|
6
|
+
* active claims that reference the same entity via the entity trapdoor, decrypts
|
|
7
|
+
* them, and asks the WASM core to detect contradictions in the [0.3, 0.85) band.
|
|
8
|
+
* Each contradicting pair is then resolved via the P2-3 formula; the winner is
|
|
9
|
+
* kept on-chain, the loser is queued for tombstoning. Pinned claims are never
|
|
10
|
+
* touched — a contradiction against a pinned claim always causes the new write
|
|
11
|
+
* to be skipped with reason `existing_pinned`.
|
|
12
|
+
*
|
|
13
|
+
* This module mirrors the structure of `digest-sync.ts`: pure functions at the
|
|
14
|
+
* core, I/O behind dependency injection so the test file can run the real WASM
|
|
15
|
+
* while stubbing subgraph + filesystem.
|
|
16
|
+
*/
|
|
17
|
+
import { createRequire } from 'node:module';
|
|
18
|
+
import fs from 'node:fs';
|
|
19
|
+
import os from 'node:os';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
import { computeEntityTrapdoor, isDigestBlob, } from './claims-helper.js';
|
|
22
|
+
const requireWasm = createRequire(import.meta.url);
|
|
23
|
+
let _wasm = null;
|
|
24
|
+
function getWasm() {
|
|
25
|
+
if (!_wasm)
|
|
26
|
+
_wasm = requireWasm('@totalreclaw/core');
|
|
27
|
+
return _wasm;
|
|
28
|
+
}
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
// Paths + file I/O
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
/** Where feedback, decisions, and weights live. `~/.totalreclaw/` by default. */
|
|
33
|
+
function resolveStateDir() {
|
|
34
|
+
const override = process.env.TOTALRECLAW_STATE_DIR;
|
|
35
|
+
if (override && override.length > 0)
|
|
36
|
+
return override;
|
|
37
|
+
return path.join(os.homedir(), '.totalreclaw');
|
|
38
|
+
}
|
|
39
|
+
function ensureStateDir() {
|
|
40
|
+
const dir = resolveStateDir();
|
|
41
|
+
try {
|
|
42
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// Caller handles downstream read/write failures.
|
|
46
|
+
}
|
|
47
|
+
return dir;
|
|
48
|
+
}
|
|
49
|
+
export function decisionsLogPath() {
|
|
50
|
+
return path.join(resolveStateDir(), 'decisions.jsonl');
|
|
51
|
+
}
|
|
52
|
+
export function weightsFilePath() {
|
|
53
|
+
return path.join(resolveStateDir(), 'weights.json');
|
|
54
|
+
}
|
|
55
|
+
/** Cap on the decisions.jsonl log — oldest lines are dropped above this. */
|
|
56
|
+
export const DECISION_LOG_MAX_LINES = 10_000;
|
|
57
|
+
/** Soft cap on candidates fetched per entity during contradiction detection. */
|
|
58
|
+
export const CONTRADICTION_CANDIDATE_CAP = 20;
|
|
59
|
+
/**
|
|
60
|
+
* Minimum score gap required to auto-resolve a `supersede_existing` decision.
|
|
61
|
+
*
|
|
62
|
+
* When the formula winner beats the loser by less than this amount, the
|
|
63
|
+
* decision is treated as a tie and both claims stay active. The tie is still
|
|
64
|
+
* logged to decisions.jsonl (with action='tie_leave_both') for audit.
|
|
65
|
+
*
|
|
66
|
+
* Rationale: the scoring formula (confidence + corroboration + recency +
|
|
67
|
+
* validation) produces near-identical scores when two claims describe
|
|
68
|
+
* different use cases of the same overall concept (e.g., Postgres for OLTP
|
|
69
|
+
* and DuckDB for OLAP — both explicitly-stated, both recent, both with
|
|
70
|
+
* single-occurrence corroboration). Auto-superseding in this zone promotes
|
|
71
|
+
* rounding noise into "the user changed their mind" and produces false
|
|
72
|
+
* positives that the weight-tuning loop cannot correct (see project
|
|
73
|
+
* memory: tombstoned claims are unrecoverable via pin).
|
|
74
|
+
*
|
|
75
|
+
* 0.01 (1%) is a deliberately tight guard: it catches noise-margin ties
|
|
76
|
+
* without blocking genuine narrow wins. Calibrated against the 2026-04-14
|
|
77
|
+
* Postgres/DuckDB false positive where the gap was 9 parts per million.
|
|
78
|
+
*/
|
|
79
|
+
export const TIE_ZONE_SCORE_TOLERANCE = 0.01;
|
|
80
|
+
/**
|
|
81
|
+
* Append one entry to the decision log, rotating if it grows past the cap.
|
|
82
|
+
* Never throws — logging is best-effort.
|
|
83
|
+
*/
|
|
84
|
+
export async function appendDecisionLog(entry) {
|
|
85
|
+
try {
|
|
86
|
+
const dir = ensureStateDir();
|
|
87
|
+
const p = path.join(dir, 'decisions.jsonl');
|
|
88
|
+
let existing = '';
|
|
89
|
+
try {
|
|
90
|
+
existing = fs.readFileSync(p, 'utf-8');
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
existing = '';
|
|
94
|
+
}
|
|
95
|
+
const line = JSON.stringify(entry);
|
|
96
|
+
let next = existing;
|
|
97
|
+
if (next.length === 0) {
|
|
98
|
+
next = line + '\n';
|
|
99
|
+
}
|
|
100
|
+
else if (next.endsWith('\n')) {
|
|
101
|
+
next = next + line + '\n';
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
next = next + '\n' + line + '\n';
|
|
105
|
+
}
|
|
106
|
+
// Rotate via the WASM core helper (same primitive used by feedback.jsonl).
|
|
107
|
+
// `rotateFeedbackLog` expects a BigInt for max_lines.
|
|
108
|
+
const rotated = getWasm().rotateFeedbackLog(next, BigInt(DECISION_LOG_MAX_LINES));
|
|
109
|
+
fs.writeFileSync(p, rotated, 'utf-8');
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
// Logging failures are never fatal.
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Load the per-user weights file, falling back to defaults when the file
|
|
117
|
+
* does not exist or is malformed. Never throws.
|
|
118
|
+
*/
|
|
119
|
+
export async function loadWeightsFile(nowUnixSeconds) {
|
|
120
|
+
const core = getWasm();
|
|
121
|
+
const p = weightsFilePath();
|
|
122
|
+
try {
|
|
123
|
+
const raw = fs.readFileSync(p, 'utf-8');
|
|
124
|
+
const parsedJson = core.parseWeightsFile(raw);
|
|
125
|
+
return JSON.parse(parsedJson);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
// File missing / malformed / wrong version → return fresh defaults.
|
|
129
|
+
const freshJson = core.defaultWeightsFile(BigInt(Math.floor(nowUnixSeconds)));
|
|
130
|
+
return JSON.parse(freshJson);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
export async function saveWeightsFile(file) {
|
|
134
|
+
const core = getWasm();
|
|
135
|
+
try {
|
|
136
|
+
const dir = ensureStateDir();
|
|
137
|
+
const p = path.join(dir, 'weights.json');
|
|
138
|
+
const serialized = core.serializeWeightsFile(JSON.stringify(file));
|
|
139
|
+
fs.writeFileSync(p, serialized, 'utf-8');
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// best-effort
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Parse a decrypted Claim blob into a `{claim, status}` pair.
|
|
147
|
+
*
|
|
148
|
+
* Accepts the canonical short-key Claim shape (`{t, c, cf, i, sa, ea, ...}`).
|
|
149
|
+
* Returns null for legacy docs, digest blobs, entity-infrastructure claims,
|
|
150
|
+
* or anything that fails to parse cleanly — these are all excluded from
|
|
151
|
+
* contradiction detection.
|
|
152
|
+
*/
|
|
153
|
+
export function parseCandidateClaim(decryptedJson) {
|
|
154
|
+
if (isDigestBlob(decryptedJson))
|
|
155
|
+
return null;
|
|
156
|
+
let obj;
|
|
157
|
+
try {
|
|
158
|
+
obj = JSON.parse(decryptedJson);
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
if (typeof obj.t !== 'string' || typeof obj.c !== 'string')
|
|
164
|
+
return null;
|
|
165
|
+
// Filter out infra categories — digests and entity rollup claims are not
|
|
166
|
+
// user-facing knowledge and must never be considered contradictions.
|
|
167
|
+
if (obj.c === 'dig' || obj.c === 'ent')
|
|
168
|
+
return null;
|
|
169
|
+
return obj;
|
|
170
|
+
}
|
|
171
|
+
/** Is this candidate claim pinned (status `p`)? Delegates to WASM core. */
|
|
172
|
+
export function isPinnedClaim(claim) {
|
|
173
|
+
try {
|
|
174
|
+
return getWasm().isPinnedClaim(JSON.stringify(claim));
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
// Fallback: local check if WASM fails (e.g. malformed claim).
|
|
178
|
+
return typeof claim.st === 'string' && claim.st === 'p';
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Collect active claim candidates for every entity on the new claim.
|
|
183
|
+
*
|
|
184
|
+
* For each entity:
|
|
185
|
+
* 1. Compute its trapdoor (same primitive as the write path).
|
|
186
|
+
* 2. Query the subgraph for facts matching that single-element trapdoor.
|
|
187
|
+
* 3. Decrypt + parse each row, filtering infra claims and bad blobs.
|
|
188
|
+
* 4. Recover the embedding (reusing the stored one when possible).
|
|
189
|
+
*
|
|
190
|
+
* Deduplicates by subgraph fact id across entities so the same claim is
|
|
191
|
+
* never processed twice. Caps the total number of candidates per entity at
|
|
192
|
+
* `CONTRADICTION_CANDIDATE_CAP` to keep the write-path cost bounded.
|
|
193
|
+
*/
|
|
194
|
+
export async function collectCandidatesForEntities(newClaim, newClaimId, subgraphOwner, authKeyHex, encryptionKey, deps, logger) {
|
|
195
|
+
const entities = Array.isArray(newClaim.e) ? newClaim.e : [];
|
|
196
|
+
if (entities.length === 0)
|
|
197
|
+
return [];
|
|
198
|
+
const seenIds = new Set();
|
|
199
|
+
const out = [];
|
|
200
|
+
for (const entity of entities) {
|
|
201
|
+
const name = typeof entity?.n === 'string' ? entity.n : null;
|
|
202
|
+
if (!name)
|
|
203
|
+
continue;
|
|
204
|
+
const trapdoor = computeEntityTrapdoor(name);
|
|
205
|
+
let rows = [];
|
|
206
|
+
try {
|
|
207
|
+
rows = await deps.searchSubgraph(subgraphOwner, [trapdoor], CONTRADICTION_CANDIDATE_CAP, authKeyHex);
|
|
208
|
+
}
|
|
209
|
+
catch (err) {
|
|
210
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
211
|
+
logger.warn(`Contradiction: subgraph query failed for entity "${name}": ${msg}`);
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
for (const row of rows) {
|
|
215
|
+
if (!row || !row.id || row.id === newClaimId)
|
|
216
|
+
continue;
|
|
217
|
+
if (row.isActive === false)
|
|
218
|
+
continue;
|
|
219
|
+
if (seenIds.has(row.id))
|
|
220
|
+
continue;
|
|
221
|
+
seenIds.add(row.id);
|
|
222
|
+
let decrypted;
|
|
223
|
+
try {
|
|
224
|
+
decrypted = deps.decryptFromHex(row.encryptedBlob, encryptionKey);
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const parsed = parseCandidateClaim(decrypted);
|
|
230
|
+
if (!parsed)
|
|
231
|
+
continue;
|
|
232
|
+
let embedding = [];
|
|
233
|
+
if (row.encryptedEmbedding) {
|
|
234
|
+
try {
|
|
235
|
+
const emb = JSON.parse(deps.decryptFromHex(row.encryptedEmbedding, encryptionKey));
|
|
236
|
+
if (Array.isArray(emb) && emb.every((x) => typeof x === 'number')) {
|
|
237
|
+
embedding = emb;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
embedding = [];
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
out.push({ claim: parsed, id: row.id, embedding });
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return out;
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Run WASM contradiction detection + resolution on in-memory data only.
|
|
251
|
+
*
|
|
252
|
+
* Tries `core.resolveWithCandidates()` first (full pipeline in one call:
|
|
253
|
+
* detect contradictions + resolve pairs + pin check + tie-zone guard).
|
|
254
|
+
* Falls back to the legacy `detectContradictions` + `resolvePair` approach
|
|
255
|
+
* when `resolveWithCandidates` is not available (core < 1.5.0).
|
|
256
|
+
*
|
|
257
|
+
* Split out from `detectAndResolveContradictions` so the write path can test
|
|
258
|
+
* the decision logic without mocking file I/O or the subgraph.
|
|
259
|
+
*/
|
|
260
|
+
export function resolveWithCore(input) {
|
|
261
|
+
const { newClaim, newClaimId, newEmbedding, candidates, weightsJson, thresholdLower, thresholdUpper, nowUnixSeconds, mode, logger, } = input;
|
|
262
|
+
if (mode === 'off')
|
|
263
|
+
return [];
|
|
264
|
+
if (candidates.length === 0)
|
|
265
|
+
return [];
|
|
266
|
+
if (newEmbedding.length === 0) {
|
|
267
|
+
logger.warn('Contradiction: new claim has no embedding; skipping detection');
|
|
268
|
+
return [];
|
|
269
|
+
}
|
|
270
|
+
const core = getWasm();
|
|
271
|
+
// Build the WASM candidates payload. Drop any candidate whose embedding
|
|
272
|
+
// is missing — the core short-circuits on empty vectors but we also drop
|
|
273
|
+
// them from the lookup map so we don't waste a reference.
|
|
274
|
+
const items = candidates
|
|
275
|
+
.filter((c) => c.embedding.length > 0)
|
|
276
|
+
.map((c) => ({ claim: c.claim, id: c.id, embedding: c.embedding }));
|
|
277
|
+
if (items.length === 0)
|
|
278
|
+
return [];
|
|
279
|
+
// Index candidates by id for fast lookup during resolve.
|
|
280
|
+
const byId = new Map();
|
|
281
|
+
for (const c of items)
|
|
282
|
+
byId.set(c.id, { claim: c.claim, id: c.id, embedding: c.embedding });
|
|
283
|
+
// Try the unified core.resolveWithCandidates() (available in core >= 1.5.0).
|
|
284
|
+
// Falls back to legacy detectContradictions + resolvePair if not available.
|
|
285
|
+
if (typeof core.resolveWithCandidates === 'function') {
|
|
286
|
+
return _resolveWithCandidatesCore(core, input, items, byId);
|
|
287
|
+
}
|
|
288
|
+
return _resolveWithLegacyPipeline(core, input, items, byId);
|
|
289
|
+
}
|
|
290
|
+
/** Core >= 1.5.0 path: single resolveWithCandidates call. */
|
|
291
|
+
function _resolveWithCandidatesCore(core, input, items, byId) {
|
|
292
|
+
const { newClaim, newClaimId, newEmbedding, weightsJson, thresholdLower, thresholdUpper, nowUnixSeconds, logger } = input;
|
|
293
|
+
let actionsJson;
|
|
294
|
+
try {
|
|
295
|
+
actionsJson = core.resolveWithCandidates(JSON.stringify(newClaim), newClaimId, JSON.stringify(newEmbedding), JSON.stringify(items), weightsJson, thresholdLower, thresholdUpper, Math.floor(nowUnixSeconds), TIE_ZONE_SCORE_TOLERANCE);
|
|
296
|
+
}
|
|
297
|
+
catch (err) {
|
|
298
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
299
|
+
logger.warn(`Contradiction: resolveWithCandidates failed: ${msg}`);
|
|
300
|
+
return [];
|
|
301
|
+
}
|
|
302
|
+
let actions;
|
|
303
|
+
try {
|
|
304
|
+
actions = JSON.parse(actionsJson);
|
|
305
|
+
}
|
|
306
|
+
catch {
|
|
307
|
+
return [];
|
|
308
|
+
}
|
|
309
|
+
if (actions.length === 0)
|
|
310
|
+
return [];
|
|
311
|
+
// Map core ResolutionAction (tagged enum with "type" field) → local ResolutionDecision.
|
|
312
|
+
const decisions = [];
|
|
313
|
+
for (const action of actions) {
|
|
314
|
+
const type = action.type;
|
|
315
|
+
if (type === 'supersede_existing') {
|
|
316
|
+
const existing = byId.get(action.existing_id);
|
|
317
|
+
decisions.push({
|
|
318
|
+
action: 'supersede_existing',
|
|
319
|
+
existingFactId: action.existing_id,
|
|
320
|
+
existingClaim: existing?.claim ?? {},
|
|
321
|
+
entityId: action.entity_id ?? '',
|
|
322
|
+
similarity: action.similarity ?? 0,
|
|
323
|
+
winnerScore: action.winner_score ?? 0,
|
|
324
|
+
loserScore: action.loser_score ?? 0,
|
|
325
|
+
winnerComponents: action.winner_components,
|
|
326
|
+
loserComponents: action.loser_components,
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
else if (type === 'skip_new') {
|
|
330
|
+
const reason = action.reason;
|
|
331
|
+
decisions.push({
|
|
332
|
+
action: 'skip_new',
|
|
333
|
+
reason: reason === 'existing_pinned' ? 'existing_pinned' : 'existing_wins',
|
|
334
|
+
existingFactId: action.existing_id,
|
|
335
|
+
entityId: action.entity_id ?? '',
|
|
336
|
+
similarity: action.similarity ?? 0,
|
|
337
|
+
winnerScore: action.winner_score,
|
|
338
|
+
loserScore: action.loser_score,
|
|
339
|
+
winnerComponents: action.winner_components,
|
|
340
|
+
loserComponents: action.loser_components,
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
else if (type === 'tie_leave_both') {
|
|
344
|
+
decisions.push({
|
|
345
|
+
action: 'tie_leave_both',
|
|
346
|
+
existingFactId: action.existing_id,
|
|
347
|
+
entityId: action.entity_id ?? '',
|
|
348
|
+
similarity: action.similarity ?? 0,
|
|
349
|
+
winnerScore: action.winner_score ?? 0,
|
|
350
|
+
loserScore: action.loser_score ?? 0,
|
|
351
|
+
winnerComponents: action.winner_components,
|
|
352
|
+
loserComponents: action.loser_components,
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
// NoContradiction actions are ignored — same as before.
|
|
356
|
+
}
|
|
357
|
+
return decisions;
|
|
358
|
+
}
|
|
359
|
+
/** Legacy path (core < 1.5.0): detectContradictions + resolvePair loop. */
|
|
360
|
+
function _resolveWithLegacyPipeline(core, input, items, byId) {
|
|
361
|
+
const { newClaim, newClaimId, newEmbedding, weightsJson, thresholdLower, thresholdUpper, nowUnixSeconds, logger } = input;
|
|
362
|
+
let contradictionsJson;
|
|
363
|
+
try {
|
|
364
|
+
contradictionsJson = core.detectContradictions(JSON.stringify(newClaim), newClaimId, JSON.stringify(newEmbedding), JSON.stringify(items), thresholdLower, thresholdUpper);
|
|
365
|
+
}
|
|
366
|
+
catch (err) {
|
|
367
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
368
|
+
logger.warn(`Contradiction: detectContradictions failed: ${msg}`);
|
|
369
|
+
return [];
|
|
370
|
+
}
|
|
371
|
+
let contradictions;
|
|
372
|
+
try {
|
|
373
|
+
contradictions = JSON.parse(contradictionsJson);
|
|
374
|
+
}
|
|
375
|
+
catch {
|
|
376
|
+
return [];
|
|
377
|
+
}
|
|
378
|
+
if (contradictions.length === 0)
|
|
379
|
+
return [];
|
|
380
|
+
const decisions = [];
|
|
381
|
+
const nowSecondsBig = BigInt(Math.floor(nowUnixSeconds));
|
|
382
|
+
for (const contradiction of contradictions) {
|
|
383
|
+
const existing = byId.get(contradiction.claim_b_id);
|
|
384
|
+
if (!existing)
|
|
385
|
+
continue;
|
|
386
|
+
// Pinned existing claims are untouchable.
|
|
387
|
+
if (isPinnedClaim(existing.claim)) {
|
|
388
|
+
decisions.push({
|
|
389
|
+
action: 'skip_new',
|
|
390
|
+
reason: 'existing_pinned',
|
|
391
|
+
existingFactId: existing.id,
|
|
392
|
+
entityId: contradiction.entity_id,
|
|
393
|
+
similarity: contradiction.similarity,
|
|
394
|
+
});
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
let outcomeJson;
|
|
398
|
+
try {
|
|
399
|
+
outcomeJson = core.resolvePair(JSON.stringify(newClaim), newClaimId, JSON.stringify(existing.claim), existing.id, nowSecondsBig, weightsJson);
|
|
400
|
+
}
|
|
401
|
+
catch (err) {
|
|
402
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
403
|
+
logger.warn(`Contradiction: resolvePair failed for ${existing.id.slice(0, 10)}…: ${msg}`);
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
let outcome;
|
|
407
|
+
try {
|
|
408
|
+
outcome = JSON.parse(outcomeJson);
|
|
409
|
+
}
|
|
410
|
+
catch {
|
|
411
|
+
continue;
|
|
412
|
+
}
|
|
413
|
+
if (outcome.winner_id === newClaimId) {
|
|
414
|
+
decisions.push({
|
|
415
|
+
action: 'supersede_existing',
|
|
416
|
+
existingFactId: existing.id,
|
|
417
|
+
existingClaim: existing.claim,
|
|
418
|
+
entityId: contradiction.entity_id,
|
|
419
|
+
similarity: contradiction.similarity,
|
|
420
|
+
winnerScore: outcome.winner_score,
|
|
421
|
+
loserScore: outcome.loser_score,
|
|
422
|
+
winnerComponents: outcome.winner_components,
|
|
423
|
+
loserComponents: outcome.loser_components,
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
else {
|
|
427
|
+
decisions.push({
|
|
428
|
+
action: 'skip_new',
|
|
429
|
+
reason: 'existing_wins',
|
|
430
|
+
existingFactId: existing.id,
|
|
431
|
+
entityId: contradiction.entity_id,
|
|
432
|
+
similarity: contradiction.similarity,
|
|
433
|
+
winnerScore: outcome.winner_score,
|
|
434
|
+
loserScore: outcome.loser_score,
|
|
435
|
+
winnerComponents: outcome.winner_components,
|
|
436
|
+
loserComponents: outcome.loser_components,
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
return decisions;
|
|
441
|
+
}
|
|
442
|
+
/**
|
|
443
|
+
* Write-path entry point. See Slice 2d of the Phase 2 design doc.
|
|
444
|
+
*
|
|
445
|
+
* Returns a list of `ResolutionDecision`s:
|
|
446
|
+
* - `supersede_existing`: caller queues a tombstone for `existingFactId`
|
|
447
|
+
* - `skip_new`: caller skips the new write (existing wins or is pinned)
|
|
448
|
+
* - `no_contradiction`: never explicitly returned — an empty list means this
|
|
449
|
+
*
|
|
450
|
+
* Never throws. On any failure (subgraph, decrypt, WASM), returns `[]` so the
|
|
451
|
+
* write path falls back to Phase 1 behaviour.
|
|
452
|
+
*/
|
|
453
|
+
export async function detectAndResolveContradictions(input) {
|
|
454
|
+
const { newClaim, newClaimId, newEmbedding, subgraphOwner, authKeyHex, encryptionKey, deps, logger, } = input;
|
|
455
|
+
// Read env per-call so tests can toggle without module reload.
|
|
456
|
+
const raw = (process.env.TOTALRECLAW_AUTO_RESOLVE_MODE ?? '').trim().toLowerCase();
|
|
457
|
+
const mode = raw === 'off' ? 'off' : raw === 'shadow' ? 'shadow' : 'active';
|
|
458
|
+
if (mode === 'off')
|
|
459
|
+
return [];
|
|
460
|
+
// No entities → nothing to check (same contract as detect_contradictions).
|
|
461
|
+
const entities = Array.isArray(newClaim.e) ? newClaim.e : [];
|
|
462
|
+
if (entities.length === 0)
|
|
463
|
+
return [];
|
|
464
|
+
const nowUnixSeconds = typeof deps.nowUnixSeconds === 'number'
|
|
465
|
+
? deps.nowUnixSeconds
|
|
466
|
+
: Math.floor(Date.now() / 1000);
|
|
467
|
+
// Load per-user weights file (defaults if missing/malformed).
|
|
468
|
+
const weightsFile = await loadWeightsFile(nowUnixSeconds);
|
|
469
|
+
const weightsJson = JSON.stringify(weightsFile.weights ?? {});
|
|
470
|
+
const thresholdLower = typeof weightsFile.threshold_lower === 'number' ? weightsFile.threshold_lower : 0.3;
|
|
471
|
+
const thresholdUpper = typeof weightsFile.threshold_upper === 'number' ? weightsFile.threshold_upper : 0.85;
|
|
472
|
+
// Retrieve + decrypt candidates.
|
|
473
|
+
let candidates;
|
|
474
|
+
try {
|
|
475
|
+
candidates = await collectCandidatesForEntities(newClaim, newClaimId, subgraphOwner, authKeyHex, encryptionKey, deps, logger);
|
|
476
|
+
}
|
|
477
|
+
catch (err) {
|
|
478
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
479
|
+
logger.warn(`Contradiction: candidate retrieval failed: ${msg}`);
|
|
480
|
+
return [];
|
|
481
|
+
}
|
|
482
|
+
if (candidates.length === 0)
|
|
483
|
+
return [];
|
|
484
|
+
const rawDecisions = resolveWithCore({
|
|
485
|
+
newClaim,
|
|
486
|
+
newClaimId,
|
|
487
|
+
newEmbedding,
|
|
488
|
+
candidates,
|
|
489
|
+
weightsJson,
|
|
490
|
+
thresholdLower,
|
|
491
|
+
thresholdUpper,
|
|
492
|
+
nowUnixSeconds,
|
|
493
|
+
mode,
|
|
494
|
+
logger,
|
|
495
|
+
});
|
|
496
|
+
// Tie-zone guard: when the formula winner beats the loser by less than
|
|
497
|
+
// TIE_ZONE_SCORE_TOLERANCE, the "contradiction" is rounding noise between
|
|
498
|
+
// two complementary claims (e.g. Postgres for OLTP, DuckDB for OLAP — both
|
|
499
|
+
// recent, both explicit, both single-corroboration). Mark these as ties so
|
|
500
|
+
// the caller does not tombstone either claim. The tie is still logged for
|
|
501
|
+
// operator visibility and for future feedback-loop calibration.
|
|
502
|
+
//
|
|
503
|
+
// When using core >= 1.5.0 (resolveWithCandidates), the tie-zone guard is
|
|
504
|
+
// already applied by the core and this map is a no-op (no supersede decisions
|
|
505
|
+
// with sub-tolerance gaps will appear). For the legacy path, this remains
|
|
506
|
+
// necessary.
|
|
507
|
+
const decisions = rawDecisions.map((d) => {
|
|
508
|
+
if (d.action === 'supersede_existing' &&
|
|
509
|
+
Math.abs(d.winnerScore - d.loserScore) < TIE_ZONE_SCORE_TOLERANCE) {
|
|
510
|
+
return {
|
|
511
|
+
action: 'tie_leave_both',
|
|
512
|
+
existingFactId: d.existingFactId,
|
|
513
|
+
entityId: d.entityId,
|
|
514
|
+
similarity: d.similarity,
|
|
515
|
+
winnerScore: d.winnerScore,
|
|
516
|
+
loserScore: d.loserScore,
|
|
517
|
+
winnerComponents: d.winnerComponents,
|
|
518
|
+
loserComponents: d.loserComponents,
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
return d;
|
|
522
|
+
});
|
|
523
|
+
// Build decision log entries. Try core.buildDecisionLogEntries (>= 1.5.0)
|
|
524
|
+
// first; fall back to inline logging if not available or on failure.
|
|
525
|
+
const core = getWasm();
|
|
526
|
+
const useCoreDecisionLog = typeof core.buildDecisionLogEntries === 'function';
|
|
527
|
+
if (useCoreDecisionLog) {
|
|
528
|
+
try {
|
|
529
|
+
const coreActions = _decisionsToCoreActions(decisions, newClaimId);
|
|
530
|
+
const existingClaimsMap = {};
|
|
531
|
+
for (const d of decisions) {
|
|
532
|
+
if (d.action === 'supersede_existing') {
|
|
533
|
+
try {
|
|
534
|
+
existingClaimsMap[d.existingFactId] = JSON.stringify(d.existingClaim);
|
|
535
|
+
}
|
|
536
|
+
catch { /* skip */ }
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
const entriesJson = core.buildDecisionLogEntries(JSON.stringify(coreActions), JSON.stringify(newClaim), JSON.stringify(existingClaimsMap), mode === 'shadow' ? 'shadow' : mode, Math.floor(nowUnixSeconds));
|
|
540
|
+
const entries = JSON.parse(entriesJson);
|
|
541
|
+
for (const entry of entries) {
|
|
542
|
+
await appendDecisionLog(entry);
|
|
543
|
+
if (entry.action === 'tie_leave_both') {
|
|
544
|
+
logger.info(`Contradiction: tie (gap=${Math.abs((entry.winner_score ?? 0) - (entry.loser_score ?? 0)).toFixed(6)} < ${TIE_ZONE_SCORE_TOLERANCE}, sim=${entry.similarity.toFixed(3)}, entity=${entry.entity_id}) — leaving both active`);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
catch (err) {
|
|
549
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
550
|
+
logger.warn(`Contradiction: buildDecisionLogEntries failed, falling back to inline: ${msg}`);
|
|
551
|
+
await _appendDecisionLogInline(decisions, newClaimId, nowUnixSeconds, mode, logger);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
else {
|
|
555
|
+
await _appendDecisionLogInline(decisions, newClaimId, nowUnixSeconds, mode, logger);
|
|
556
|
+
}
|
|
557
|
+
// Shadow mode filtering. Try core.filterShadowMode (>= 1.5.0) first.
|
|
558
|
+
if (typeof core.filterShadowMode === 'function') {
|
|
559
|
+
try {
|
|
560
|
+
const coreActions = _decisionsToCoreActions(decisions, newClaimId);
|
|
561
|
+
const filteredJson = core.filterShadowMode(JSON.stringify(coreActions), mode);
|
|
562
|
+
const filteredActions = JSON.parse(filteredJson);
|
|
563
|
+
const byExistingId = new Map();
|
|
564
|
+
for (const d of decisions) {
|
|
565
|
+
if (d.action === 'supersede_existing' || d.action === 'skip_new' || d.action === 'tie_leave_both') {
|
|
566
|
+
byExistingId.set(d.existingFactId, d);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
return filteredActions
|
|
570
|
+
.map((a) => byExistingId.get(a.existing_id))
|
|
571
|
+
.filter((d) => d !== undefined);
|
|
572
|
+
}
|
|
573
|
+
catch {
|
|
574
|
+
// Fall through to local filtering.
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
// Local fallback: shadow → empty, active → filter out ties.
|
|
578
|
+
if (mode === 'shadow')
|
|
579
|
+
return [];
|
|
580
|
+
return decisions.filter((d) => d.action !== 'tie_leave_both');
|
|
581
|
+
}
|
|
582
|
+
/** Convert ResolutionDecision[] to core ResolutionAction JSON format. */
|
|
583
|
+
function _decisionsToCoreActions(decisions, newClaimId) {
|
|
584
|
+
return decisions.map((d) => {
|
|
585
|
+
if (d.action === 'supersede_existing') {
|
|
586
|
+
return {
|
|
587
|
+
type: 'supersede_existing',
|
|
588
|
+
existing_id: d.existingFactId, new_id: newClaimId,
|
|
589
|
+
similarity: d.similarity, score_gap: Math.abs(d.winnerScore - d.loserScore),
|
|
590
|
+
entity_id: d.entityId, winner_score: d.winnerScore, loser_score: d.loserScore,
|
|
591
|
+
winner_components: d.winnerComponents, loser_components: d.loserComponents,
|
|
592
|
+
};
|
|
593
|
+
}
|
|
594
|
+
else if (d.action === 'skip_new') {
|
|
595
|
+
return {
|
|
596
|
+
type: 'skip_new', reason: d.reason,
|
|
597
|
+
existing_id: d.existingFactId, new_id: newClaimId,
|
|
598
|
+
entity_id: d.entityId, similarity: d.similarity,
|
|
599
|
+
winner_score: d.winnerScore, loser_score: d.loserScore,
|
|
600
|
+
winner_components: d.winnerComponents, loser_components: d.loserComponents,
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
else if (d.action === 'tie_leave_both') {
|
|
604
|
+
return {
|
|
605
|
+
type: 'tie_leave_both',
|
|
606
|
+
existing_id: d.existingFactId, new_id: newClaimId,
|
|
607
|
+
similarity: d.similarity, score_gap: Math.abs(d.winnerScore - d.loserScore),
|
|
608
|
+
entity_id: d.entityId, winner_score: d.winnerScore, loser_score: d.loserScore,
|
|
609
|
+
winner_components: d.winnerComponents, loser_components: d.loserComponents,
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
return { type: 'no_contradiction' };
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
/** Inline decision log building fallback (core < 1.5.0 or on failure). */
|
|
616
|
+
async function _appendDecisionLogInline(decisions, newClaimId, nowUnixSeconds, mode, logger) {
|
|
617
|
+
for (const d of decisions) {
|
|
618
|
+
if (d.action === 'supersede_existing') {
|
|
619
|
+
let loserClaimJson;
|
|
620
|
+
try {
|
|
621
|
+
loserClaimJson = JSON.stringify(d.existingClaim);
|
|
622
|
+
}
|
|
623
|
+
catch {
|
|
624
|
+
loserClaimJson = undefined;
|
|
625
|
+
}
|
|
626
|
+
await appendDecisionLog({
|
|
627
|
+
ts: nowUnixSeconds, entity_id: d.entityId,
|
|
628
|
+
new_claim_id: newClaimId, existing_claim_id: d.existingFactId,
|
|
629
|
+
similarity: d.similarity,
|
|
630
|
+
action: mode === 'shadow' ? 'shadow' : 'supersede_existing',
|
|
631
|
+
reason: 'new_wins',
|
|
632
|
+
winner_score: d.winnerScore, loser_score: d.loserScore,
|
|
633
|
+
winner_components: d.winnerComponents, loser_components: d.loserComponents,
|
|
634
|
+
loser_claim_json: loserClaimJson, mode,
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
else if (d.action === 'skip_new') {
|
|
638
|
+
await appendDecisionLog({
|
|
639
|
+
ts: nowUnixSeconds, entity_id: d.entityId,
|
|
640
|
+
new_claim_id: newClaimId, existing_claim_id: d.existingFactId,
|
|
641
|
+
similarity: d.similarity,
|
|
642
|
+
action: mode === 'shadow' ? 'shadow' : 'skip_new',
|
|
643
|
+
reason: d.reason,
|
|
644
|
+
winner_score: d.winnerScore, loser_score: d.loserScore,
|
|
645
|
+
winner_components: d.winnerComponents, loser_components: d.loserComponents,
|
|
646
|
+
mode,
|
|
647
|
+
});
|
|
648
|
+
}
|
|
649
|
+
else if (d.action === 'tie_leave_both') {
|
|
650
|
+
await appendDecisionLog({
|
|
651
|
+
ts: nowUnixSeconds, entity_id: d.entityId,
|
|
652
|
+
new_claim_id: newClaimId, existing_claim_id: d.existingFactId,
|
|
653
|
+
similarity: d.similarity,
|
|
654
|
+
action: 'tie_leave_both', reason: 'tie_below_tolerance',
|
|
655
|
+
winner_score: d.winnerScore, loser_score: d.loserScore,
|
|
656
|
+
winner_components: d.winnerComponents, loser_components: d.loserComponents,
|
|
657
|
+
mode,
|
|
658
|
+
});
|
|
659
|
+
logger.info(`Contradiction: tie (gap=${Math.abs(d.winnerScore - d.loserScore).toFixed(6)} < ${TIE_ZONE_SCORE_TOLERANCE}, sim=${d.similarity.toFixed(3)}, entity=${d.entityId}) — leaving both active`);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
// ---------------------------------------------------------------------------
|
|
664
|
+
// Slice 2f: feedback wiring (pin path) + weight-tuning loop
|
|
665
|
+
// ---------------------------------------------------------------------------
|
|
666
|
+
/** Path to `~/.totalreclaw/feedback.jsonl` honouring TOTALRECLAW_STATE_DIR. */
|
|
667
|
+
export function feedbackLogPath() {
|
|
668
|
+
return path.join(resolveStateDir(), 'feedback.jsonl');
|
|
669
|
+
}
|
|
670
|
+
/** Cap on feedback.jsonl lines; oldest dropped above this. */
|
|
671
|
+
export const FEEDBACK_LOG_MAX_LINES = 10_000;
|
|
672
|
+
/**
|
|
673
|
+
* Default minimum seconds between consecutive tuning-loop runs.
|
|
674
|
+
*
|
|
675
|
+
* Production uses one hour to protect against hot loops during rapid debugging.
|
|
676
|
+
* QA workflows that need to validate a feedback → weight-tuning cycle in a
|
|
677
|
+
* single agent session can override this via
|
|
678
|
+
* `TOTALRECLAW_TUNING_MIN_INTERVAL_OVERRIDE_SECONDS`. The override is
|
|
679
|
+
* unbounded — set it to 1 if you want every pin to immediately tune.
|
|
680
|
+
*/
|
|
681
|
+
export const TUNING_LOOP_MIN_INTERVAL_SECONDS = 3600;
|
|
682
|
+
/**
|
|
683
|
+
* Read the effective tuning-loop rate-limit interval, honouring the QA override.
|
|
684
|
+
*
|
|
685
|
+
* Reads the env var per-call so tests can flip it without module reload.
|
|
686
|
+
* Returns `TUNING_LOOP_MIN_INTERVAL_SECONDS` if the override is missing,
|
|
687
|
+
* empty, non-numeric, or negative.
|
|
688
|
+
*/
|
|
689
|
+
export function getTuningLoopMinIntervalSeconds() {
|
|
690
|
+
const raw = process.env.TOTALRECLAW_TUNING_MIN_INTERVAL_OVERRIDE_SECONDS;
|
|
691
|
+
if (raw === undefined || raw === null || raw === '') {
|
|
692
|
+
return TUNING_LOOP_MIN_INTERVAL_SECONDS;
|
|
693
|
+
}
|
|
694
|
+
const parsed = Number(raw);
|
|
695
|
+
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
696
|
+
return TUNING_LOOP_MIN_INTERVAL_SECONDS;
|
|
697
|
+
}
|
|
698
|
+
return parsed;
|
|
699
|
+
}
|
|
700
|
+
/**
|
|
701
|
+
* Walk `decisions.jsonl` in reverse and find the most recent `supersede_existing`
|
|
702
|
+
* entry that the target fact participated in. Slice 2f uses this to decide
|
|
703
|
+
* whether a pin/unpin call is a real counterexample (gradient signal) or a
|
|
704
|
+
* voluntary pin (no signal).
|
|
705
|
+
*
|
|
706
|
+
* `role` selects which side of the decision to match: `'loser'` finds entries
|
|
707
|
+
* where the fact was tombstoned by the formula (regular pin-after-override),
|
|
708
|
+
* `'winner'` finds entries where the fact was the formula's pick (reverse
|
|
709
|
+
* unpin-the-winner path).
|
|
710
|
+
*
|
|
711
|
+
* Returns null if the log is absent, empty, or has no matching entry with the
|
|
712
|
+
* Slice 2f component breakdown.
|
|
713
|
+
*/
|
|
714
|
+
export function findDecisionForPin(factId, role, logContent) {
|
|
715
|
+
if (!logContent || logContent.length === 0)
|
|
716
|
+
return null;
|
|
717
|
+
try {
|
|
718
|
+
const result = getWasm().findDecisionForPin(factId, role, logContent);
|
|
719
|
+
if (result === 'null')
|
|
720
|
+
return null;
|
|
721
|
+
return JSON.parse(result);
|
|
722
|
+
}
|
|
723
|
+
catch {
|
|
724
|
+
// Fallback: local implementation if WASM fails.
|
|
725
|
+
const lines = logContent.split('\n').filter((l) => l.length > 0);
|
|
726
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
727
|
+
let entry;
|
|
728
|
+
try {
|
|
729
|
+
entry = JSON.parse(lines[i]);
|
|
730
|
+
}
|
|
731
|
+
catch {
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
if (entry.action !== 'supersede_existing')
|
|
735
|
+
continue;
|
|
736
|
+
if (!entry.winner_components || !entry.loser_components)
|
|
737
|
+
continue;
|
|
738
|
+
if (role === 'loser' && entry.existing_claim_id === factId)
|
|
739
|
+
return entry;
|
|
740
|
+
if (role === 'winner' && entry.new_claim_id === factId)
|
|
741
|
+
return entry;
|
|
742
|
+
}
|
|
743
|
+
return null;
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
/**
|
|
747
|
+
* Walk `decisions.jsonl` in reverse and return the most recent canonical Claim
|
|
748
|
+
* JSON for a fact that was tombstoned by a `supersede_existing` decision.
|
|
749
|
+
*
|
|
750
|
+
* Used by the pin tool's recovery path: when `decryptBlob` fails on a
|
|
751
|
+
* tombstoned (`0x00`) on-chain blob, the pin tool falls back to this lookup
|
|
752
|
+
* to reconstruct the loser plaintext from the decision log instead of failing
|
|
753
|
+
* outright. See Phase 2.1 in the implementation plan.
|
|
754
|
+
*
|
|
755
|
+
* Reads `decisions.jsonl` synchronously from the active state dir. Returns
|
|
756
|
+
* null when the log is missing, empty, or has no matching row. Never throws —
|
|
757
|
+
* the recovery path treats any failure as "no recovery available".
|
|
758
|
+
*
|
|
759
|
+
* Only matches `supersede_existing` rows (not `tie_leave_both` or `skip_new`)
|
|
760
|
+
* because those are the only rows that actually tombstone the existing fact.
|
|
761
|
+
* Only returns rows that have `loser_claim_json` populated — pre-Phase-2.1
|
|
762
|
+
* supersede rows do not carry the field and cannot be recovered from.
|
|
763
|
+
*/
|
|
764
|
+
export function findLoserClaimInDecisionLog(factId) {
|
|
765
|
+
let logContent = '';
|
|
766
|
+
try {
|
|
767
|
+
logContent = fs.readFileSync(decisionsLogPath(), 'utf-8');
|
|
768
|
+
}
|
|
769
|
+
catch {
|
|
770
|
+
return null;
|
|
771
|
+
}
|
|
772
|
+
if (!logContent || logContent.length === 0)
|
|
773
|
+
return null;
|
|
774
|
+
try {
|
|
775
|
+
const result = getWasm().findLoserClaimInDecisionLog(factId, logContent);
|
|
776
|
+
return result === 'null' ? null : result;
|
|
777
|
+
}
|
|
778
|
+
catch {
|
|
779
|
+
// Fallback: local implementation if WASM fails.
|
|
780
|
+
const lines = logContent.split('\n').filter((l) => l.length > 0);
|
|
781
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
782
|
+
let entry;
|
|
783
|
+
try {
|
|
784
|
+
entry = JSON.parse(lines[i]);
|
|
785
|
+
}
|
|
786
|
+
catch {
|
|
787
|
+
continue;
|
|
788
|
+
}
|
|
789
|
+
if (entry.action !== 'supersede_existing')
|
|
790
|
+
continue;
|
|
791
|
+
if (entry.existing_claim_id !== factId)
|
|
792
|
+
continue;
|
|
793
|
+
if (typeof entry.loser_claim_json !== 'string' || entry.loser_claim_json.length === 0) {
|
|
794
|
+
continue;
|
|
795
|
+
}
|
|
796
|
+
return entry.loser_claim_json;
|
|
797
|
+
}
|
|
798
|
+
return null;
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* Build a `FeedbackEntry` from a matching decision-log row + pin action.
|
|
803
|
+
*
|
|
804
|
+
* For `supersede_existing`, the formula's winner is always the new claim
|
|
805
|
+
* (`new_claim_id`) and the loser is the existing claim. If the user later
|
|
806
|
+
* pins the loser, `user_decision = 'pin_a'` with `claim_a` = the loser
|
|
807
|
+
* (so claim_a is what the user wants kept). If the user later unpins the
|
|
808
|
+
* winner — an inverse override — we record `user_decision = 'pin_b'` to
|
|
809
|
+
* keep the schema symmetrical.
|
|
810
|
+
*/
|
|
811
|
+
export function buildFeedbackFromDecision(decision, action, nowUnixSeconds) {
|
|
812
|
+
if (!decision.winner_components || !decision.loser_components)
|
|
813
|
+
return null;
|
|
814
|
+
try {
|
|
815
|
+
const result = getWasm().buildFeedbackFromDecision(JSON.stringify(decision), action, Math.floor(nowUnixSeconds));
|
|
816
|
+
if (result === 'null')
|
|
817
|
+
return null;
|
|
818
|
+
return JSON.parse(result);
|
|
819
|
+
}
|
|
820
|
+
catch {
|
|
821
|
+
// Fallback: local implementation if WASM fails.
|
|
822
|
+
if (action === 'pin_loser') {
|
|
823
|
+
return {
|
|
824
|
+
ts: nowUnixSeconds,
|
|
825
|
+
claim_a_id: decision.existing_claim_id,
|
|
826
|
+
claim_b_id: decision.new_claim_id,
|
|
827
|
+
formula_winner: 'b',
|
|
828
|
+
user_decision: 'pin_a',
|
|
829
|
+
winner_components: decision.winner_components,
|
|
830
|
+
loser_components: decision.loser_components,
|
|
831
|
+
};
|
|
832
|
+
}
|
|
833
|
+
return {
|
|
834
|
+
ts: nowUnixSeconds,
|
|
835
|
+
claim_a_id: decision.existing_claim_id,
|
|
836
|
+
claim_b_id: decision.new_claim_id,
|
|
837
|
+
formula_winner: 'b',
|
|
838
|
+
user_decision: 'pin_b',
|
|
839
|
+
winner_components: decision.winner_components,
|
|
840
|
+
loser_components: decision.loser_components,
|
|
841
|
+
};
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
/**
|
|
845
|
+
* Append one feedback entry to `~/.totalreclaw/feedback.jsonl`, rotating if
|
|
846
|
+
* over the cap. Uses the WASM core bindings so the file format is byte-for-byte
|
|
847
|
+
* compatible with the Rust + Python clients. Never throws.
|
|
848
|
+
*/
|
|
849
|
+
export async function appendFeedbackLog(entry) {
|
|
850
|
+
try {
|
|
851
|
+
const core = getWasm();
|
|
852
|
+
const dir = ensureStateDir();
|
|
853
|
+
const p = path.join(dir, 'feedback.jsonl');
|
|
854
|
+
let existing = '';
|
|
855
|
+
try {
|
|
856
|
+
existing = fs.readFileSync(p, 'utf-8');
|
|
857
|
+
}
|
|
858
|
+
catch {
|
|
859
|
+
existing = '';
|
|
860
|
+
}
|
|
861
|
+
const appended = core.appendFeedbackToJsonl(existing, JSON.stringify(entry));
|
|
862
|
+
const rotated = core.rotateFeedbackLog(appended, BigInt(FEEDBACK_LOG_MAX_LINES));
|
|
863
|
+
fs.writeFileSync(p, rotated, 'utf-8');
|
|
864
|
+
}
|
|
865
|
+
catch {
|
|
866
|
+
// Best-effort; feedback logging is never fatal.
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
/**
|
|
870
|
+
* Slice 2f glue: on pin/unpin, consult `decisions.jsonl` and write a feedback
|
|
871
|
+
* row if the user override contradicts a prior formula decision.
|
|
872
|
+
*
|
|
873
|
+
* Returns the entry that was appended, or null when the pin/unpin was
|
|
874
|
+
* voluntary (no matching decision row). Logs info-level on voluntary pins
|
|
875
|
+
* and debug-level on each counterexample written.
|
|
876
|
+
*/
|
|
877
|
+
export async function maybeWriteFeedbackForPin(factId, targetStatus, nowUnixSeconds, logger) {
|
|
878
|
+
let logContent = '';
|
|
879
|
+
try {
|
|
880
|
+
logContent = fs.readFileSync(decisionsLogPath(), 'utf-8');
|
|
881
|
+
}
|
|
882
|
+
catch {
|
|
883
|
+
logContent = '';
|
|
884
|
+
}
|
|
885
|
+
// For pin: the user is saying the loser was right → match loser.
|
|
886
|
+
// For unpin: the user is flipping the winner back → match winner.
|
|
887
|
+
const role = targetStatus === 'pinned' ? 'loser' : 'winner';
|
|
888
|
+
const decision = findDecisionForPin(factId, role, logContent);
|
|
889
|
+
if (!decision) {
|
|
890
|
+
logger.info(targetStatus === 'pinned'
|
|
891
|
+
? `Pin feedback: no matching auto-resolution for ${factId.slice(0, 10)}… (voluntary pin, no tuning signal)`
|
|
892
|
+
: `Unpin feedback: no matching auto-resolution for ${factId.slice(0, 10)}… (voluntary unpin, no tuning signal)`);
|
|
893
|
+
return null;
|
|
894
|
+
}
|
|
895
|
+
const action = targetStatus === 'pinned' ? 'pin_loser' : 'unpin_winner';
|
|
896
|
+
const entry = buildFeedbackFromDecision(decision, action, nowUnixSeconds);
|
|
897
|
+
if (!entry)
|
|
898
|
+
return null;
|
|
899
|
+
await appendFeedbackLog(entry);
|
|
900
|
+
logger.info(`Pin feedback: recorded counterexample (${entry.user_decision}) for ${factId.slice(0, 10)}…`);
|
|
901
|
+
return entry;
|
|
902
|
+
}
|
|
903
|
+
/**
|
|
904
|
+
* Core of the weight-tuning loop. Pure enough to test in isolation: reads
|
|
905
|
+
* `feedback.jsonl`, replays every entry newer than `weightsFile.last_tuning_ts`
|
|
906
|
+
* through the WASM `feedbackToCounterexample` + `applyFeedback` pair, writes
|
|
907
|
+
* back the adjusted weights. Idempotent: re-running with the same feedback
|
|
908
|
+
* file does nothing because the timestamp advances each pass.
|
|
909
|
+
*
|
|
910
|
+
* Rate-limited: if the current `updated_at` is within
|
|
911
|
+
* `TUNING_LOOP_MIN_INTERVAL_SECONDS` of `nowUnixSeconds`, returns early with
|
|
912
|
+
* `skipped: 'rate-limited'`. Never throws.
|
|
913
|
+
*/
|
|
914
|
+
export async function runWeightTuningLoop(nowUnixSeconds, logger) {
|
|
915
|
+
const core = getWasm();
|
|
916
|
+
let weightsFile;
|
|
917
|
+
try {
|
|
918
|
+
weightsFile = await loadWeightsFile(nowUnixSeconds);
|
|
919
|
+
}
|
|
920
|
+
catch {
|
|
921
|
+
return { processed: 0, gradientSteps: 0, skipped: 'no-weights', lastTuningTs: 0 };
|
|
922
|
+
}
|
|
923
|
+
// Rate limit: if the weights file was touched very recently and there is a
|
|
924
|
+
// last_tuning_ts, skip — protects against hot loops during rapid debugging.
|
|
925
|
+
// QA can lower the interval via TOTALRECLAW_TUNING_MIN_INTERVAL_OVERRIDE_SECONDS.
|
|
926
|
+
const updatedAt = typeof weightsFile.updated_at === 'number' ? weightsFile.updated_at : 0;
|
|
927
|
+
const priorTuningTs = typeof weightsFile.last_tuning_ts === 'number' ? weightsFile.last_tuning_ts : 0;
|
|
928
|
+
const minInterval = getTuningLoopMinIntervalSeconds();
|
|
929
|
+
if (priorTuningTs > 0 &&
|
|
930
|
+
updatedAt > 0 &&
|
|
931
|
+
nowUnixSeconds - updatedAt < minInterval) {
|
|
932
|
+
return {
|
|
933
|
+
processed: 0,
|
|
934
|
+
gradientSteps: 0,
|
|
935
|
+
skipped: 'rate-limited',
|
|
936
|
+
lastTuningTs: priorTuningTs,
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
// Read feedback.jsonl.
|
|
940
|
+
let feedbackContent = '';
|
|
941
|
+
try {
|
|
942
|
+
feedbackContent = fs.readFileSync(feedbackLogPath(), 'utf-8');
|
|
943
|
+
}
|
|
944
|
+
catch {
|
|
945
|
+
feedbackContent = '';
|
|
946
|
+
}
|
|
947
|
+
if (!feedbackContent || feedbackContent.length === 0) {
|
|
948
|
+
return {
|
|
949
|
+
processed: 0,
|
|
950
|
+
gradientSteps: 0,
|
|
951
|
+
skipped: 'no-new-entries',
|
|
952
|
+
lastTuningTs: priorTuningTs,
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
let parsed;
|
|
956
|
+
try {
|
|
957
|
+
parsed = JSON.parse(core.readFeedbackJsonl(feedbackContent));
|
|
958
|
+
}
|
|
959
|
+
catch (err) {
|
|
960
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
961
|
+
logger.warn(`Tuning loop: failed to parse feedback.jsonl: ${msg}`);
|
|
962
|
+
return {
|
|
963
|
+
processed: 0,
|
|
964
|
+
gradientSteps: 0,
|
|
965
|
+
skipped: 'no-new-entries',
|
|
966
|
+
lastTuningTs: priorTuningTs,
|
|
967
|
+
};
|
|
968
|
+
}
|
|
969
|
+
for (const w of parsed.warnings)
|
|
970
|
+
logger.warn(`Tuning loop: ${w}`);
|
|
971
|
+
const newEntries = parsed.entries.filter((e) => e.ts > priorTuningTs);
|
|
972
|
+
if (newEntries.length === 0) {
|
|
973
|
+
return {
|
|
974
|
+
processed: 0,
|
|
975
|
+
gradientSteps: 0,
|
|
976
|
+
skipped: 'no-new-entries',
|
|
977
|
+
lastTuningTs: priorTuningTs,
|
|
978
|
+
};
|
|
979
|
+
}
|
|
980
|
+
let weightsJson = JSON.stringify(weightsFile.weights ?? {});
|
|
981
|
+
let gradientSteps = 0;
|
|
982
|
+
let maxTs = priorTuningTs;
|
|
983
|
+
for (const entry of newEntries) {
|
|
984
|
+
if (entry.ts > maxTs)
|
|
985
|
+
maxTs = entry.ts;
|
|
986
|
+
let cxJson;
|
|
987
|
+
try {
|
|
988
|
+
cxJson = core.feedbackToCounterexample(JSON.stringify(entry));
|
|
989
|
+
}
|
|
990
|
+
catch (err) {
|
|
991
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
992
|
+
logger.warn(`Tuning loop: feedbackToCounterexample failed: ${msg}`);
|
|
993
|
+
continue;
|
|
994
|
+
}
|
|
995
|
+
if (cxJson === 'null')
|
|
996
|
+
continue;
|
|
997
|
+
try {
|
|
998
|
+
weightsJson = core.applyFeedback(weightsJson, cxJson);
|
|
999
|
+
gradientSteps += 1;
|
|
1000
|
+
}
|
|
1001
|
+
catch (err) {
|
|
1002
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1003
|
+
logger.warn(`Tuning loop: applyFeedback failed: ${msg}`);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
let adjustedWeights;
|
|
1007
|
+
try {
|
|
1008
|
+
adjustedWeights = JSON.parse(weightsJson);
|
|
1009
|
+
}
|
|
1010
|
+
catch {
|
|
1011
|
+
return {
|
|
1012
|
+
processed: newEntries.length,
|
|
1013
|
+
gradientSteps,
|
|
1014
|
+
skipped: 'no-weights',
|
|
1015
|
+
lastTuningTs: maxTs,
|
|
1016
|
+
};
|
|
1017
|
+
}
|
|
1018
|
+
const nextFile = {
|
|
1019
|
+
...weightsFile,
|
|
1020
|
+
weights: adjustedWeights,
|
|
1021
|
+
updated_at: nowUnixSeconds,
|
|
1022
|
+
last_tuning_ts: maxTs,
|
|
1023
|
+
feedback_count: (typeof weightsFile.feedback_count === 'number' ? weightsFile.feedback_count : 0) +
|
|
1024
|
+
newEntries.length,
|
|
1025
|
+
};
|
|
1026
|
+
await saveWeightsFile(nextFile);
|
|
1027
|
+
logger.info(`Tuning loop: processed ${newEntries.length} feedback entries, applied ${gradientSteps} gradient steps`);
|
|
1028
|
+
return {
|
|
1029
|
+
processed: newEntries.length,
|
|
1030
|
+
gradientSteps,
|
|
1031
|
+
skipped: null,
|
|
1032
|
+
lastTuningTs: maxTs,
|
|
1033
|
+
};
|
|
1034
|
+
}
|