memhouse 0.18.12 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +22 -151
- package/CHANGELOG.md +263 -29
- package/LICENSE +202 -22
- package/LICENSES/agentlytics-MIT.txt +21 -0
- package/LICENSES/memhouse-MIT-until-0.18.11.txt +22 -0
- package/LICENSES/models.dev-MIT.txt +22 -0
- package/NOTICE +28 -0
- package/README.md +42 -20
- package/SECURITY.md +21 -8
- package/bin/memhouse.js +2223 -311
- package/docs/README.md +11 -9
- package/docs/guides/admin-password.md +37 -10
- package/docs/guides/configuration.md +13 -6
- package/docs/guides/daemons.md +64 -3
- package/docs/guides/import-claude-ai.md +204 -0
- package/docs/guides/install.md +11 -6
- package/docs/guides/invite.md +21 -7
- package/docs/guides/operate.md +114 -7
- package/docs/guides/search-and-resume.md +72 -3
- package/docs/guides/share.md +91 -52
- package/docs/guides/storage.md +44 -4
- package/docs/guides/troubleshooting.md +121 -14
- package/docs/guides/uninstall.md +4 -3
- package/docs/guides/update.md +16 -0
- package/docs/guides/upgrade-to-0.20.md +188 -0
- package/docs/releases/README.md +2 -1
- package/docs/releases/v0.18.10.md +5 -5
- package/docs/releases/v0.18.11.md +3 -3
- package/docs/releases/v0.20.0.md +627 -0
- package/docs/releases/v0.20.1.md +114 -0
- package/docs/tutorials/01-first-house.md +10 -7
- package/docs/tutorials/02-join-a-house.md +24 -15
- package/docs/tutorials/03-share-your-memory.md +40 -31
- package/editors/antigravity.js +1 -80
- package/editors/base.js +463 -9
- package/editors/bridge.js +119 -0
- package/editors/claude.js +273 -105
- package/editors/codex.js +278 -52
- package/editors/copilot.js +1 -70
- package/editors/cursor.js +1 -75
- package/editors/derive-fingerprints.json +18 -0
- package/editors/harness-paths.js +69 -0
- package/editors/index.js +1 -20
- package/editors/normalise/agy.js +910 -0
- package/editors/normalise/copilot-cli.js +321 -0
- package/editors/normalise/cursor-agent.js +852 -0
- package/editors/normalise/cursor.js +474 -0
- package/editors/normalise/devin.js +575 -0
- package/editors/normalise/gemini-cli.js +602 -0
- package/editors/normalise/goose.js +673 -0
- package/editors/normalise/opencode.js +633 -0
- package/editors/normalise/vscode.js +549 -0
- package/editors/normalise/zed.js +473 -0
- package/editors/opencode-v4.js +119 -0
- package/editors/scope.js +14 -3
- package/editors/sessions-log.js +349 -0
- package/editors/sqlite.js +28 -1
- package/editors/vscode.js +1 -72
- package/editors/windsurf.js +1 -143
- package/examples/01-local-one-machine/README.md +24 -19
- package/examples/01-local-one-machine/run.sh +1 -1
- package/examples/02-existing-clickhouse/README.md +30 -22
- package/examples/02-existing-clickhouse/run.sh +5 -4
- package/examples/03-team-house/README.md +44 -37
- package/examples/03-team-house/run.sh +35 -18
- package/examples/04-self-hosted-team/README.md +29 -14
- package/examples/04-self-hosted-team/compose.yml +5 -2
- package/examples/04-self-hosted-team/smoke.sh +26 -12
- package/examples/README.md +8 -4
- package/examples/fixtures/README.md +4 -1
- package/examples/fixtures/make-sessions.js +8 -5
- package/memhouse/admin-secret.js +89 -24
- package/memhouse/argv-secrets.js +64 -0
- package/memhouse/channel.js +49 -9
- package/memhouse/codex-plugin.js +194 -0
- package/memhouse/delivery/AGENT-INSTALL-PROMPT.md +2 -1
- package/memhouse/delivery/AGENT-INSTALL.md +22 -11
- package/memhouse/delivery/INVITATION.md.tpl +44 -45
- package/memhouse/delivery/PROMPT.md +53 -5
- package/memhouse/delivery/UPGRADE.md.tpl +1 -1
- package/memhouse/delivery/codex/AGENTS-BLOCK.md +11 -0
- package/memhouse/delivery/codex/SKILL.md +50 -0
- package/memhouse/delivery/kernel-install.md +17 -9
- package/memhouse/delivery/plugin/.claude-plugin/plugin.json +2 -2
- package/memhouse/delivery/plugin/reference/HOUSE.md +168 -41
- package/memhouse/delivery/plugin/skills/access/SKILL.md +13 -5
- package/memhouse/delivery/plugin/skills/admin/SKILL.md +14 -6
- package/memhouse/delivery/plugin/skills/house/SKILL.md +25 -8
- package/memhouse/delivery/plugin/skills/recall/SKILL.md +66 -8
- package/memhouse/delivery/plugin/skills/sql/SKILL.md +66 -7
- package/memhouse/deploy.js +1 -1
- package/memhouse/flags.js +35 -18
- package/memhouse/help.js +37 -0
- package/memhouse/host.js +82 -2
- package/memhouse/house/HOUSE.md +2 -2
- package/memhouse/house/convert-profile.js +638 -0
- package/memhouse/house/convert-v3.js +742 -0
- package/memhouse/house/doctor-v3.js +324 -0
- package/memhouse/house/host-alias.js +85 -0
- package/memhouse/house/house.js +3 -3
- package/memhouse/house/migrate.js +2 -2
- package/memhouse/house/profile-dedupe.js +183 -0
- package/memhouse/house/profile-edges.js +554 -0
- package/memhouse/house/profile-house.js +430 -0
- package/memhouse/house/profile-rollup.js +190 -0
- package/memhouse/house/q-cli.js +863 -0
- package/memhouse/house/queries-v3.js +1013 -0
- package/memhouse/house/reconcile-v3.js +192 -0
- package/memhouse/house/records-to-profile.js +174 -0
- package/memhouse/house/records-to-rows.js +169 -0
- package/memhouse/house/relocate.js +2 -1
- package/memhouse/house/schema-choice.js +121 -0
- package/memhouse/house/schema-profile.js +290 -0
- package/memhouse/house/schema-profile.sql.tpl +398 -0
- package/memhouse/house/schema-v3.js +267 -0
- package/memhouse/house/schema-v3.sql.tpl +213 -0
- package/memhouse/import/claude-ai.js +420 -0
- package/memhouse/import/import.js +372 -0
- package/memhouse/import/pyjson.js +206 -0
- package/memhouse/import/records.js +64 -0
- package/memhouse/import/writer-profile.js +192 -0
- package/memhouse/import/writer-v3.js +240 -0
- package/memhouse/import/zip.js +112 -0
- package/memhouse/invitation.js +102 -0
- package/memhouse/invitefile.js +5 -2
- package/memhouse/plugin-copies.js +176 -0
- package/memhouse/provision.js +7 -2
- package/memhouse/release-hint.js +24 -0
- package/memhouse/resume.js +4 -4
- package/memhouse/server/api-v3.js +632 -0
- package/memhouse/server/dashboard-auth.js +181 -0
- package/memhouse/server/dashboard-port.js +91 -0
- package/memhouse/server/queries.js +30 -0
- package/memhouse/server/server.js +118 -20
- package/memhouse/service.js +1 -1
- package/memhouse/share.js +687 -1
- package/memhouse/shipper/adapter-capture.js +756 -0
- package/memhouse/shipper/capture.js +2571 -0
- package/memhouse/shipper/dedupe-profile.js +237 -0
- package/memhouse/shipper/dedupe-v3.js +1027 -0
- package/memhouse/shipper/forks.js +233 -0
- package/memhouse/shipper/log-adapter.js +219 -0
- package/memhouse/shipper/maintenance.js +49 -0
- package/memhouse/shipper/normalise-run.js +148 -0
- package/memhouse/shipper/passlog.js +68 -5
- package/memhouse/shipper/profile-capture.js +903 -0
- package/memhouse/shipper/ship.js +91 -8
- package/memhouse/shipper/snapshots.js +193 -0
- package/memhouse/shipper/source-roots.js +111 -0
- package/memhouse/version-notice.js +66 -0
- package/package.json +14 -7
- package/public/THIRD_PARTY_NOTICES.txt +2 -2
- package/public/assets/index-CcNnhfM3.js +78 -0
- package/public/index.html +1 -1
- package/ui/package-lock.json +56 -37
- package/ui/package.json +1 -0
- package/ui/src/App.jsx +29 -14
- package/ui/src/components/ChatSidebar.jsx +69 -12
- package/ui/src/lib/api.js +76 -0
- package/ui/src/lib/dashboard-token.js +41 -0
- package/ui/src/main.jsx +22 -5
- package/ui/src/pages/Dashboard.jsx +22 -36
- package/ui/third-party-notices.js +119 -0
- package/ui/vite.config.js +5 -1
- package/docs/releases/v0.18.12.md +0 -59
- package/memhouse/COMPETITION.md +0 -221
- package/memhouse/DESIGN.md +0 -222
- package/memhouse/RESUME-AND-UPDATE.md +0 -182
- package/public/assets/index-CnWzrjF3.js +0 -78
|
@@ -0,0 +1,554 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Engine edges and fork billing once on the profile (M4; sessions-model §2.5, harness-sessions-sql §4.5 and
|
|
3
|
+
// §5 step 4). A fork copies its origin's messages, and often their usage: summed per session, the
|
|
4
|
+
// house would bill the copied prefix twice. The adapters state the link (session_links, owner
|
|
5
|
+
// 'adapter'); the engine resolves the statements into edges (owner 'engine', keyed on the child, the
|
|
6
|
+
// origin as the other side), verifies each on content, and lists on it the fork's usage rows that copy
|
|
7
|
+
// the origin's (r_copied_usage). The rollup reads only edges and leaves those rows out of the fork's
|
|
8
|
+
// totals; they stay in the fork, searchable, and count in the origin.
|
|
9
|
+
//
|
|
10
|
+
// THE RULES (each line is a decision)
|
|
11
|
+
// statements every link kind (§2.5: subagent, fork, rewind_copy, clear, continuation, and any
|
|
12
|
+
// kind an adapter states), in one harness store: the other side's host and data folder
|
|
13
|
+
// empty (the stating session's own store, §2.5) or the same. Another store's link is
|
|
14
|
+
// never resolved here
|
|
15
|
+
// lineage every kind gets its engine edge (harness-sessions-sql §4.5). Only fork and rewind_copy,
|
|
16
|
+
// outside the native harnesses, are verified and billed (below); the others are
|
|
17
|
+
// lineage edges: verified null, nothing copied, their statements and evidence kept
|
|
18
|
+
// merge (§2.5) statements of one (child, origin, kind) merge into one edge when their `at`
|
|
19
|
+
// is equal or empty on one side, and each could pair with no other statement on the
|
|
20
|
+
// other side; the edge keeps the non-empty `at` and is stated_on both. Conflicting
|
|
21
|
+
// `at`s stay separate edges. Statements are taken in a fixed order, so an edge's key
|
|
22
|
+
// never depends on the order the house returns them in. Statements with one `at` share
|
|
23
|
+
// an edge even where §2.5 would keep them apart (a statement that could pair with two):
|
|
24
|
+
// the key (child, kind, origin, at) holds one edge
|
|
25
|
+
// incarnation a reference without one takes the only live one of that id in the store; none
|
|
26
|
+
// (the other side not captured yet) or two leave the edge unverified
|
|
27
|
+
// verified the fork's main-line messages (no sub-agent, not reverted, retracted or off the
|
|
28
|
+
// active branch), in order from its first, equal the origin's from its first, each on
|
|
29
|
+
// a digest of its role and its parts' content in order (the cmp_sha projection of
|
|
30
|
+
// sessions-model §2: text, tool name and args, output and is_error, media and bytes).
|
|
31
|
+
// Time is never compared; ids only where a statement says ids_preserved. The match
|
|
32
|
+
// stops after the edge's through_message_id, else after its `at` (the fork point).
|
|
33
|
+
// A truncated part on either side in the compared run, or a first message that
|
|
34
|
+
// differs, leaves it unverified
|
|
35
|
+
// captured where a session holds both captured rows and converted 0.18 ones (adapter
|
|
36
|
+
// first 'legacy', until the dedupe absorbs them), only the captured are compared and billed,
|
|
37
|
+
// per table. Converted rows the dedupe keeps (their transcript gone) are not compared:
|
|
38
|
+
// the match stops at the first of them, and the copies from there on are billed in
|
|
39
|
+
// the fork as well (never too little)
|
|
40
|
+
// billed once a fork usage row anchored at a copied message (or its response) whose numbers
|
|
41
|
+
// equal an origin usage row's at the paired message (each origin row matched once).
|
|
42
|
+
// A message the fork re-ran is billed where it ran. Session- and compaction-anchored
|
|
43
|
+
// usage has no place in the prefix and is billed as given
|
|
44
|
+
// kept current recomputed for every session the capture writes, as child and as origin: a
|
|
45
|
+
// rewound origin gives its copies back, a fork captured first is verified when its
|
|
46
|
+
// origin arrives. An edge no statement makes any more ends (a removed version). Every
|
|
47
|
+
// fork an edge ever joined is reported (children), ended ones too, so a refold can
|
|
48
|
+
// refresh them all
|
|
49
|
+
// inferred (sessions-model §2.5: inferred links belong to the consumer) the harnesses that
|
|
50
|
+
// keep no fork pointer and mint new ids (OpenCode, VS Code, cursor-agent, Cursor,
|
|
51
|
+
// Goose): a session whose first main-line assistant text (at least 40 characters) is
|
|
52
|
+
// another's, born strictly earlier in the same store, is nominated as that one's fork,
|
|
53
|
+
// and verified as above (no `at`, no ids). The longest verified prefix wins; at an equal
|
|
54
|
+
// length the earliest born (two forks of one origin each verify against the other's
|
|
55
|
+
// copies as far as against the origin); an equal length at an equal birth is no
|
|
56
|
+
// edge. An inferred edge (evidence 'inferred', stated_on '', no statements) ends when
|
|
57
|
+
// its verification no longer holds. A session with an explicit statement as the fork
|
|
58
|
+
// gets no inferred edge. Kept current as the explicit ones: as the fork, and as the
|
|
59
|
+
// origin (the forks its inferred edges name, and the sessions sharing its first reply
|
|
60
|
+
// born after it). Two independent sessions whose first exchanges are the same, word
|
|
61
|
+
// for word and token for token, are indistinguishable from a fork
|
|
62
|
+
// budget a pass verifies at most FORK_BUDGET nominated forks; the rest are returned as
|
|
63
|
+
// deferred, and the capture owes them to the next pass. A recompute that starts with
|
|
64
|
+
// the budget spent owes only itself (it expands on the next pass); one that runs out
|
|
65
|
+
// part way owes each fork it did not reach. A recompute after the session's rows changed
|
|
66
|
+
// re-nominates its forks from the origin's side, whatever this pass did with them before
|
|
67
|
+
// (its rows may be new); one whose rows did not change (an owed session, `expand`
|
|
68
|
+
// false) nominates only itself, unless it still owes the expansion it skipped
|
|
69
|
+
// (`expansionOwed`, which the capture keeps in its local state)
|
|
70
|
+
// native Claude Code and Codex copies are not verified to carry usage: their edges are
|
|
71
|
+
// lineage only. Their sessions are queried only when the caller says the session has
|
|
72
|
+
// or had a link record, or is captured for the first time (`linked`): an ordinary
|
|
73
|
+
// session costs no query, and an edge named only from the other side is made when
|
|
74
|
+
// that side is captured
|
|
75
|
+
|
|
76
|
+
const crypto = require('crypto');
|
|
77
|
+
|
|
78
|
+
const COPY_KINDS = ['fork', 'rewind_copy'];
|
|
79
|
+
const NATIVE = new Set(['claude-code', 'codex']);
|
|
80
|
+
const INFERRED = new Set(['opencode', 'vscode', 'cursor-agent', 'cursor', 'goose']);
|
|
81
|
+
const MIN_NOMINATION_TEXT = 40;
|
|
82
|
+
const NOMINEES = 20; // candidates verified per fork: the 10 earliest born and the 10 latest born before it
|
|
83
|
+
const SHARERS = 1000; // sessions read back as sharing one first reply
|
|
84
|
+
const FORK_BUDGET = 100; // nominations a pass verifies (each up to NOMINEES candidates)
|
|
85
|
+
const EDGE_ADAPTER = 'memhouse';
|
|
86
|
+
const EDGE_VER = 3; // 3: every link kind; 2: inferred edges; 1: explicit fork and rewind_copy
|
|
87
|
+
const TOK = ['i', 'o', 'cr', 'cw', 'rs', 'tp'];
|
|
88
|
+
|
|
89
|
+
const lit = (v) => `'${String(v ?? '').replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
|
|
90
|
+
const STORE = '(host, harness, harness_path, session_id, incarnation)';
|
|
91
|
+
const storeWhere = (ref) => `${STORE} = (${[ref.host, ref.harness, ref.harness_path, ref.session_id, ref.incarnation || ''].map(lit).join(', ')})`;
|
|
92
|
+
const inStore = (s) => `host = ${lit(s.host)} AND harness = ${lit(s.harness)} AND harness_path = ${lit(s.harness_path)}`;
|
|
93
|
+
const skey = (r) => [r.host, r.harness, r.harness_path, r.session_id, r.incarnation || ''].map((x) => String(x ?? '')).join('\u0000');
|
|
94
|
+
const sha = (s) => crypto.createHash('sha256').update(s).digest('hex');
|
|
95
|
+
|
|
96
|
+
/** The adapter's fork statements that name this session, as the stating side or the other one. */
|
|
97
|
+
function statementsSQL(P, ref) {
|
|
98
|
+
return `SELECT session_id, incarnation, toString(kind) AS kind, toString(stated_on) AS stated_on, at,
|
|
99
|
+
other_session_id, other_incarnation, child_session_id, child_incarnation, origin_session_id, origin_incarnation,
|
|
100
|
+
through_message_id, ids_preserved, marked, usage_copied, child_stored
|
|
101
|
+
FROM ${P.session_links} FINAL
|
|
102
|
+
WHERE owner = 'adapter' AND removed = 0 AND ${inStore(ref)}
|
|
103
|
+
AND other_host IN ('', ${lit(ref.host)}) AND other_harness_path IN ('', ${lit(ref.harness_path)}) AND other_harness IN ('', ${lit(ref.harness)})
|
|
104
|
+
AND (session_id = ${lit(ref.session_id)} OR other_session_id = ${lit(ref.session_id)})`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The engine edges this session is the child or the origin of, ended ones (removed 1) included. */
|
|
108
|
+
function edgesSQL(P, ref) {
|
|
109
|
+
return `SELECT session_id, incarnation, toString(kind) AS kind, toString(other_host) AS other_host, toString(other_harness_path) AS other_harness_path,
|
|
110
|
+
toString(other_harness) AS other_harness, other_session_id, other_incarnation, at, toString(stated_on) AS stated_on,
|
|
111
|
+
through_message_id, ids_preserved, marked, usage_copied, child_stored, toString(evidence) AS evidence, verified, statements, r_copied_usage, r_copied_messages, r_unverified, removed, toString(rev) AS rev
|
|
112
|
+
FROM ${P.session_links} FINAL
|
|
113
|
+
WHERE owner = 'engine' AND ${inStore(ref)}
|
|
114
|
+
AND (session_id = ${lit(ref.session_id)} OR (other_host = ${lit(ref.host)} AND other_harness_path = ${lit(ref.harness_path)} AND other_session_id = ${lit(ref.session_id)}))`;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** The live lives (incarnations) of these session ids in one store. */
|
|
118
|
+
function livesSQL(P, store, ids) {
|
|
119
|
+
return `SELECT DISTINCT session_id, incarnation FROM ${P.session_parts} FINAL
|
|
120
|
+
WHERE ${inStore(store)} AND removed = 0 AND session_id IN (${ids.map(lit).join(', ')})`;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// One part's content, as sessions-model §2's cmp_sha projects it: never an id or a time.
|
|
124
|
+
const PART_LINE = `concat(toString(kind), '\\x1f', text, '\\x1f', toString(tool_name), '\\x1f', args, '\\x1f', output, '\\x1f',
|
|
125
|
+
toString(ifNull(is_error, false)), '\\x1f', output_ref_sha256, '\\x1f', toString(media_type), '\\x1f',
|
|
126
|
+
if(inline_b64 != '', lower(hex(SHA256(tryBase64Decode(inline_b64)))), ref), '\\x1f', toString(ifNull(opaque, false)))`;
|
|
127
|
+
// A session's captured rows where it has any, else its converted ones (header: captured first).
|
|
128
|
+
const capturedFirst = (table, ref) => `(adapter != 'legacy' OR (SELECT count() FROM ${table} FINAL WHERE ${storeWhere(ref)} AND removed = 0 AND adapter != 'legacy') = 0)`;
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* A session's main-line messages in order: id, role (mrole), response id, truncated, and a content
|
|
132
|
+
* digest. No alias shadows a column: ClickHouse would resolve the column's name to the alias.
|
|
133
|
+
*/
|
|
134
|
+
function messagesSQL(P, ref) {
|
|
135
|
+
return `SELECT message_id AS id, any(toString(role)) AS mrole, anyIf(response_id, response_id != '') AS rid,
|
|
136
|
+
max(ifNull(truncated, false)) AS trunc,
|
|
137
|
+
lower(hex(SHA256(concat(any(toString(role)), '\\x1e', arrayStringConcat(arrayMap(x -> x.2, arraySort(x -> x.1, groupArray((part_index, ${PART_LINE})))), '\\x1e'))))) AS digest,
|
|
138
|
+
min(order_key) AS ok, toString(min(ts)) AS mts
|
|
139
|
+
FROM ${P.session_parts} FINAL
|
|
140
|
+
WHERE ${storeWhere(ref)} AND removed = 0 AND agent_id = '' AND ifNull(on_active_branch, true)
|
|
141
|
+
AND if(part_state != '', part_state, state) NOT IN ('reverted', 'retracted') AND ${capturedFirst(P.session_parts, ref)}
|
|
142
|
+
GROUP BY message_id ORDER BY ok, mts, id`;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// The main-line assistant text parts nomination reads (as messagesSQL's line).
|
|
146
|
+
const REPLY_PARTS = `removed = 0 AND agent_id = '' AND ifNull(on_active_branch, true) AND if(part_state != '', part_state, state) NOT IN ('reverted', 'retracted')
|
|
147
|
+
AND role = 'assistant' AND kind = 'text' AND text != ''`;
|
|
148
|
+
|
|
149
|
+
// A session's parts in order: its messages (order, then time), then each message's parts. The part
|
|
150
|
+
// index breaks the tie between the text parts of one message.
|
|
151
|
+
const PART_ORDER = `(order_key, ifNull(ts, toDateTime64(0, 3, 'UTC')), part_index)`;
|
|
152
|
+
|
|
153
|
+
/** A session's first main-line assistant text ('' when it has none). */
|
|
154
|
+
function firstReplySQL(P, ref) {
|
|
155
|
+
return `SELECT argMin(text, ${PART_ORDER}) AS t FROM ${P.session_parts} FINAL
|
|
156
|
+
WHERE ${storeWhere(ref)} AND ${REPLY_PARTS} AND ${capturedFirst(P.session_parts, ref)}`;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* The sessions of a store whose first main-line assistant text is `text`, captured rows first as in
|
|
161
|
+
* firstReplySQL. The prefilter reads the text column only where text_len (code points, as every
|
|
162
|
+
* writer counts it) already matches.
|
|
163
|
+
*/
|
|
164
|
+
function sharingSQL(P, store, text) {
|
|
165
|
+
const captured = `(adapter != 'legacy' OR (session_id, incarnation) NOT IN (SELECT DISTINCT session_id, incarnation FROM ${P.session_parts} FINAL
|
|
166
|
+
WHERE ${inStore(store)} AND removed = 0 AND adapter != 'legacy'))`;
|
|
167
|
+
return `SELECT session_id, incarnation FROM (
|
|
168
|
+
SELECT session_id, incarnation, argMin(text, ${PART_ORDER}) AS t FROM ${P.session_parts} FINAL
|
|
169
|
+
WHERE ${inStore(store)} AND ${REPLY_PARTS} AND ${captured}
|
|
170
|
+
AND (session_id, incarnation) IN (SELECT DISTINCT session_id, incarnation FROM ${P.session_parts}
|
|
171
|
+
WHERE ${inStore(store)} AND role = 'assistant' AND kind = 'text' AND text_len = ${[...text].length} AND text = ${lit(text)})
|
|
172
|
+
GROUP BY session_id, incarnation)
|
|
173
|
+
WHERE t = ${lit(text)} ORDER BY session_id, incarnation LIMIT ${SHARERS}`;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Each session's birth (epoch ms, as a string; null for none): its record's created_at, else its
|
|
178
|
+
* first part's own time. A session with neither has no birth, and no verdict either way.
|
|
179
|
+
*/
|
|
180
|
+
function bornSQL(P, store, ids) {
|
|
181
|
+
const list = ids.map(lit).join(', ');
|
|
182
|
+
return `SELECT p.session_id AS session_id, p.incarnation AS incarnation, toString(toUnixTimestamp64Milli(coalesce(s.created_at, p.first))) AS born
|
|
183
|
+
FROM (SELECT session_id, incarnation, min(ts) AS first FROM ${P.session_parts} FINAL
|
|
184
|
+
WHERE ${inStore(store)} AND removed = 0 AND session_id IN (${list}) GROUP BY session_id, incarnation) AS p
|
|
185
|
+
LEFT JOIN (SELECT session_id, incarnation, created_at FROM ${P.sessions} FINAL WHERE ${inStore(store)} AND is_deleted = 0 AND session_id IN (${list})) AS s
|
|
186
|
+
ON s.session_id = p.session_id AND s.incarnation = p.incarnation`;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** A session's live usage rows, with the numbers billing compares. */
|
|
190
|
+
function usageSQL(P, ref) {
|
|
191
|
+
return `SELECT usage_id, toString(anchor_kind) AS ak, anchor_id AS an,
|
|
192
|
+
toString(ifNull(input, 0)) AS i, toString(ifNull(output, 0)) AS o, toString(ifNull(cache_read, 0)) AS cr,
|
|
193
|
+
toString(ifNull(cache_write, 0)) AS cw, toString(ifNull(reasoning, 0)) AS rs, toString(ifNull(tool_prompt, 0)) AS tp
|
|
194
|
+
FROM ${P.session_usage} FINAL WHERE ${storeWhere(ref)} AND removed = 0 AND ${capturedFirst(P.session_usage, ref)}`;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* The verified copied prefix of `child` from `origin` (messagesSQL lists): { pairs, unverified }.
|
|
199
|
+
* `through`: the last copied message id; the match stops after it. `ids`: pairs must match on id.
|
|
200
|
+
*/
|
|
201
|
+
function verifyPrefix(child, origin, { through = '', ids = false } = {}) {
|
|
202
|
+
const pairs = [];
|
|
203
|
+
const n = Math.min(child.length, origin.length);
|
|
204
|
+
for (let i = 0; i < n; i++) {
|
|
205
|
+
const c = child[i]; const o = origin[i];
|
|
206
|
+
if (c.role !== o.role || c.digest !== o.digest || (ids && c.id !== o.id)) break;
|
|
207
|
+
if (truthy(c.trunc) || truthy(o.trunc)) return { pairs: [], unverified: `message ${i + 1} of the copy is truncated: it cannot be compared in full` };
|
|
208
|
+
pairs.push([c, o]);
|
|
209
|
+
if (through && (c.id === through || o.id === through)) break;
|
|
210
|
+
}
|
|
211
|
+
return pairs.length ? { pairs, unverified: '' } : { pairs, unverified: 'the first message differs from the origin\'s' };
|
|
212
|
+
}
|
|
213
|
+
const truthy = (v) => v === true || v === 1 || v === '1' || v === 'true';
|
|
214
|
+
|
|
215
|
+
/** The child's usage_ids that copy the origin's at the paired messages (each origin row matched once). */
|
|
216
|
+
function copiedUsage(pairs, childUsage, originUsage) {
|
|
217
|
+
const at = new Map(); // the child's anchor -> the origin's
|
|
218
|
+
for (const [c, o] of pairs) {
|
|
219
|
+
at.set(`message\u0000${c.id}`, `message\u0000${o.id}`);
|
|
220
|
+
if (c.rid && o.rid) at.set(`response\u0000${c.rid}`, `response\u0000${o.rid}`);
|
|
221
|
+
}
|
|
222
|
+
const nums = (u) => TOK.map((k) => String(u[k] ?? '0')).join(',');
|
|
223
|
+
const bag = new Map();
|
|
224
|
+
for (const u of originUsage) {
|
|
225
|
+
const k = `${u.ak}\u0000${u.an}\u0000${nums(u)}`;
|
|
226
|
+
bag.set(k, (bag.get(k) || 0) + 1);
|
|
227
|
+
}
|
|
228
|
+
const out = [];
|
|
229
|
+
for (const u of [...childUsage].sort((a, b) => String(a.usage_id).localeCompare(String(b.usage_id)))) {
|
|
230
|
+
const to = at.get(`${u.ak}\u0000${u.an}`);
|
|
231
|
+
if (!to || TOK.every((k) => String(u[k] ?? '0') === '0')) continue;
|
|
232
|
+
const k = `${to}\u0000${nums(u)}`;
|
|
233
|
+
if (!bag.get(k)) continue;
|
|
234
|
+
bag.set(k, bag.get(k) - 1);
|
|
235
|
+
out.push(String(u.usage_id));
|
|
236
|
+
}
|
|
237
|
+
return out;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Statements resolved into edges (header: merge). Returns Map edgeId -> { child, origin, kind, at,
|
|
242
|
+
* why, statements: [...] }; edgeId names the pair and the edge's `at`.
|
|
243
|
+
*/
|
|
244
|
+
function groupStatements(stmts, store, lives) {
|
|
245
|
+
const life = (id, inc) => {
|
|
246
|
+
if (inc != null && inc !== '') return { inc: String(inc) };
|
|
247
|
+
const l = lives.get(String(id)) || [];
|
|
248
|
+
return l.length === 1 ? { inc: l[0] } : { inc: '', why: l.length ? `two lives of session ${id} in the store` : `session ${id} is not in the house yet` };
|
|
249
|
+
};
|
|
250
|
+
const ord = (s) => [s.stated_on === 'origin' ? 1 : 0, String(s.at || ''), String(s.session_id), String(s.incarnation || ''), String(s.through_message_id || '')].join('\u0000');
|
|
251
|
+
const pairs = new Map();
|
|
252
|
+
for (const s of [...stmts].sort((a, b) => ord(a).localeCompare(ord(b)))) {
|
|
253
|
+
const stating = { session_id: String(s.session_id), incarnation: String(s.incarnation || '') };
|
|
254
|
+
let child; let origin;
|
|
255
|
+
if (s.stated_on === 'origin') {
|
|
256
|
+
origin = { ...stating, why: '' };
|
|
257
|
+
const id = String(s.child_session_id || s.other_session_id || '');
|
|
258
|
+
const l = life(id, s.child_incarnation != null ? s.child_incarnation : s.other_incarnation);
|
|
259
|
+
child = { session_id: id, incarnation: l.inc, why: l.why || '' };
|
|
260
|
+
} else {
|
|
261
|
+
child = { ...stating, why: '' };
|
|
262
|
+
const id = String(s.origin_session_id || s.other_session_id || '');
|
|
263
|
+
const l = life(id, s.origin_incarnation != null ? s.origin_incarnation : s.other_incarnation);
|
|
264
|
+
origin = { session_id: id, incarnation: l.inc, why: l.why || '' };
|
|
265
|
+
}
|
|
266
|
+
// A child that cannot be named has no key for its edge: nothing to write.
|
|
267
|
+
if (!child.session_id || !origin.session_id || child.why) continue;
|
|
268
|
+
const c = { ...store, session_id: child.session_id, incarnation: child.incarnation };
|
|
269
|
+
const o = { ...store, session_id: origin.session_id, incarnation: origin.incarnation };
|
|
270
|
+
const k = `${skey(c)}\u0001${s.kind}\u0001${skey(o)}`;
|
|
271
|
+
if (!pairs.has(k)) pairs.set(k, { child: c, origin: o, kind: s.kind, why: origin.why, side: { child: [], origin: [] } });
|
|
272
|
+
pairs.get(k).side[s.stated_on === 'origin' ? 'origin' : 'child'].push({ ...s, at: String(s.at || ''), stating });
|
|
273
|
+
}
|
|
274
|
+
const out = new Map();
|
|
275
|
+
const fits = (a, b) => a.at === b.at || a.at === '' || b.at === '';
|
|
276
|
+
for (const [k, p] of pairs) {
|
|
277
|
+
// A statement merges with the one statement on the other side it fits, when that one fits no other.
|
|
278
|
+
const at = new Map();
|
|
279
|
+
for (const c of p.side.child) {
|
|
280
|
+
const os = p.side.origin.filter((o) => fits(c, o));
|
|
281
|
+
if (os.length === 1 && p.side.child.filter((x) => fits(x, os[0])).length === 1) {
|
|
282
|
+
const merged = c.at || os[0].at;
|
|
283
|
+
at.set(c, merged); at.set(os[0], merged);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
for (const s of [...p.side.child, ...p.side.origin]) {
|
|
287
|
+
const a = at.has(s) ? at.get(s) : s.at;
|
|
288
|
+
const id = `${k}\u0001${a}`;
|
|
289
|
+
if (!out.has(id)) out.set(id, { child: p.child, origin: p.origin, kind: p.kind, at: a, why: p.why, statements: [] });
|
|
290
|
+
out.get(id).statements.push(s);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return out;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** The edge's own fields (the ones compared to tell a changed edge from an unchanged one). */
|
|
297
|
+
function edgeFields(g, v, copied, { lineage = false } = {}) {
|
|
298
|
+
const st = g.statements;
|
|
299
|
+
const sides = new Set(st.map((s) => (s.stated_on === 'origin' ? 'origin' : 'child')));
|
|
300
|
+
const flag = (f) => (st.some((s) => s[f] === true) ? true : st.some((s) => s[f] === false) ? false : null);
|
|
301
|
+
return {
|
|
302
|
+
kind: g.kind, at: g.at,
|
|
303
|
+
stated_on: sides.size === 2 ? 'both' : [...sides][0],
|
|
304
|
+
through_message_id: v.pairs.length ? String(v.pairs[v.pairs.length - 1][1].id) : throughOf(g),
|
|
305
|
+
ids_preserved: flag('ids_preserved'), marked: flag('marked'), usage_copied: flag('usage_copied'),
|
|
306
|
+
// A statement that says the child is not stored wins; one that says it is, next; none says: unknown.
|
|
307
|
+
child_stored: st.some((s) => nb(s.child_stored) === false) ? false : st.some((s) => nb(s.child_stored) === true) ? true : null,
|
|
308
|
+
verified: lineage ? null : !v.unverified, evidence: 'explicit',
|
|
309
|
+
statements: [...new Map(st.map((s) => [skey({ ...g.child, ...s.stating }), { host: g.child.host, harness: g.child.harness, harness_path: g.child.harness_path, session_id: s.stating.session_id, incarnation: s.stating.incarnation }])).values()]
|
|
310
|
+
.sort((a, b) => skey(a).localeCompare(skey(b))),
|
|
311
|
+
r_copied_usage: [...copied].sort(), r_copied_messages: v.pairs.length, r_unverified: v.unverified,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
/** An inferred edge's own fields (header: inferred). */
|
|
315
|
+
function inferredFields(v, copied) {
|
|
316
|
+
return {
|
|
317
|
+
kind: 'fork', at: '', stated_on: '', through_message_id: String(v.pairs[v.pairs.length - 1][1].id),
|
|
318
|
+
ids_preserved: null, marked: null, usage_copied: null, child_stored: true, verified: true, evidence: 'inferred', statements: [],
|
|
319
|
+
r_copied_usage: [...copied].sort(), r_copied_messages: v.pairs.length, r_unverified: '',
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
/** The statements' last copied message: the child's statement first (header: verified). */
|
|
323
|
+
const throughOf = (g) => String((g.statements.find((s) => s.stated_on !== 'origin' && s.through_message_id) || g.statements.find((s) => s.through_message_id) || {}).through_message_id || '');
|
|
324
|
+
const nb = (v) => (v === true || v === 'true' || v === 1 ? true : v === false || v === 'false' || v === 0 ? false : null);
|
|
325
|
+
const fieldsOfRow = (e) => ({
|
|
326
|
+
kind: e.kind, at: String(e.at || ''), stated_on: e.stated_on, through_message_id: String(e.through_message_id || ''),
|
|
327
|
+
ids_preserved: nb(e.ids_preserved), marked: nb(e.marked), usage_copied: nb(e.usage_copied), child_stored: nb(e.child_stored),
|
|
328
|
+
verified: nb(e.verified),
|
|
329
|
+
evidence: String(e.evidence || 'explicit'),
|
|
330
|
+
statements: (e.statements || []).map((t) => (Array.isArray(t) ? { host: t[0], harness: t[1], harness_path: t[2], session_id: t[3], incarnation: t[4] } : t))
|
|
331
|
+
.map((t) => ({ host: String(t.host), harness: String(t.harness), harness_path: String(t.harness_path), session_id: String(t.session_id), incarnation: String(t.incarnation) }))
|
|
332
|
+
.sort((a, b) => skey(a).localeCompare(skey(b))),
|
|
333
|
+
r_copied_usage: [...(e.r_copied_usage || [])].map(String).sort(), r_copied_messages: Number(e.r_copied_messages || 0), r_unverified: String(e.r_unverified || ''),
|
|
334
|
+
});
|
|
335
|
+
const digestOf = (f) => JSON.stringify(f);
|
|
336
|
+
|
|
337
|
+
/** The engine edge row for a group, at `rev`. */
|
|
338
|
+
function edgeRow(g, f, rev) {
|
|
339
|
+
const spec = {
|
|
340
|
+
owner: 'engine', kind: f.kind, child: g.child, origin: g.origin, at: f.at, stated_on: f.stated_on, through_message_id: f.through_message_id,
|
|
341
|
+
ids_preserved: f.ids_preserved, marked: f.marked, usage_copied: f.usage_copied, evidence: f.evidence, verified: f.verified, statements: f.statements,
|
|
342
|
+
};
|
|
343
|
+
return {
|
|
344
|
+
host: g.child.host, harness: g.child.harness, harness_path: g.child.harness_path, session_id: g.child.session_id, incarnation: g.child.incarnation,
|
|
345
|
+
owner: 'engine', kind: f.kind, other_host: g.origin.host, other_harness_path: g.origin.harness_path, other_harness: g.origin.harness,
|
|
346
|
+
other_session_id: g.origin.session_id, other_incarnation: g.origin.incarnation, at: f.at, stated_on: f.stated_on,
|
|
347
|
+
child_host: g.child.host, child_harness_path: g.child.harness_path, child_harness: g.child.harness, child_session_id: g.child.session_id, child_incarnation: g.child.incarnation,
|
|
348
|
+
origin_host: g.origin.host, origin_harness_path: g.origin.harness_path, origin_harness: g.origin.harness, origin_session_id: g.origin.session_id, origin_incarnation: g.origin.incarnation,
|
|
349
|
+
through_message_id: f.through_message_id, ids_preserved: f.ids_preserved, marked: f.marked, usage_copied: f.usage_copied, child_stored: f.child_stored,
|
|
350
|
+
evidence: f.evidence, verified: f.verified, statements: f.statements, content_sha: sha(JSON.stringify(spec)),
|
|
351
|
+
adapter: EDGE_ADAPTER, adapter_ver: String(EDGE_VER), model_version: '', rev: String(rev), is_deleted: 0, removed: 0,
|
|
352
|
+
r_copied_usage: f.r_copied_usage, r_copied_messages: f.r_copied_messages, r_unverified: f.r_unverified,
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
const edgeKey = (e) => ['engine', e.kind, e.other_host, e.other_harness_path, e.other_harness, e.other_session_id, e.other_incarnation || '', e.at || ''].map((x) => String(x ?? '')).join('\u0000');
|
|
356
|
+
const childOf = (store, e) => ({ ...store, session_id: e.session_id, incarnation: e.incarnation });
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Recompute the edges `ref` is the child or the origin of (harness-sessions-sql §5 step 4), after its rows
|
|
360
|
+
* are written. `house`: { P, rows(sql), insert(table, rows), retire(table, ref, keys, rev) }.
|
|
361
|
+
* `budget`: { left } nominated forks this pass may still verify (header: budget). `expand`: also
|
|
362
|
+
* nominate the forks on the origin's side (false: only `ref`). `born`: `ref`'s
|
|
363
|
+
* own birth (epoch ms) when its sessions row is not written yet (a first capture computes its edges
|
|
364
|
+
* before that row).
|
|
365
|
+
* Returns { changed: the children whose edge was written or ended (their totals moved), children:
|
|
366
|
+
* every child of an edge `ref` takes part in, live or ended, deferred: the nominated forks left for
|
|
367
|
+
* the next pass, written, ended }.
|
|
368
|
+
*/
|
|
369
|
+
async function recomputeEdges(house, ref, { rev, budget = null, born = null, expand = true, linked = true }) {
|
|
370
|
+
const none = { changed: [], children: [], deferred: [], expansionOwed: false, written: 0, ended: 0 };
|
|
371
|
+
if (!house || !house.P || typeof house.rows !== 'function' || (NATIVE.has(ref.harness) && !linked)) return none;
|
|
372
|
+
const P = house.P;
|
|
373
|
+
const store = { host: ref.host, harness: ref.harness, harness_path: ref.harness_path };
|
|
374
|
+
const inferring = INFERRED.has(ref.harness);
|
|
375
|
+
const stmts = await house.rows(statementsSQL(P, ref));
|
|
376
|
+
const edges = await house.rows(edgesSQL(P, ref));
|
|
377
|
+
if (!stmts.length && !edges.length && !inferring) return none;
|
|
378
|
+
const unknown = new Set();
|
|
379
|
+
for (const s of stmts) {
|
|
380
|
+
const other = s.stated_on === 'origin' ? [s.child_session_id || s.other_session_id, s.child_incarnation != null ? s.child_incarnation : s.other_incarnation]
|
|
381
|
+
: [s.origin_session_id || s.other_session_id, s.origin_incarnation != null ? s.origin_incarnation : s.other_incarnation];
|
|
382
|
+
if (other[0] && (other[1] == null || other[1] === '')) unknown.add(String(other[0]));
|
|
383
|
+
}
|
|
384
|
+
const lives = new Map();
|
|
385
|
+
if (unknown.size) for (const r of await house.rows(livesSQL(P, store, [...unknown]))) lives.set(r.session_id, [...(lives.get(r.session_id) || []), r.incarnation]);
|
|
386
|
+
const groups = groupStatements(stmts, store, lives);
|
|
387
|
+
// The edges written before, by their key on the child; an ended one only names its fork, and
|
|
388
|
+
// gives its key's floor (an edge made again is written above its removed version). The inferred
|
|
389
|
+
// ones are the nomination's, below.
|
|
390
|
+
const children = new Map();
|
|
391
|
+
const held = new Map();
|
|
392
|
+
const last = new Map();
|
|
393
|
+
const idOf = (c, e) => `${skey(c)}\u0001${edgeKey(e)}`;
|
|
394
|
+
for (const e of edges) {
|
|
395
|
+
const c = childOf(store, e);
|
|
396
|
+
children.set(skey(c), c);
|
|
397
|
+
last.set(idOf(c, e), e);
|
|
398
|
+
if (!truthy(e.removed) && e.evidence !== 'inferred') held.set(idOf(c, e), e);
|
|
399
|
+
}
|
|
400
|
+
const cache = new Map();
|
|
401
|
+
const load = async (sql, r) => {
|
|
402
|
+
const k = `${sql === messagesSQL ? 'm' : sql === usageSQL ? 'u' : 'r'}${skey(r)}`;
|
|
403
|
+
if (!cache.has(k)) cache.set(k, (await house.rows(sql(P, r))).map((x) => (sql === messagesSQL ? { ...x, role: x.mrole } : x)));
|
|
404
|
+
return cache.get(k);
|
|
405
|
+
};
|
|
406
|
+
const changed = new Map();
|
|
407
|
+
let written = 0; let ended = 0;
|
|
408
|
+
const above = (e) => (e && BigInt(e.rev) >= BigInt(rev) ? BigInt(e.rev) + 1n : BigInt(rev));
|
|
409
|
+
/** Write the edge unless its live version says the same; true when written. */
|
|
410
|
+
const put = async (child, row, f, prev) => {
|
|
411
|
+
const id = idOf(child, row);
|
|
412
|
+
children.set(skey(child), child);
|
|
413
|
+
if (prev && digestOf(fieldsOfRow(prev)) === digestOf(f)) return false;
|
|
414
|
+
// One version per key per write: a changed edge is written above its last version, live or
|
|
415
|
+
// removed, never retired first.
|
|
416
|
+
await house.insert('session_links', [{ ...row, rev: String(above(last.get(id))) }]);
|
|
417
|
+
written++;
|
|
418
|
+
changed.set(skey(child), child);
|
|
419
|
+
return true;
|
|
420
|
+
};
|
|
421
|
+
// The keys the explicit pass wrote or kept: an inferred row on one of them was replaced by it.
|
|
422
|
+
const explicitIds = new Set();
|
|
423
|
+
const end = async (child, e) => {
|
|
424
|
+
await house.retire('session_links', child, [edgeKey(e)], above(e));
|
|
425
|
+
ended++;
|
|
426
|
+
changed.set(skey(child), child);
|
|
427
|
+
};
|
|
428
|
+
for (const g of groups.values()) {
|
|
429
|
+
let v; let copied = [];
|
|
430
|
+
const lineage = !COPY_KINDS.includes(g.kind) || NATIVE.has(store.harness);
|
|
431
|
+
if (g.why || lineage) v = { pairs: [], unverified: g.why || '' };
|
|
432
|
+
else {
|
|
433
|
+
const [cm, om] = [await load(messagesSQL, g.child), await load(messagesSQL, g.origin)];
|
|
434
|
+
const absent = !om.length ? g.origin : !cm.length ? g.child : null;
|
|
435
|
+
const ids = g.statements.some((s) => s.ids_preserved === true);
|
|
436
|
+
v = absent ? { pairs: [], unverified: `session ${absent.session_id} is not in the house yet` } : verifyPrefix(cm, om, { through: throughOf(g) || g.at, ids });
|
|
437
|
+
if (!v.unverified) copied = copiedUsage(v.pairs, await load(usageSQL, g.child), await load(usageSQL, g.origin));
|
|
438
|
+
}
|
|
439
|
+
const f = edgeFields(g, v, copied, { lineage });
|
|
440
|
+
const row = edgeRow(g, f, 0n);
|
|
441
|
+
const id = idOf(g.child, row);
|
|
442
|
+
const prev = held.get(id);
|
|
443
|
+
held.delete(id);
|
|
444
|
+
explicitIds.add(id);
|
|
445
|
+
await put(g.child, row, f, prev);
|
|
446
|
+
}
|
|
447
|
+
// An edge no statement makes any more (its statements gone, or its `at` changed) ends.
|
|
448
|
+
for (const e of held.values()) await end(childOf(store, e), e);
|
|
449
|
+
|
|
450
|
+
// Inferred edges (header: inferred).
|
|
451
|
+
const deferred = [];
|
|
452
|
+
let expansionOwed = false;
|
|
453
|
+
if (inferring) {
|
|
454
|
+
const reply = async (r) => String(((await load(firstReplySQL, r))[0] || {}).t || '');
|
|
455
|
+
const births = new Map(Number.isFinite(born) ? [[skey(ref), born]] : []);
|
|
456
|
+
const bornOf = async (refs) => {
|
|
457
|
+
const need = refs.filter((r) => !births.has(skey(r)));
|
|
458
|
+
if (need.length) {
|
|
459
|
+
for (const r of need) births.set(skey(r), NaN);
|
|
460
|
+
for (const b of await house.rows(bornSQL(P, store, [...new Set(need.map((r) => r.session_id))]))) {
|
|
461
|
+
const k = skey({ ...store, session_id: b.session_id, incarnation: b.incarnation });
|
|
462
|
+
if (births.has(k) && b.born != null && b.born !== '' && b.born !== '\\N') births.set(k, Number(b.born));
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
return births;
|
|
466
|
+
};
|
|
467
|
+
const shared = new Map();
|
|
468
|
+
const sharing = async (text) => {
|
|
469
|
+
if (!shared.has(text)) shared.set(text, (await house.rows(sharingSQL(P, store, text))).map((r) => ({ ...store, session_id: r.session_id, incarnation: r.incarnation })));
|
|
470
|
+
return shared.get(text);
|
|
471
|
+
};
|
|
472
|
+
// The origin `w` copies: the longest verified prefix among the earlier-born sessions sharing its
|
|
473
|
+
// first reply, the earliest born at an equal length; null for none or a tie, 'deferred' past the budget.
|
|
474
|
+
const nominate = async (w) => {
|
|
475
|
+
const text = await reply(w);
|
|
476
|
+
if (text.length < MIN_NOMINATION_TEXT) return null;
|
|
477
|
+
const cands = (await sharing(text)).filter((c) => skey(c) !== skey(w));
|
|
478
|
+
if (!cands.length) return null;
|
|
479
|
+
const b = await bornOf([w, ...cands]);
|
|
480
|
+
const wb = b.get(skey(w));
|
|
481
|
+
if (!Number.isFinite(wb)) return null;
|
|
482
|
+
const before = cands.filter((c) => b.get(skey(c)) < wb)
|
|
483
|
+
.sort((x, y) => b.get(skey(x)) - b.get(skey(y)) || skey(x).localeCompare(skey(y)));
|
|
484
|
+
// The earliest born (an origin of many forks) and the latest born (a fork of a fork's parent).
|
|
485
|
+
const earlier = before.length <= NOMINEES ? before : [...before.slice(0, NOMINEES / 2), ...before.slice(-NOMINEES / 2)];
|
|
486
|
+
if (!earlier.length) return null;
|
|
487
|
+
if (budget && budget.left <= 0) return 'deferred';
|
|
488
|
+
if (budget) budget.left--;
|
|
489
|
+
const cm = await load(messagesSQL, w);
|
|
490
|
+
let best = null; let tie = false;
|
|
491
|
+
for (const c of earlier) {
|
|
492
|
+
const v = verifyPrefix(cm, await load(messagesSQL, c));
|
|
493
|
+
if (v.unverified) continue;
|
|
494
|
+
if (!best || v.pairs.length > best.v.pairs.length) { best = { origin: c, v, born: b.get(skey(c)) }; tie = false; } else if (v.pairs.length === best.v.pairs.length && b.get(skey(c)) === best.born) tie = true;
|
|
495
|
+
}
|
|
496
|
+
if (!best || tie) return null;
|
|
497
|
+
return { origin: best.origin, v: best.v, copied: copiedUsage(best.v.pairs, await load(usageSQL, w), await load(usageSQL, best.origin)) };
|
|
498
|
+
};
|
|
499
|
+
// The forks to nominate again: `ref` itself, the forks its inferred edges name, and the sessions
|
|
500
|
+
// sharing its first reply born after it (only those can be its forks; 3.2's bound). With the
|
|
501
|
+
// budget already spent, only `ref`: owing itself, it expands on the next pass instead, so its
|
|
502
|
+
// forks are never left behind and nothing is read for them now.
|
|
503
|
+
const spent = !!budget && budget.left <= 0;
|
|
504
|
+
if (spent && expand) expansionOwed = true;
|
|
505
|
+
const want = new Map([[skey(ref), ref]]);
|
|
506
|
+
for (const e of spent || !expand ? [] : edges) {
|
|
507
|
+
if (!truthy(e.removed) && e.evidence === 'inferred' && e.other_session_id === ref.session_id && String(e.other_incarnation || '') === String(ref.incarnation || '')) want.set(skey(childOf(store, e)), childOf(store, e));
|
|
508
|
+
}
|
|
509
|
+
const mine = spent || !expand ? '' : await reply(ref);
|
|
510
|
+
if (mine.length >= MIN_NOMINATION_TEXT) {
|
|
511
|
+
const shared = (await sharing(mine)).filter((c) => !want.has(skey(c)));
|
|
512
|
+
const b = await bornOf([ref, ...shared]);
|
|
513
|
+
for (const c of shared) if (b.get(skey(c)) > b.get(skey(ref))) want.set(skey(c), c);
|
|
514
|
+
}
|
|
515
|
+
const owedNow = budget ? (budget.owed || (budget.owed = new Set())) : null;
|
|
516
|
+
for (const w of want.values()) {
|
|
517
|
+
const self = skey(w) === skey(ref);
|
|
518
|
+
// Owed once a pass from other sessions' sides; always by itself, whose own state is saved after.
|
|
519
|
+
const owe = () => { if (self || !owedNow.has(skey(w))) { owedNow.add(skey(w)); deferred.push(w); } };
|
|
520
|
+
// Past the budget a fork is owed before anything is read for it: another session's, or `ref`
|
|
521
|
+
// itself when the pass started spent (it reads nothing now and expands next pass). With budget
|
|
522
|
+
// left, `ref` is owed only if it has a candidate to verify (header: budget).
|
|
523
|
+
if (spent && self) { owe(); continue; }
|
|
524
|
+
if (!self && budget && budget.left <= 0) { owe(); continue; }
|
|
525
|
+
// A fork that states its origin explicitly takes no inferred edge.
|
|
526
|
+
const wStmts = self ? stmts : await house.rows(statementsSQL(P, w));
|
|
527
|
+
const explicit = wStmts.some((x) => COPY_KINDS.includes(x.kind)
|
|
528
|
+
&& (x.stated_on === 'origin' ? String(x.child_session_id || x.other_session_id) === w.session_id : String(x.session_id) === w.session_id));
|
|
529
|
+
const wEdges = (self ? edges : await house.rows(edgesSQL(P, w)))
|
|
530
|
+
.filter((e) => e.evidence === 'inferred' && e.session_id === w.session_id && String(e.incarnation || '') === String(w.incarnation || ''));
|
|
531
|
+
for (const e of wEdges) last.set(idOf(w, e), e);
|
|
532
|
+
const live = wEdges.filter((e) => !truthy(e.removed));
|
|
533
|
+
const found = explicit ? null : await nominate(w);
|
|
534
|
+
if (found === 'deferred') { owe(); continue; }
|
|
535
|
+
let keep = null;
|
|
536
|
+
if (found) {
|
|
537
|
+
const f = inferredFields(found.v, found.copied);
|
|
538
|
+
const row = edgeRow({ child: w, origin: found.origin }, f, 0n);
|
|
539
|
+
keep = edgeKey(row);
|
|
540
|
+
await put(w, row, f, live.find((e) => edgeKey(e) === keep));
|
|
541
|
+
}
|
|
542
|
+
// An inferred edge the nomination no longer makes ends, unless the explicit pass wrote its key.
|
|
543
|
+
for (const e of live) if (edgeKey(e) !== keep && !explicitIds.has(idOf(w, e))) await end(w, e);
|
|
544
|
+
if (wEdges.length) children.set(skey(w), w);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
return { changed: [...changed.values()], children: [...children.values()], deferred, expansionOwed, written, ended };
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
module.exports = {
|
|
551
|
+
recomputeEdges, verifyPrefix, copiedUsage, groupStatements, edgeFields, inferredFields, fieldsOfRow, edgeRow, edgeKey,
|
|
552
|
+
statementsSQL, edgesSQL, messagesSQL, usageSQL, livesSQL, firstReplySQL, sharingSQL, bornSQL,
|
|
553
|
+
COPY_KINDS, NATIVE, INFERRED, MIN_NOMINATION_TEXT, NOMINEES, FORK_BUDGET, EDGE_ADAPTER, EDGE_VER,
|
|
554
|
+
};
|