memhouse 0.18.9 → 0.18.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +194 -0
- package/CHANGELOG.md +1097 -0
- package/README.md +54 -423
- package/SECURITY.md +68 -0
- package/bin/memhouse.js +299 -133
- package/docs/README.md +46 -0
- package/docs/guides/admin-password.md +95 -0
- package/docs/guides/configuration.md +109 -0
- package/docs/guides/daemons.md +53 -0
- package/docs/guides/install.md +128 -0
- package/docs/guides/invite.md +87 -0
- package/docs/guides/operate.md +112 -0
- package/docs/guides/search-and-resume.md +95 -0
- package/docs/guides/share.md +102 -0
- package/docs/guides/storage.md +93 -0
- package/docs/guides/troubleshooting.md +78 -0
- package/docs/guides/uninstall.md +37 -0
- package/docs/guides/update.md +68 -0
- package/docs/releases/README.md +11 -0
- package/docs/releases/v0.18.10.md +226 -0
- package/docs/releases/v0.18.11.md +191 -0
- package/docs/tutorials/01-first-house.md +132 -0
- package/docs/tutorials/02-join-a-house.md +91 -0
- package/docs/tutorials/03-share-your-memory.md +96 -0
- package/editors/claude.js +68 -10
- package/examples/01-local-one-machine/README.md +99 -0
- package/examples/01-local-one-machine/run.sh +61 -0
- package/examples/02-existing-clickhouse/README.md +121 -0
- package/examples/02-existing-clickhouse/run.sh +71 -0
- package/examples/03-team-house/README.md +129 -0
- package/examples/03-team-house/run.sh +124 -0
- package/examples/04-self-hosted-team/.env.example +22 -0
- package/examples/04-self-hosted-team/README.md +187 -0
- package/examples/04-self-hosted-team/compose.yml +158 -0
- package/examples/04-self-hosted-team/smoke.sh +113 -0
- package/examples/README.md +42 -0
- package/examples/fixtures/README.md +24 -0
- package/examples/fixtures/make-sessions.js +63 -0
- package/memhouse/DESIGN.md +9 -2
- package/memhouse/admin-secret.js +81 -0
- package/memhouse/delivery/plugin/.claude-plugin/plugin.json +1 -1
- package/memhouse/delivery/plugin/reference/HOUSE.md +18 -6
- package/memhouse/delivery/plugin/skills/access/SKILL.md +1 -1
- package/memhouse/delivery/plugin/skills/admin/SKILL.md +1 -1
- package/memhouse/delivery/plugin/skills/house/SKILL.md +1 -1
- package/memhouse/delivery/plugin/skills/recall/SKILL.md +34 -14
- package/memhouse/delivery/plugin/skills/sql/SKILL.md +1 -1
- package/memhouse/deploy.js +4 -2
- package/memhouse/flags.js +32 -5
- package/memhouse/host.js +11 -6
- package/memhouse/house/HOUSE.md +1 -1
- package/memhouse/house/house.js +27 -1
- package/memhouse/invite-target.js +37 -0
- package/memhouse/provision.js +12 -3
- package/memhouse/server/queries.js +9 -4
- package/memhouse/service.js +5 -0
- package/memhouse/shipper/ship.js +148 -31
- package/package.json +10 -2
- package/public/THIRD_PARTY_NOTICES.txt +2114 -0
- package/ui/vite.config.js +4 -1
package/memhouse/host.js
CHANGED
|
@@ -14,13 +14,18 @@
|
|
|
14
14
|
// * One machine splits. Rename the box and every row shipped afterwards lands under a
|
|
15
15
|
// new host, so the machine's own history appears to end and a stranger's to begin.
|
|
16
16
|
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
17
|
+
// So the fingerprint comes from the OS's own machine id (IOPlatformUUID, /etc/machine-id,
|
|
18
|
+
// MachineGuid), hashed with a fixed salt (machineFingerprint), never from the hostname: two
|
|
19
|
+
// machines cannot share it and a rename cannot move it. Since 0.18.6. Before that it was
|
|
20
|
+
// random, which fixed both of the above but split a machine whenever MEMHOUSE_HOME was
|
|
21
|
+
// wiped: a re-imaged laptop came back as a stranger. Where no machine id can be read it is
|
|
22
|
+
// still random.
|
|
19
23
|
//
|
|
20
|
-
// It
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
+
// It is written once to MEMHOUSE_HOME/host.json and an existing file always wins, so no
|
|
25
|
+
// install changes identity under it. `uninstall --full-removal` drops the file, and the
|
|
26
|
+
// next install derives the same fingerprint again from the same machine. The id also
|
|
27
|
+
// carries the hostname the machine had at first install, so it comes back the same unless
|
|
28
|
+
// the machine was renamed in between, or has no readable machine id.
|
|
24
29
|
|
|
25
30
|
const fs = require('fs');
|
|
26
31
|
const os = require('os');
|
package/memhouse/house/HOUSE.md
CHANGED
|
@@ -17,7 +17,7 @@ Two columns say where every row came from:
|
|
|
17
17
|
| Column | Meaning | Comes from |
|
|
18
18
|
|---|---|---|
|
|
19
19
|
| `user_id` | who wrote it | `String MATERIALIZED currentUser()` — stamped by the server, unforgeable by clients. `async_insert = 0 CONST` is pinned on each user (`ADD SETTING`, which merges — a bare `SETTINGS` clause replaces the user's whole list) because an async flush stores the stamp as the empty string (measured on 25.11.9.34) |
|
|
20
|
-
| `host` | which machine | the
|
|
20
|
+
| `host` | which machine | the machine's fingerprint (`../host.js`) — derived from the OS machine id (random only where none can be read), so two machines with the same hostname cannot collide and renaming one cannot split its history |
|
|
21
21
|
|
|
22
22
|
`WHERE user_id = 'alice'` is one person. `WHERE host = 'macbook-4127a95b'` is one
|
|
23
23
|
machine. No filter is the whole house — which is what a team dashboard wants.
|
package/memhouse/house/house.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
// clients (async_insert is pinned to 0
|
|
9
9
|
// on the user so the stamp cannot be
|
|
10
10
|
// skipped)
|
|
11
|
-
// host LowCardinality(String) — the machine's fingerprint,
|
|
11
|
+
// host LowCardinality(String) — the machine's fingerprint, derived from its OS machine id
|
|
12
12
|
// per install (../host.js)
|
|
13
13
|
//
|
|
14
14
|
// That pair IS the provenance model.
|
|
@@ -412,6 +412,31 @@ function statRefreshStatements(rooms, at = 'now64(3)') {
|
|
|
412
412
|
`INSERT INTO ${names[k]} SELECT ${at} AS refreshed_at, * FROM (${bodies[k].body})`);
|
|
413
413
|
}
|
|
414
414
|
|
|
415
|
+
/**
|
|
416
|
+
* When the shipper refreshes the stat tables (G11). After every pass that shipped something,
|
|
417
|
+
* as before — and ALSO when the newest generation is older than STATS_MAX_AGE_MS, even if
|
|
418
|
+
* nothing shipped. The tables carry a one-day TTL, so a house whose machines went quiet for
|
|
419
|
+
* a day used to lose every generation and the dashboard went empty while the rooms still
|
|
420
|
+
* held everything. `ageMs` null means "could not tell" (no table, no rights): do not refresh
|
|
421
|
+
* on age then; the read layer is already on the inline rollup.
|
|
422
|
+
*/
|
|
423
|
+
const STATS_MAX_AGE_MS = 60 * 60 * 1000;
|
|
424
|
+
function statsNeedRefresh({ shipped, ageMs, maxAgeMs = STATS_MAX_AGE_MS }) {
|
|
425
|
+
if (shipped > 0) return true;
|
|
426
|
+
return ageMs !== null && ageMs !== undefined && ageMs > maxAgeMs;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* How long a reader may keep its resolved rooms (G11). resolveRooms() decides once whether
|
|
431
|
+
* the stat tables are usable or the inline rollup must answer; a long-running dashboard that
|
|
432
|
+
* never asked again kept reading a stat table after its TTL had emptied it. Re-resolving
|
|
433
|
+
* every few minutes costs one small round trip and lets it follow the house either way.
|
|
434
|
+
*/
|
|
435
|
+
const ROOMS_TTL_MS = 5 * 60 * 1000;
|
|
436
|
+
function roomsCacheStale(resolvedAt, now = Date.now(), ttlMs = ROOMS_TTL_MS) {
|
|
437
|
+
return !resolvedAt || now - resolvedAt > ttlMs;
|
|
438
|
+
}
|
|
439
|
+
|
|
415
440
|
/** The read-side source for one stat table: its newest generation, and nothing else. */
|
|
416
441
|
function latestGeneration(table) {
|
|
417
442
|
return `(SELECT * FROM ${table} WHERE refreshed_at = (SELECT max(refreshed_at) FROM ${table}))`;
|
|
@@ -598,4 +623,5 @@ module.exports = {
|
|
|
598
623
|
sessionsRollup, currentParse, createStatement, roomNames, currentUser, resolveRooms,
|
|
599
624
|
SESSION_STATS, MODEL_STATS, TOOL_STATS, statTableNames, hasSessionStats, statTables,
|
|
600
625
|
statCreateStatements, statRefreshStatements, latestGeneration,
|
|
626
|
+
STATS_MAX_AGE_MS, statsNeedRefresh, ROOMS_TTL_MS, roomsCacheStale,
|
|
601
627
|
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// Where `invite` may send a credential — and the one place that decides it (G4).
|
|
2
|
+
//
|
|
3
|
+
// `--url` is the address the INVITEE will reach the house at. It goes into the invite file;
|
|
4
|
+
// it is typed on a command line, so it can be a typo, a tunnel, or somebody else's server.
|
|
5
|
+
// A credential this install KEEPS — the member's own, or an admin saved with --keep-admin —
|
|
6
|
+
// belongs to the house it was saved for, and is only ever sent there. Before 0.18.11 the
|
|
7
|
+
// stored credential was sent to `--url`: resolveConfig() lets the flag outrank the env file,
|
|
8
|
+
// so `cfg.url` WAS `--url`, and the reachability probe, the grants read and the provisioning
|
|
9
|
+
// all authenticated against whatever `--url` named.
|
|
10
|
+
//
|
|
11
|
+
// A credential given for THIS command — `--admin-user`, or an exported MEMHOUSE_ADMIN_USER,
|
|
12
|
+
// with a password from a file, stdin, the environment or a prompt — was chosen by the person
|
|
13
|
+
// who also typed `--url`, so it provisions there, as it always has.
|
|
14
|
+
//
|
|
15
|
+
// Pure, so it is unit tested.
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {object} o
|
|
19
|
+
* @param {string} o.url the invitee-facing --url
|
|
20
|
+
* @param {string|null} o.storedUrl the house this install is configured for (env or env file)
|
|
21
|
+
* @param {'flag'|'env'|'file'|'member'} o.adminUserSource
|
|
22
|
+
* where the credential's USER came from: --admin-user, an exported MEMHOUSE_ADMIN_USER,
|
|
23
|
+
* the env file's MEMHOUSE_ADMIN_USER, or (no admin at all) the member credential
|
|
24
|
+
* @param {string|null} [o.passwordSource] resolveAdminPassword's `source` ('stored' = the env file)
|
|
25
|
+
* @returns {{ url: string, stored: boolean } | { error: string }}
|
|
26
|
+
*/
|
|
27
|
+
function provisionTarget({ url, storedUrl, adminUserSource, passwordSource = null }) {
|
|
28
|
+
const stored = adminUserSource === 'file' || adminUserSource === 'member' || passwordSource === 'stored';
|
|
29
|
+
if (!stored) return { url, stored: false };
|
|
30
|
+
if (!storedUrl) {
|
|
31
|
+
return { error: 'this install keeps no house URL to send its stored credential to — '
|
|
32
|
+
+ 'give the admin for this command instead (--admin-user, password from MEMHOUSE_ADMIN_PASSWORD or --admin-password-file)' };
|
|
33
|
+
}
|
|
34
|
+
return { url: storedUrl, stored: true };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
module.exports = { provisionTarget };
|
package/memhouse/provision.js
CHANGED
|
@@ -54,13 +54,22 @@ function sqlString(v) {
|
|
|
54
54
|
*
|
|
55
55
|
* @returns {Array<{sql:string, why:string, optional:boolean}>}
|
|
56
56
|
*/
|
|
57
|
-
|
|
57
|
+
// What a PRINTED plan carries where the password goes. Deliberately not a quoted string:
|
|
58
|
+
// run as printed, `IDENTIFIED BY <member-password>` does not parse, so nobody can create a
|
|
59
|
+
// member whose password is the placeholder text. A printed plan never carries a real
|
|
60
|
+
// password: printed output ends up in terminals, scrollback and agent transcripts, and
|
|
61
|
+
// memhouse ships transcripts into the house (G23).
|
|
62
|
+
const PASSWORD_PLACEHOLDER = '<member-password>';
|
|
63
|
+
|
|
64
|
+
function plan({ db, member, password = null, placeholder = false }) {
|
|
58
65
|
assertUsableName(db, 'house');
|
|
59
66
|
assertUsableName(member, 'member');
|
|
60
67
|
const steps = [
|
|
61
68
|
{ sql: `CREATE DATABASE IF NOT EXISTS ${db}`, why: 'the house', optional: false },
|
|
62
69
|
];
|
|
63
|
-
if (
|
|
70
|
+
if (placeholder) {
|
|
71
|
+
steps.push({ sql: `CREATE USER ${member} IDENTIFIED BY ${PASSWORD_PLACEHOLDER}`, why: `the member — replace ${PASSWORD_PLACEHOLDER} with a quoted password you generate, e.g. '…' from \`openssl rand -hex 16\`; as printed this line does not parse, on purpose`, optional: false });
|
|
72
|
+
} else if (password !== null) {
|
|
64
73
|
// No IF NOT EXISTS. The live path checks first and refuses an existing user; the printed
|
|
65
74
|
// plan used to no-op here, apply the grants, and hand out a password the server never saw —
|
|
66
75
|
// found on a realm that mints its own users. Now it fails loudly on line one instead.
|
|
@@ -100,4 +109,4 @@ function render(steps, { db, member }) {
|
|
|
100
109
|
return out.join('\n');
|
|
101
110
|
}
|
|
102
111
|
|
|
103
|
-
module.exports = { ROOM_PRIVILEGES, roomGrant, plan, render };
|
|
112
|
+
module.exports = { ROOM_PRIVILEGES, PASSWORD_PLACEHOLDER, roomGrant, plan, render };
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
|
|
19
19
|
const { createClient, ClickHouseLogLevel } = require('@clickhouse/client');
|
|
20
20
|
const { calculateCost, normalizeModelName } = require('../../pricing');
|
|
21
|
-
const { resolveRooms, READ_SETTINGS } = require('../house/house');
|
|
21
|
+
const { resolveRooms, READ_SETTINGS, roomsCacheStale } = require('../house/house');
|
|
22
22
|
|
|
23
23
|
const config = {
|
|
24
24
|
url: process.env.MEMHOUSE_URL || 'http://localhost:8123',
|
|
@@ -90,11 +90,16 @@ const TZ = Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
|
|
|
90
90
|
// braces, no type.
|
|
91
91
|
const ROOM_TOKEN = /\{\{([a-z_]+)\}\}/g;
|
|
92
92
|
let roomsPromise = null;
|
|
93
|
+
let roomsAt = 0;
|
|
93
94
|
|
|
94
95
|
function rooms() {
|
|
95
|
-
//
|
|
96
|
-
//
|
|
97
|
-
|
|
96
|
+
// Cached, because resolveRooms asks the server for currentUser() and which stat tables
|
|
97
|
+
// are usable, and every dashboard request would otherwise pay that round trip. But not
|
|
98
|
+
// for the life of the process: whether the stat tables can answer changes — their TTL
|
|
99
|
+
// empties them on a quiet house, a refresh fills them again — and a dashboard that
|
|
100
|
+
// decided once kept reading an emptied table and showed nothing (G11).
|
|
101
|
+
if (!roomsPromise || roomsCacheStale(roomsAt)) {
|
|
102
|
+
roomsAt = Date.now();
|
|
98
103
|
roomsPromise = resolveRooms(getClient()).catch((e) => { roomsPromise = null; throw e; });
|
|
99
104
|
}
|
|
100
105
|
return roomsPromise;
|
package/memhouse/service.js
CHANGED
|
@@ -190,6 +190,11 @@ function install({ shipJs, envFile, logDir, interval = 300, home = null, force =
|
|
|
190
190
|
// another member's rooms — which either fails on permissions or, if it can read them,
|
|
191
191
|
// quietly ships into the wrong place.
|
|
192
192
|
const env = parseEnvFile(envFile);
|
|
193
|
+
// The shipper never uses the admin credential, and the unit inlines every key it is
|
|
194
|
+
// given — a kept admin password would land in a second file and in the daemon's
|
|
195
|
+
// environment for its whole life.
|
|
196
|
+
delete env.MEMHOUSE_ADMIN_USER;
|
|
197
|
+
delete env.MEMHOUSE_ADMIN_PASSWORD;
|
|
193
198
|
// MEMHOUSE_HOME is not in the env file — it is where the env file itself lives — and a
|
|
194
199
|
// service installed under a custom home must keep using it, or it reads a different
|
|
195
200
|
// config after a reboot than the one just written.
|
package/memhouse/shipper/ship.js
CHANGED
|
@@ -66,7 +66,7 @@ const selfSnap = selfUpdate.snapshot(__filename);
|
|
|
66
66
|
const {
|
|
67
67
|
resolveRooms, READ_SETTINGS, ROOM_TYPES, META_TYPES, SCHEMA_VERSION, MEMBER_PIN, createStatement,
|
|
68
68
|
SUPPORTED_SCHEMAS, keyProblem, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor,
|
|
69
|
-
statCreateStatements, statRefreshStatements, statTableNames,
|
|
69
|
+
statCreateStatements, statRefreshStatements, statTableNames, statsNeedRefresh,
|
|
70
70
|
} = require('../house/house');
|
|
71
71
|
// Rooms plus the house's own record of itself. Every table the template declares, which
|
|
72
72
|
// is what the column healer and the drift warning have to cover — a column added to
|
|
@@ -109,7 +109,7 @@ function chTs(ms) {
|
|
|
109
109
|
|
|
110
110
|
// Whether the adapter gave this message its own timestamp, or the shipper has to
|
|
111
111
|
// interpolate one. The distinction matters beyond accuracy: an interpolated ts is a
|
|
112
|
-
// function of `
|
|
112
|
+
// function of `pos / (total - 1)` (position in the parse), so EVERY already-stored row's ts moves when the
|
|
113
113
|
// session grows. That is what forces a session with no per-message timestamps to
|
|
114
114
|
// re-ship whole rather than tail-only. See rowsForChat's `tsInterpolated` and tailSafe.
|
|
115
115
|
function hasOwnTs(msg) {
|
|
@@ -127,12 +127,12 @@ function hasOwnTs(msg) {
|
|
|
127
127
|
// and the search skill reported those to the user as fact.
|
|
128
128
|
//
|
|
129
129
|
// Fall back to interpolating across [createdAt, lastUpdatedAt] for the adapters that
|
|
130
|
-
// genuinely have nothing per message — monotonic
|
|
130
|
+
// genuinely have nothing per message — monotonic by position, so sessions_v started/ended
|
|
131
131
|
// still line up with the session bounds (same approximation as agency/ingest.js;
|
|
132
132
|
// documented in DESIGN.md). Real timestamps are NOT necessarily monotonic by seq — a
|
|
133
133
|
// folded subagent transcript is appended after its parent's turns but ran during them.
|
|
134
134
|
// Nothing depends on that: ordering is by `seq`, and started/ended are min/max.
|
|
135
|
-
function messageTs(chat,
|
|
135
|
+
function messageTs(chat, pos, total, msg) {
|
|
136
136
|
const at = Number(msg && msg._ts);
|
|
137
137
|
// Seconds or milliseconds, depending on the format: goose stores seconds, Claude Code
|
|
138
138
|
// and opencode milliseconds. Real epoch-ms is > 1e12 and real epoch-seconds ~1.7e9, so
|
|
@@ -141,7 +141,9 @@ function messageTs(chat, seq, total, msg) {
|
|
|
141
141
|
const start = chat.createdAt || chat.lastUpdatedAt || Date.now();
|
|
142
142
|
const end = chat.lastUpdatedAt || chat.createdAt || start;
|
|
143
143
|
if (total <= 1) return chTs(start);
|
|
144
|
-
|
|
144
|
+
// `pos` is the message's position in the parse, NOT its seq: a folded subagent's seq
|
|
145
|
+
// is sparse (see subagentSeq) and would put its interpolated time far past the session.
|
|
146
|
+
return chTs(start + Math.round((end - start) * (pos / (total - 1))));
|
|
145
147
|
}
|
|
146
148
|
|
|
147
149
|
/**
|
|
@@ -875,6 +877,42 @@ async function loadExisting(client, rooms) {
|
|
|
875
877
|
// m._toolCalls: matches '[tool-call: Name(' and '[tool-call: Name]' forms.
|
|
876
878
|
const TOOL_CALL_RE = /\[tool-call: ([^(\]]+)/g;
|
|
877
879
|
|
|
880
|
+
// Where a folded subagent's rows are numbered. A session's own turns keep the dense
|
|
881
|
+
// 0, 1, 2, … they always had; subagent number `slot` (see getMessages in
|
|
882
|
+
// editors/claude.js) owns the block
|
|
883
|
+
//
|
|
884
|
+
// seq = SUBAGENT_SEQ_BASE + slot * SUBAGENT_SEQ_STRIDE + turn
|
|
885
|
+
//
|
|
886
|
+
// and its tool calls the same block of `idx`, counted within that subagent. So a row's
|
|
887
|
+
// seq depends only on the subagent it belongs to and its position inside it — never on
|
|
888
|
+
// how long the parent is, how long any other subagent is, or how many there are.
|
|
889
|
+
//
|
|
890
|
+
// It used to be the row's position in the whole parse, with subagents folded in after the
|
|
891
|
+
// parent. One new parent turn then moved every subagent row down by one; decideEpoch saw
|
|
892
|
+
// stored rows whose line_hash (which covers seq) no longer matched, forked the session to
|
|
893
|
+
// a new epoch, and re-sent it whole — on every pass, for every session that had ever
|
|
894
|
+
// spawned a subagent. Measured on a real house: 1.07 M message rows in 24 h against 10.2 M
|
|
895
|
+
// in total, most of them full copies of five sessions, one stored 60 times in a day.
|
|
896
|
+
//
|
|
897
|
+
// Decimal so a seq reads at a glance (1,003,000,042 = subagent 3, turn 42), and sized for
|
|
898
|
+
// UInt32: 1e9 parent turns, 32,949 subagents of 100,000 turns each (the largest sessions
|
|
899
|
+
// today hold ~33 k parent lines and 35 subagents). A parse that does not fit falls back to
|
|
900
|
+
// positional numbering, which is correct and merely re-ships whole, as before.
|
|
901
|
+
const SUBAGENT_SEQ_BASE = 1000000000;
|
|
902
|
+
const SUBAGENT_SEQ_STRIDE = 100000;
|
|
903
|
+
const SUBAGENT_SLOTS = Math.floor((2 ** 32 - SUBAGENT_SEQ_BASE) / SUBAGENT_SEQ_STRIDE);
|
|
904
|
+
|
|
905
|
+
// The block a folded subagent turn belongs to, or null for the session's own turns and for
|
|
906
|
+
// adapters that fold nothing.
|
|
907
|
+
function subagentSlot(m) {
|
|
908
|
+
const a = m && m._agent;
|
|
909
|
+
return a && Number.isInteger(a.slot) && Number.isInteger(a.turn) ? a.slot : null;
|
|
910
|
+
}
|
|
911
|
+
function subagentSeq(slot, n) {
|
|
912
|
+
if (slot < 0 || slot >= SUBAGENT_SLOTS || n < 0 || n >= SUBAGENT_SEQ_STRIDE) return null;
|
|
913
|
+
return SUBAGENT_SEQ_BASE + slot * SUBAGENT_SEQ_STRIDE + n;
|
|
914
|
+
}
|
|
915
|
+
|
|
878
916
|
// Build the typed rows for one chat. Returns null when the chat is unreadable
|
|
879
917
|
// (adapter threw — e.g. a partially-written or locked session file): the caller
|
|
880
918
|
// must write NOTHING for it, so the next incremental pass retries. A chat that
|
|
@@ -891,6 +929,13 @@ function rowsForChat(chat, host) {
|
|
|
891
929
|
adapterErrorSink.record(chat.source, e, chat.composerId);
|
|
892
930
|
return null;
|
|
893
931
|
}
|
|
932
|
+
// The adapter asked to wait a pass: what it read is not yet the session's settled shape
|
|
933
|
+
// (a subagent with no timestamped line — see editors/claude.js). Not an error; the next
|
|
934
|
+
// pass reads it again.
|
|
935
|
+
if (messages._defer) {
|
|
936
|
+
console.log(`[memhouse] ${chat.source}:${chat.composerId} held for one pass — ${messages._defer}`);
|
|
937
|
+
return null;
|
|
938
|
+
}
|
|
894
939
|
|
|
895
940
|
const source = chat.source;
|
|
896
941
|
// Canonical globally-unique session id: '<source>:<adapter-local id>'.
|
|
@@ -918,19 +963,47 @@ function rowsForChat(chat, host) {
|
|
|
918
963
|
extra: { bubbleCount: chat.bubbleCount || 0 },
|
|
919
964
|
};
|
|
920
965
|
|
|
966
|
+
// Stable numbering first; positional if the parse does not fit it (see subagentSeq).
|
|
967
|
+
const built = buildRows(chat, messages, { id, source, host, folder, project }, true)
|
|
968
|
+
|| buildRows(chat, messages, { id, source, host, folder, project }, false);
|
|
969
|
+
// What a complete ship of this session looks like, recorded ON the session row so the
|
|
970
|
+
// next pass can tell "finished" from "got part way". message_count already carries the
|
|
971
|
+
// message half; without the tool half a pass that dies between the messages insert and
|
|
972
|
+
// the tool_calls insert leaves a session that looks finished forever. See loadExisting.
|
|
973
|
+
session.extra.toolCallCount = built.toolRows.length;
|
|
974
|
+
return { session, ...built };
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
// The message and tool_call rows for one parse. `stable` numbers folded subagent rows by
|
|
978
|
+
// their slot (subagentSeq) and returns null if any of them does not fit; otherwise every
|
|
979
|
+
// row is numbered by its position, the pre-0.18.10 scheme. For a session with no subagents
|
|
980
|
+
// the two are identical.
|
|
981
|
+
function buildRows(chat, messages, { id, source, host, folder, project }, stable) {
|
|
982
|
+
const total = messages.length;
|
|
921
983
|
const msgRows = [];
|
|
922
984
|
const toolRows = [];
|
|
923
|
-
let
|
|
985
|
+
let seqNext = 0; // the session's own turns, dense from 0
|
|
986
|
+
let idx = 0; // the session's own tool calls, dense from 0
|
|
987
|
+
const subIdx = new Map(); // slot → tool calls seen so far in that subagent
|
|
924
988
|
// Set as soon as one message has no timestamp of its own. One is enough: the
|
|
925
989
|
// interpolation denominator is the SESSION's length, so a single interpolated row
|
|
926
990
|
// means this session's stored timestamps drift the next time it grows.
|
|
927
991
|
let tsInterpolated = false;
|
|
928
|
-
for (let
|
|
929
|
-
const m = messages[
|
|
992
|
+
for (let pos = 0; pos < total; pos++) {
|
|
993
|
+
const m = messages[pos];
|
|
994
|
+
const slot = stable ? subagentSlot(m) : null;
|
|
995
|
+
let seq;
|
|
996
|
+
if (slot === null) {
|
|
997
|
+
if (stable && seqNext >= SUBAGENT_SEQ_BASE) return null;
|
|
998
|
+
seq = seqNext++;
|
|
999
|
+
} else {
|
|
1000
|
+
seq = subagentSeq(slot, m._agent.turn);
|
|
1001
|
+
if (seq === null) return null;
|
|
1002
|
+
}
|
|
930
1003
|
if (!hasOwnTs(m)) tsInterpolated = true;
|
|
931
1004
|
const raw = typeof m.content === 'string' ? m.content : JSON.stringify(m.content) ?? '';
|
|
932
1005
|
const text = truncate(raw, TEXT_MAX);
|
|
933
|
-
const ts = messageTs(chat,
|
|
1006
|
+
const ts = messageTs(chat, pos, total, m);
|
|
934
1007
|
const row = {
|
|
935
1008
|
session_id: id,
|
|
936
1009
|
seq,
|
|
@@ -973,10 +1046,18 @@ function rowsForChat(chat, host) {
|
|
|
973
1046
|
}
|
|
974
1047
|
for (const tc of calls) {
|
|
975
1048
|
const args = JSON.stringify(tc.args);
|
|
1049
|
+
let at;
|
|
1050
|
+
if (slot === null) at = idx++;
|
|
1051
|
+
else {
|
|
1052
|
+
const n = subIdx.get(slot) || 0;
|
|
1053
|
+
subIdx.set(slot, n + 1);
|
|
1054
|
+
at = subagentSeq(slot, n);
|
|
1055
|
+
if (at === null) return null;
|
|
1056
|
+
}
|
|
976
1057
|
toolRows.push({
|
|
977
1058
|
session_id: id,
|
|
978
1059
|
seq,
|
|
979
|
-
idx:
|
|
1060
|
+
idx: at,
|
|
980
1061
|
source,
|
|
981
1062
|
host,
|
|
982
1063
|
tool_name: tc.name,
|
|
@@ -987,12 +1068,7 @@ function rowsForChat(chat, host) {
|
|
|
987
1068
|
});
|
|
988
1069
|
}
|
|
989
1070
|
}
|
|
990
|
-
|
|
991
|
-
// next pass can tell "finished" from "got part way". message_count already carries the
|
|
992
|
-
// message half; without the tool half a pass that dies between the messages insert and
|
|
993
|
-
// the tool_calls insert leaves a session that looks finished forever. See loadExisting.
|
|
994
|
-
session.extra.toolCallCount = toolRows.length;
|
|
995
|
-
return { session, msgRows, toolRows, tsInterpolated };
|
|
1071
|
+
return { msgRows, toolRows, tsInterpolated };
|
|
996
1072
|
}
|
|
997
1073
|
|
|
998
1074
|
/**
|
|
@@ -1037,26 +1113,43 @@ function decideEpoch(stored, incoming) {
|
|
|
1037
1113
|
|
|
1038
1114
|
const maxSeq = Number.isFinite(stored.maxSeq) ? stored.maxSeq : -1;
|
|
1039
1115
|
const maxIdx = Number.isFinite(stored.maxIdx) ? stored.maxIdx : -1;
|
|
1040
|
-
|
|
1041
|
-
|
|
1116
|
+
const hashes = stored.hashes || new Map();
|
|
1117
|
+
const tools = stored.tools || new Map();
|
|
1118
|
+
|
|
1119
|
+
// SHRINK is "the house holds a row this parse does not produce", checked per key. It
|
|
1120
|
+
// used to be `parsed - 1 < maxSeq`, which is the same question only while seq is dense
|
|
1121
|
+
// 0..N-1; a folded subagent's rows are numbered from a sparse block (subagentSeq), so a
|
|
1122
|
+
// vanished subagent or a stale pre-0.18.10 numbering leaves stored keys inside the range
|
|
1123
|
+
// that nothing overwrites. The max comparison stays for a caller holding no hashes.
|
|
1124
|
+
const orphans = (have, rows, key, max) => {
|
|
1125
|
+
const want = new Set(rows.map((r) => r[key]));
|
|
1126
|
+
let missing = 0;
|
|
1127
|
+
for (const k of have.keys()) if (!want.has(k)) missing++;
|
|
1128
|
+
const inMax = rows.reduce((a, r) => Math.max(a, r[key]), -1);
|
|
1129
|
+
// How many the house holds, for the log line. `max + 1` is a count only while keys are
|
|
1130
|
+
// dense; with subagent blocks max is ~1e9, so it said "1000000123 messages stored".
|
|
1131
|
+
return { missing, beyond: max > inMax, stored: have.size || max + 1 };
|
|
1132
|
+
};
|
|
1133
|
+
const mo = orphans(hashes, incoming.msgRows, 'seq', maxSeq);
|
|
1134
|
+
if (mo.missing || mo.beyond) {
|
|
1135
|
+
return bump(`${mo.stored} messages stored, ${incoming.msgRows.length} parsed${mo.missing ? `, ${mo.missing} of them not in the new parse` : ''}`);
|
|
1042
1136
|
}
|
|
1043
|
-
|
|
1044
|
-
|
|
1137
|
+
const to = orphans(tools, incoming.toolRows, 'idx', maxIdx);
|
|
1138
|
+
if (to.missing || to.beyond) {
|
|
1139
|
+
return bump(`${to.stored} tool calls stored, ${incoming.toolRows.length} parsed${to.missing ? `, ${to.missing} of them not in the new parse` : ''}`);
|
|
1045
1140
|
}
|
|
1046
1141
|
|
|
1047
1142
|
// A seq the house does not hold is a gap, not a disagreement — there is nothing there
|
|
1048
1143
|
// to destroy, so it is not a reason to fork the session.
|
|
1049
|
-
const hashes = stored.hashes || new Map();
|
|
1050
1144
|
for (const row of incoming.msgRows) {
|
|
1051
|
-
if (row.seq > maxSeq)
|
|
1145
|
+
if (row.seq > maxSeq) continue;
|
|
1052
1146
|
const was = hashes.get(row.seq);
|
|
1053
1147
|
if (was !== undefined && was !== String(row.line_hash)) {
|
|
1054
1148
|
return bump(`message ${row.seq} was rewritten`);
|
|
1055
1149
|
}
|
|
1056
1150
|
}
|
|
1057
|
-
const tools = stored.tools || new Map();
|
|
1058
1151
|
for (const row of incoming.toolRows) {
|
|
1059
|
-
if (row.idx > maxIdx)
|
|
1152
|
+
if (row.idx > maxIdx) continue;
|
|
1060
1153
|
const was = tools.get(row.idx);
|
|
1061
1154
|
if (was !== undefined && (was.tool_name !== row.tool_name || was.args !== row.args)) {
|
|
1062
1155
|
return bump(`tool call ${row.idx} was rewritten`);
|
|
@@ -1072,7 +1165,7 @@ function decideEpoch(stored, incoming) {
|
|
|
1072
1165
|
* token counts, text. Three shipped columns sit outside it and are not derived from
|
|
1073
1166
|
* anything inside it, so a row can be stale in a way no hash comparison can see:
|
|
1074
1167
|
*
|
|
1075
|
-
* ts. Interpolated across [createdAt, lastUpdatedAt] as `
|
|
1168
|
+
* ts. Interpolated across [createdAt, lastUpdatedAt] as `pos / (total - 1)` (position in the parse, not seq) for the
|
|
1076
1169
|
* adapters that record no per-message time. The denominator is the session's length,
|
|
1077
1170
|
* so every stored row's ts moves the moment the session grows. Adapters that DO supply
|
|
1078
1171
|
* `_ts` (Claude Code, opencode, goose, …) are exempt — their timestamps are facts, not
|
|
@@ -1084,8 +1177,11 @@ function decideEpoch(stored, incoming) {
|
|
|
1084
1177
|
* folder / project. Denormalized onto every message and tool_call row; a moved or
|
|
1085
1178
|
* renamed project has to reach the rows that carry it.
|
|
1086
1179
|
*
|
|
1087
|
-
* `source` cannot change (it is the session_id prefix)
|
|
1088
|
-
* `text`,
|
|
1180
|
+
* `source` cannot change (it is the session_id prefix) and `is_subagent` is derived from
|
|
1181
|
+
* `text`, so neither needs a check. Messages' `extra` carries a folded subagent's
|
|
1182
|
+
* `agent` stamp; its id and turn are fixed by the row's seq, and a description the parent
|
|
1183
|
+
* supplies only later (a background fork named by its task notification) is not worth
|
|
1184
|
+
* re-shipping a session for.
|
|
1089
1185
|
*
|
|
1090
1186
|
* When this returns false the session ships whole, exactly as it always has. Pure.
|
|
1091
1187
|
*/
|
|
@@ -1169,6 +1265,24 @@ async function loadStoredParse(client, rooms, id, uid, epoch) {
|
|
|
1169
1265
|
return { hashes, tools };
|
|
1170
1266
|
}
|
|
1171
1267
|
|
|
1268
|
+
/**
|
|
1269
|
+
* How old the newest stats generation is, in ms. Infinity when the table is empty (a TTL
|
|
1270
|
+
* emptied it, or no refresh ever landed); null when it cannot be read at all, so a house
|
|
1271
|
+
* without the tables is not refreshed on every idle pass. Cheap: count() is metadata and
|
|
1272
|
+
* refreshed_at leads the sorting key.
|
|
1273
|
+
*/
|
|
1274
|
+
async function statsAgeMs(client, rooms) {
|
|
1275
|
+
try {
|
|
1276
|
+
const t = statTableNames(rooms.member).session_stats;
|
|
1277
|
+
const rs = await client.query({
|
|
1278
|
+
query: `SELECT count() AS n, dateDiff('second', max(refreshed_at), now64(3)) AS age FROM ${t}`,
|
|
1279
|
+
format: 'JSONEachRow',
|
|
1280
|
+
});
|
|
1281
|
+
const [r] = await rs.json();
|
|
1282
|
+
return Number(r.n) > 0 ? Number(r.age) * 1000 : Infinity;
|
|
1283
|
+
} catch { return null; }
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1172
1286
|
/**
|
|
1173
1287
|
* Recompute the three stat tables — one generation, one INSERT each, all stamped with the
|
|
1174
1288
|
* same refreshed_at. Runs at the end of every ship pass, which is exactly the cadence a
|
|
@@ -1358,9 +1472,12 @@ async function runShip(client, opts = {}) {
|
|
|
1358
1472
|
for (const r of send.toolRows) { r.epoch = epoch; await push('tool_calls', scrub(r)); toolRows++; }
|
|
1359
1473
|
}
|
|
1360
1474
|
for (const table of Object.keys(batches)) await flush(table);
|
|
1361
|
-
// The dashboard's tables, recomputed from what this pass just wrote.
|
|
1362
|
-
//
|
|
1363
|
-
|
|
1475
|
+
// The dashboard's tables, recomputed from what this pass just wrote. A pass that shipped
|
|
1476
|
+
// nothing leaves an exact generation alone — unless it is getting old: the tables carry a
|
|
1477
|
+
// one-day TTL, and a quiet house used to age out of its dashboard entirely (G11).
|
|
1478
|
+
if (statsNeedRefresh({ shipped: sessions, ageMs: sessions > 0 ? 0 : await statsAgeMs(client, rooms) })) {
|
|
1479
|
+
await refreshStats(client, rooms);
|
|
1480
|
+
}
|
|
1364
1481
|
// Anything that only failed while reading messages — the sink is reset by the
|
|
1365
1482
|
// next getAllChats(), so unreported here means never reported at all.
|
|
1366
1483
|
reportAdapterErrors(warned);
|
|
@@ -1541,7 +1658,7 @@ async function main() {
|
|
|
1541
1658
|
}
|
|
1542
1659
|
}
|
|
1543
1660
|
|
|
1544
|
-
module.exports = { runShip, ensureSchema, refreshStats, templateColumns, decideEpoch, tailRows, tailSafe };
|
|
1661
|
+
module.exports = { runShip, ensureSchema, refreshStats, templateColumns, decideEpoch, tailRows, tailSafe, rowsForChat, buildRows, SUBAGENT_SEQ_BASE, SUBAGENT_SEQ_STRIDE };
|
|
1545
1662
|
|
|
1546
1663
|
if (require.main === module) {
|
|
1547
1664
|
main().catch((e) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "memhouse",
|
|
3
|
-
"version": "0.18.
|
|
3
|
+
"version": "0.18.11",
|
|
4
4
|
"description": "memhouse \u2014 agent conversation memory across 17 editors: parse-on-client shippers, a typed ClickHouse store with one set of rooms per member, kernel-agency install, and the agentlytics dashboard",
|
|
5
5
|
"main": "bin/memhouse.js",
|
|
6
6
|
"bin": {
|
|
@@ -20,7 +20,15 @@
|
|
|
20
20
|
"ui/vite.config.js",
|
|
21
21
|
"ui/eslint.config.js",
|
|
22
22
|
"public/",
|
|
23
|
-
"README.md"
|
|
23
|
+
"README.md",
|
|
24
|
+
"docs/README.md",
|
|
25
|
+
"docs/tutorials/",
|
|
26
|
+
"docs/guides/",
|
|
27
|
+
"docs/releases/",
|
|
28
|
+
"examples/",
|
|
29
|
+
"AGENTS.md",
|
|
30
|
+
"SECURITY.md",
|
|
31
|
+
"CHANGELOG.md"
|
|
24
32
|
],
|
|
25
33
|
"scripts": {
|
|
26
34
|
"start": "node bin/memhouse.js",
|