hippo-memory 3.6.0 → 3.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -191
- package/dist/api/context/context-pools.js +2 -2
- package/dist/base/config/config-keys.js +2 -1
- package/dist/base/config/config.d.ts +3 -1
- package/dist/base/config/config.js +1 -0
- package/dist/base/memory/entry.d.ts +10 -2
- package/dist/base/memory/entry.js +12 -4
- package/dist/base/scope/read-admission.js +2 -2
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/domain/consolidate/conflicts.js +3 -1
- package/dist/domain/context/select-policy.d.ts +1 -1
- package/dist/domain/context/select-policy.js +3 -3
- package/dist/domain/sharing/share.js +3 -1
- package/dist/store/sqlite/capture/captures.js +19 -6
- package/dist/store/sqlite/recall/candidates.js +3 -2
- package/dist/surface/capture/capture-error.d.ts +4 -1
- package/dist/surface/capture/capture-error.js +2 -1
- package/dist/surface/capture/command.d.ts +8 -3
- package/dist/surface/capture/command.js +61 -11
- package/dist/surface/capture/extract.d.ts +2 -0
- package/dist/surface/capture/extract.js +7 -0
- package/dist/surface/capture/failure-caller.js +1 -0
- package/dist/surface/capture/user-messages.d.ts +9 -0
- package/dist/surface/capture/user-messages.js +42 -0
- package/dist/surface/cli/hooks/capture-error.js +7 -1
- package/dist/surface/cli/hooks/codex-session-end-worker.js +1 -0
- package/dist/surface/cli/hooks/session-end-worker.js +14 -2
- package/dist/surface/cli/usage.d.ts +1 -1
- package/dist/surface/cli/usage.js +1 -1
- package/extensions/openclaw-plugin/README.md +5 -13
- package/extensions/openclaw-plugin/openclaw.plugin.json +2 -2
- package/extensions/openclaw-plugin/package.json +2 -2
- package/openclaw.plugin.json +2 -2
- package/package.json +4 -5
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Which rows a context call loads.
|
|
2
|
-
import {
|
|
2
|
+
import { isRecallOnly } from '../../base/memory/entry.js';
|
|
3
3
|
import { DEFAULT_SEARCH_CANDIDATE_LIMIT } from '../../base/rank/search-types.js';
|
|
4
4
|
import { projectNames } from '../../base/scope/project-identity.js';
|
|
5
5
|
import { ambientAdmits, ambientEnvelopeAdmits, } from '../../base/scope/read-admission.js';
|
|
@@ -40,7 +40,7 @@ async function loadAmbientEntries(source, tenantId, options) {
|
|
|
40
40
|
// SHORTCUT: a search filters notes after its limit, so a query matching 200 notes returns fewer rows or none;
|
|
41
41
|
// push it into the search SQL if a store gets there.
|
|
42
42
|
return {
|
|
43
|
-
entries: rows.filter((e) => ownTenant(e) && admit(e) && (e.pinned || !
|
|
43
|
+
entries: rows.filter((e) => ownTenant(e) && admit(e) && (e.pinned || !isRecallOnly(e))),
|
|
44
44
|
};
|
|
45
45
|
}
|
|
46
46
|
// The store applies the quality floor too, or its recent-N load stops short of a store whose
|
|
@@ -129,8 +129,9 @@ const CONFIG_TABLE = [
|
|
|
129
129
|
configKey('vectorGraph', block('an object like {"maxVectors": 200000}', D.vectorGraph, (read) => ({
|
|
130
130
|
maxVectors: read('maxVectors', count(D.vectorGraph.maxVectors)),
|
|
131
131
|
}))),
|
|
132
|
-
configKey('capture', block('an object like {"followUpRules": "model"}', D.capture, (read) => ({
|
|
132
|
+
configKey('capture', block('an object like {"followUpRules": "model", "userMessages": false}', D.capture, (read) => ({
|
|
133
133
|
followUpRules: read('followUpRules', choice(D.capture.followUpRules, 'off', 'model')),
|
|
134
|
+
userMessages: read('userMessages', flag(D.capture.userMessages)),
|
|
134
135
|
}))),
|
|
135
136
|
];
|
|
136
137
|
/** The config a parsed config.json gives: each key it sets over the defaults;
|
|
@@ -153,9 +153,11 @@ export interface HippoConfig {
|
|
|
153
153
|
maxVectors: number;
|
|
154
154
|
};
|
|
155
155
|
/** Session capture. `followUpRules: 'model'` sends the person's scrubbed turns after the first to
|
|
156
|
-
* Anthropic (ANTHROPIC_API_KEY), which picks the sentences that state a lasting rule. Default 'off'.
|
|
156
|
+
* Anthropic (ANTHROPIC_API_KEY), which picks the sentences that state a lasting rule. Default 'off'.
|
|
157
|
+
* `userMessages` saves each message the person types as a raw row. Default true. */
|
|
157
158
|
capture: {
|
|
158
159
|
followUpRules: 'off' | 'model';
|
|
160
|
+
userMessages: boolean;
|
|
159
161
|
};
|
|
160
162
|
/** What the composition root read from the environment for this command or request;
|
|
161
163
|
* config.json never sets it. */
|
|
@@ -85,8 +85,16 @@ export declare const COMPACTION_SOURCE_PREFIX = "compaction:";
|
|
|
85
85
|
export declare const USER_RULE_TAG = "from-user";
|
|
86
86
|
/** Tag and category of a later statement capture keeps with no rule word. */
|
|
87
87
|
export declare const CAPTURED_NOTE_TAG = "captured-note";
|
|
88
|
-
/**
|
|
89
|
-
export declare
|
|
88
|
+
/** Tag and source of a message the person typed, which session capture saves whole. */
|
|
89
|
+
export declare const USER_MESSAGE_TAG = "user-message";
|
|
90
|
+
export declare function isUserMessage(entry: {
|
|
91
|
+
readonly tags?: readonly string[];
|
|
92
|
+
}): boolean;
|
|
93
|
+
/** The tags that keep an unpinned row out of ambient context. */
|
|
94
|
+
export declare const RECALL_ONLY_TAGS: readonly ["captured-note", "user-message"];
|
|
95
|
+
/** A note can be a one-off order and a message is chat, so only a matching prompt or an explicit
|
|
96
|
+
* recall may bring either back. */
|
|
97
|
+
export declare function isRecallOnly(entry: {
|
|
90
98
|
readonly tags: readonly string[];
|
|
91
99
|
}): boolean;
|
|
92
100
|
/** `tags` without the tags only capture may set, for every tag a person types or vouches for: a
|
|
@@ -36,14 +36,22 @@ export const COMPACTION_SOURCE_PREFIX = 'compaction:';
|
|
|
36
36
|
export const USER_RULE_TAG = 'from-user';
|
|
37
37
|
/** Tag and category of a later statement capture keeps with no rule word. */
|
|
38
38
|
export const CAPTURED_NOTE_TAG = 'captured-note';
|
|
39
|
-
/**
|
|
40
|
-
export
|
|
41
|
-
|
|
39
|
+
/** Tag and source of a message the person typed, which session capture saves whole. */
|
|
40
|
+
export const USER_MESSAGE_TAG = 'user-message';
|
|
41
|
+
export function isUserMessage(entry) {
|
|
42
|
+
return entry.tags?.includes(USER_MESSAGE_TAG) === true;
|
|
43
|
+
}
|
|
44
|
+
/** The tags that keep an unpinned row out of ambient context. */
|
|
45
|
+
export const RECALL_ONLY_TAGS = [CAPTURED_NOTE_TAG, USER_MESSAGE_TAG];
|
|
46
|
+
/** A note can be a one-off order and a message is chat, so only a matching prompt or an explicit
|
|
47
|
+
* recall may bring either back. */
|
|
48
|
+
export function isRecallOnly(entry) {
|
|
49
|
+
return RECALL_ONLY_TAGS.some((tag) => entry.tags.includes(tag));
|
|
42
50
|
}
|
|
43
51
|
/** `tags` without the tags only capture may set, for every tag a person types or vouches for: a
|
|
44
52
|
* typed tag must not make a row print as the person's rule, and a corrected note is no longer a note. */
|
|
45
53
|
export function withoutCaptureTags(tags) {
|
|
46
|
-
return tags.filter((t) => t !== USER_RULE_TAG && t !== CAPTURED_NOTE_TAG);
|
|
54
|
+
return tags.filter((t) => t !== USER_RULE_TAG && t !== CAPTURED_NOTE_TAG && t !== USER_MESSAGE_TAG);
|
|
47
55
|
}
|
|
48
56
|
/** A memory counts as high schema fit strictly above this. */
|
|
49
57
|
export const HIGH_SCHEMA_FIT = 0.7;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Which rows an ambient load and a summary drill-down admit, as plain data: a store on a worker
|
|
2
2
|
// thread applies the same rule the caller would, since a function cannot cross a thread.
|
|
3
|
-
import { COMPACTION_MEMORY_TAG,
|
|
3
|
+
import { COMPACTION_MEMORY_TAG, isRecallOnly } from '../memory/entry.js';
|
|
4
4
|
import { isWorthSurfacing } from '../memory/memory-quality.js';
|
|
5
5
|
import { classifyOriginProject, projectNames } from './project-identity.js';
|
|
6
6
|
import { passesScopeFilterForRecall } from './recall-scope.js';
|
|
@@ -44,7 +44,7 @@ export function ambientAdmits(e, a) {
|
|
|
44
44
|
export function ambientRefusal(e, a) {
|
|
45
45
|
if (!ambientAdmits(e, a))
|
|
46
46
|
return 'admission';
|
|
47
|
-
return e.pinned || (isWorthSurfacing(e) && !
|
|
47
|
+
return e.pinned || (isWorthSurfacing(e) && !isRecallOnly(e)) ? null : 'quality';
|
|
48
48
|
}
|
|
49
49
|
export function descendantAdmits(row, a) {
|
|
50
50
|
return (passesScopeFilterForRecall(row.scope ?? null, undefined, a.ownScope) &&
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export declare const PACKAGE_VERSION = "3.
|
|
5
|
+
export declare const PACKAGE_VERSION = "3.7.1";
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export declare const EXPIRING_KEYS_MIN_BINARY = "1.64.0";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export const PACKAGE_VERSION = '3.
|
|
5
|
+
export const PACKAGE_VERSION = '3.7.1';
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export const EXPIRING_KEYS_MIN_BINARY = '1.64.0';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EXTRACTED_TAG, Layer } from '../../base/memory/entry.js';
|
|
1
|
+
import { EXTRACTED_TAG, Layer, isUserMessage } from '../../base/memory/entry.js';
|
|
2
2
|
import { calculateStrength, } from '../../base/memory/strength.js';
|
|
3
3
|
import { jaccardMinShared, overlapPartners } from './overlap-index.js';
|
|
4
4
|
import { isQuarantineScope } from '../trust/quarantine.js';
|
|
@@ -30,6 +30,8 @@ export async function detectConflicts(entries, now, options) {
|
|
|
30
30
|
const live = entry.layer !== Layer.Semantic &&
|
|
31
31
|
// An unreviewed quarantined row must not taint a visible memory as conflicted.
|
|
32
32
|
!isQuarantineScope(entry.scope ?? null) &&
|
|
33
|
+
// A message is a dated record of what the person said, not a claim kept true.
|
|
34
|
+
!isUserMessage(entry) &&
|
|
33
35
|
(rescuedIds.has(entry.id) ||
|
|
34
36
|
calculateStrength(entry, now, strength, decayOpts) >= DECAY_THRESHOLD);
|
|
35
37
|
if (!live)
|
|
@@ -25,7 +25,7 @@ export declare function reservePinned(rankedPinned: readonly PricedRow[], budget
|
|
|
25
25
|
export declare function admitWithinBudget<T extends PricedRow>(rows: readonly T[], picked: Picked<T>, budget: number, onOverBudget?: (row: T) => void): void;
|
|
26
26
|
/** The prompt-recall gate a config asks for; a value that is not a number reads as its default. */
|
|
27
27
|
export declare function promptRecallGateOf(inject: HippoConfig['pinnedInject']): PromptRecallGate;
|
|
28
|
-
/** The newest `limit` rows worth surfacing; a pin always is, a
|
|
28
|
+
/** The newest `limit` rows worth surfacing; a pin always is, a recall-only row never is. */
|
|
29
29
|
export declare function newestWorthSurfacing<T extends {
|
|
30
30
|
entry: MemoryEntry;
|
|
31
31
|
}>(rows: readonly T[], limit: number): T[];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// The ranking and budget policy of a context block, on plain rows: the api reads the stores, prices
|
|
2
2
|
// the rows and asks these functions who gets in.
|
|
3
|
-
import {
|
|
3
|
+
import { isRecallOnly } from '../../base/memory/entry.js';
|
|
4
4
|
import { isWorthSurfacing } from '../../base/memory/memory-quality.js';
|
|
5
5
|
import { compareScoredResults } from '../../base/rank/compare.js';
|
|
6
6
|
// Overlap a candidate needs with the prompt when the config sets none.
|
|
@@ -58,7 +58,7 @@ export function promptRecallGateOf(inject) {
|
|
|
58
58
|
}),
|
|
59
59
|
};
|
|
60
60
|
}
|
|
61
|
-
/** The newest `limit` rows worth surfacing; a pin always is, a
|
|
61
|
+
/** The newest `limit` rows worth surfacing; a pin always is, a recall-only row never is. */
|
|
62
62
|
export function newestWorthSurfacing(rows, limit) {
|
|
63
63
|
return ([...rows]
|
|
64
64
|
// Newest first, then id: stable within one store, but same-millisecond rows fall to random ids across ingests.
|
|
@@ -68,7 +68,7 @@ export function newestWorthSurfacing(rows, limit) {
|
|
|
68
68
|
})
|
|
69
69
|
// Filter before slice so a junk row is backfilled past, not counted against N. Pins bypass the floor: a dropped
|
|
70
70
|
// pin's share of the shared budget would go to a backfilled row, and the pin loop could not win it back.
|
|
71
|
-
.filter(({ entry }) => entry.pinned || (isWorthSurfacing(entry) && !
|
|
71
|
+
.filter(({ entry }) => entry.pinned || (isWorthSurfacing(entry) && !isRecallOnly(entry)))
|
|
72
72
|
.slice(0, limit));
|
|
73
73
|
}
|
|
74
74
|
/** No query: the strongest rows first, each kept if it still fits `left` tokens. */
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Sharing local memories to the global store: transfer scoring, share, peers and auto-share.
|
|
2
2
|
import { BadRequestError, NotFoundError } from '../../base/errors/api-errors.js';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
|
-
import { generateId, COMPACTION_MEMORY_TAG, ERROR_TAG, EXTRACTED_TAG, CAPTURED_NOTE_TAG, } from '../../base/memory/entry.js';
|
|
4
|
+
import { generateId, COMPACTION_MEMORY_TAG, ERROR_TAG, EXTRACTED_TAG, CAPTURED_NOTE_TAG, USER_MESSAGE_TAG, } from '../../base/memory/entry.js';
|
|
5
5
|
import { strengthModelFor } from '../../base/memory/eval-switches.js';
|
|
6
6
|
import { CLI_ACTOR } from '../../base/scope/actor.js';
|
|
7
7
|
import { AGENT_MEMORY_SOURCE_PREFIX, AGENT_MEMORY_TAGS, } from '../../base/memory/agent-memory-tools.js';
|
|
@@ -50,6 +50,7 @@ const TRANSFERABLE_TAGS = new Set([
|
|
|
50
50
|
* inherit them. */
|
|
51
51
|
export const NEVER_AUTO_SHARE_TAGS = new Set([
|
|
52
52
|
CAPTURED_NOTE_TAG,
|
|
53
|
+
USER_MESSAGE_TAG,
|
|
53
54
|
'git-learned',
|
|
54
55
|
'session-digest',
|
|
55
56
|
...AGENT_MEMORY_TAGS,
|
|
@@ -61,6 +62,7 @@ export function neverAutoShareTags(sources) {
|
|
|
61
62
|
* detection keeps its own list. */
|
|
62
63
|
export const NO_MERGE_TAGS = new Set([
|
|
63
64
|
CAPTURED_NOTE_TAG,
|
|
65
|
+
USER_MESSAGE_TAG,
|
|
64
66
|
EXTRACTED_TAG,
|
|
65
67
|
'session-digest',
|
|
66
68
|
COMPACTION_MEMORY_TAG,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// hippo.db's Captures: the whole batch on one handle, opened with openStore since a write sets up
|
|
2
2
|
// the mirror folders.
|
|
3
3
|
import { withWriteScope } from '../../../db/index.js';
|
|
4
|
-
import { USER_RULE_TAG } from '../../../base/memory/entry.js';
|
|
4
|
+
import { isUserMessage, USER_RULE_TAG } from '../../../base/memory/entry.js';
|
|
5
5
|
import { duplicateKey, heldTextKeys, storedTextKeys } from '../../../base/util/same-text.js';
|
|
6
6
|
import { ruleReplaces } from '../../../base/memory/rule-replaces.js';
|
|
7
7
|
import { CLI_ACTOR } from '../../../base/scope/actor.js';
|
|
@@ -36,8 +36,12 @@ function writeCapturedEntries(db, files, entries, opts) {
|
|
|
36
36
|
const written = [];
|
|
37
37
|
const actor = opts.actor ?? CLI_ACTOR;
|
|
38
38
|
const outcomes = withWriteScope(db, 'capture_write', () => {
|
|
39
|
-
|
|
39
|
+
// A message is the transcript and an item is what capture made of it, so each lane dedupes
|
|
40
|
+
// only against its own rows, and a held message never stops a rule.
|
|
41
|
+
const all = heldRows(db, entries, opts.dedup);
|
|
42
|
+
const rows = all.filter((row) => !isUserMessage(row));
|
|
40
43
|
const held = storedTextKeys(rows);
|
|
44
|
+
const heldMessages = storedTextKeys(all.filter(isUserMessage));
|
|
41
45
|
return entries.map((entry) => {
|
|
42
46
|
const stamped = stampOriginProject(files, entry);
|
|
43
47
|
// A stated rule replaces an unpinned or untagged-pin capture copy of its own project, which
|
|
@@ -45,11 +49,17 @@ function writeCapturedEntries(db, files, entries, opts) {
|
|
|
45
49
|
const project = entry.tags.includes(USER_RULE_TAG)
|
|
46
50
|
? ruleProjectNames(stamped, opts.dedup)
|
|
47
51
|
: null;
|
|
48
|
-
const stops =
|
|
52
|
+
const stops = isUserMessage(entry)
|
|
53
|
+
? heldMessages
|
|
54
|
+
: project
|
|
55
|
+
? storedTextKeys(rows.filter((r) => !ruleReplaces(r, project)))
|
|
56
|
+
: held;
|
|
49
57
|
if (stops.has(duplicateKey(entry.content)))
|
|
50
58
|
return 'skipped';
|
|
51
|
-
// One rejected entry must not abort the rest of this capture's entries.
|
|
52
|
-
|
|
59
|
+
// One rejected entry must not abort the rest of this capture's entries. A person wrote a
|
|
60
|
+
// message, as with imported notes, and the worth check drops a one-line fact.
|
|
61
|
+
const worthCheck = !isUserMessage(stamped);
|
|
62
|
+
const outcome = gatedWrite(db, files, stamped, { actor: opts.actor, worthCheck });
|
|
53
63
|
if (outcome === 'skipped:rejected')
|
|
54
64
|
return 'rejected';
|
|
55
65
|
if (outcome !== 'written')
|
|
@@ -63,8 +73,11 @@ function writeCapturedEntries(db, files, entries, opts) {
|
|
|
63
73
|
return 'captured';
|
|
64
74
|
});
|
|
65
75
|
});
|
|
76
|
+
// A message gets no markdown mirror: the transcript is its source, and a mirror is a file a team
|
|
77
|
+
// may commit.
|
|
66
78
|
for (const entry of written)
|
|
67
|
-
|
|
79
|
+
if (!isUserMessage(entry))
|
|
80
|
+
writeEntryMirrors(files.hippoRoot, entry);
|
|
68
81
|
if (!opts.lean && written.length > 0)
|
|
69
82
|
updateStatsOn(db, files, { remembered: written.length });
|
|
70
83
|
return outcomes;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RECALL_ONLY_TAGS, ERROR_TAG, Layer, schemaFitFrom, } from '../../../base/memory/entry.js';
|
|
2
2
|
import { FALLBACK_HALF_LIFE_DAYS, PRODUCTION_STRENGTH, calculateStrength, confidenceFacets, } from '../../../base/memory/strength.js';
|
|
3
3
|
import { strengthSql, originInSql, rewardFactorSql, scopeAdmitSql } from '../runtime/rule-sql.js';
|
|
4
4
|
import { DECAY_KEY_HALF_LIVES, DECAY_KEY_SLACK_DAYS, DECAY_KEY_SQL, DECAY_KEY_WHERE, } from '../../../db/decay-key.js';
|
|
@@ -255,8 +255,9 @@ function decayKeySubsetRows(db, rows, cap, now) {
|
|
|
255
255
|
* cap, every such row. */
|
|
256
256
|
export function listContextCandidates(db, model, tenantId, filter) {
|
|
257
257
|
// In the SQL, before the cap: a filter after it would let notes fill the cap and starve other rows.
|
|
258
|
+
const notRecallOnly = RECALL_ONLY_TAGS.map((tag) => `instr(tags_json, '"${tag}"') = 0`);
|
|
258
259
|
const rowsWhere = contextRowsWhere(tenantId, filter, [
|
|
259
|
-
`(pinned = 1 OR
|
|
260
|
+
`(pinned = 1 OR (${notRecallOnly.join(' AND ')}))`,
|
|
260
261
|
]);
|
|
261
262
|
const { params } = rowsWhere;
|
|
262
263
|
const cap = Math.max(0, Math.trunc(filter.cap));
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import type { ContextWith } from '../../api/types.js';
|
|
2
2
|
import type { InfraPorts } from '../../base/ports/infra.js';
|
|
3
|
+
import { type MemoryEntry } from '../../base/memory/entry.js';
|
|
3
4
|
import type { JsonValue } from '../../base/util/json.js';
|
|
4
5
|
import type { CaptureErrorOutcome } from '../../base/ports/store/rejections.js';
|
|
5
6
|
/** The store a failure is logged and its lesson kept in, with that store's config. */
|
|
6
7
|
export type FailureTarget = Pick<ContextWith<'failureLog' | 'lessonWrites'>, 'tenantId' | 'store' | 'config'> & {
|
|
7
8
|
readonly infra: Pick<InfraPorts, 'clock'>;
|
|
9
|
+
/** Embeds a stored lesson off the hook's path; without it the row has no vector. */
|
|
10
|
+
readonly embed?: (entry: MemoryEntry) => void;
|
|
8
11
|
};
|
|
9
12
|
/**
|
|
10
13
|
* Store a failure as an error memory unless it is routine or a repeat, and log it either way, even
|
|
@@ -18,5 +21,5 @@ export interface LessonCaller {
|
|
|
18
21
|
originProject: string;
|
|
19
22
|
origins: readonly string[];
|
|
20
23
|
}
|
|
21
|
-
export declare function storeLesson({ tenantId, store, config, infra }: FailureTarget, text: string, caller?: LessonCaller): Promise<'stored' | 'duplicate'>;
|
|
24
|
+
export declare function storeLesson({ tenantId, store, config, infra, embed }: FailureTarget, text: string, caller?: LessonCaller): Promise<'stored' | 'duplicate'>;
|
|
22
25
|
//# sourceMappingURL=capture-error.d.ts.map
|
|
@@ -41,7 +41,7 @@ function logFailure({ store, tenantId }, payload, lesson, outcome) {
|
|
|
41
41
|
detailHash: hash(lesson.detail),
|
|
42
42
|
});
|
|
43
43
|
}
|
|
44
|
-
export async function storeLesson({ tenantId, store, config, infra }, text, caller) {
|
|
44
|
+
export async function storeLesson({ tenantId, store, config, infra, embed }, text, caller) {
|
|
45
45
|
const sig = failureSignature(text);
|
|
46
46
|
const held = await store.lessonWrites.contentsWithTag(tenantId, 'auto-captured', caller?.origins);
|
|
47
47
|
if (held.some((content) => failureSignature(content) === sig))
|
|
@@ -58,6 +58,7 @@ export async function storeLesson({ tenantId, store, config, infra }, text, call
|
|
|
58
58
|
? { entry, actor: CLI_ACTOR }
|
|
59
59
|
: { entry: { ...entry, originProject: caller.originProject }, actor: caller.actor };
|
|
60
60
|
await store.entryWrites.writeEntry(write);
|
|
61
|
+
embed?.(write.entry);
|
|
61
62
|
return 'stored';
|
|
62
63
|
}
|
|
63
64
|
//# sourceMappingURL=capture-error.js.map
|
|
@@ -18,6 +18,11 @@ export interface CaptureOptions {
|
|
|
18
18
|
stdinText?: string;
|
|
19
19
|
stdinTimedOut?: boolean;
|
|
20
20
|
sessionTurns?: readonly SessionTurn[];
|
|
21
|
+
/** The session the turns came from; without it a transcript's file name stands in. */
|
|
22
|
+
sessionId?: string;
|
|
23
|
+
/** Save the messages the person typed too. Only a session-end worker sets it, and only for the
|
|
24
|
+
* folder's own store, so a scanned transcript or the global store never gets them. */
|
|
25
|
+
saveMessages?: boolean;
|
|
21
26
|
/** False when `sessionTurns` is a later slice of the session (a turn capture), so its first user
|
|
22
27
|
* turn answers the agent. */
|
|
23
28
|
fromStart?: boolean;
|
|
@@ -54,12 +59,12 @@ export declare function runCapture(target: CaptureTarget, options: CaptureOption
|
|
|
54
59
|
* items one, and the first stays unless a later one is the person's rule. */
|
|
55
60
|
export declare function scrubbedItems(items: readonly ExtractedItem[], scrub: (text: string) => string): ExtractedItem[];
|
|
56
61
|
/** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
|
|
57
|
-
* row: a pinned row, or one whose text is no certain defect. */
|
|
58
|
-
export declare function heldKeysOf(
|
|
62
|
+
* row: a pinned row, or one whose text is no certain defect. A held message adds its lane key. */
|
|
63
|
+
export declare function heldKeysOf(all: readonly HeldText[], twins: readonly BoldTwin[]): Set<string>;
|
|
59
64
|
/** The held keys that also stop a stated rule written for `ruleProject` (its names): every copy
|
|
60
65
|
* but one the store's write replaces, which holds the rule's exact text, so a bold twin still
|
|
61
66
|
* stops it. */
|
|
62
|
-
export declare function ruleHeldKeysOf(
|
|
67
|
+
export declare function ruleHeldKeysOf(all: readonly HeldText[], twins: readonly BoldTwin[], ruleProject: readonly string[]): Set<string>;
|
|
63
68
|
export interface CaptureTally {
|
|
64
69
|
captured: number;
|
|
65
70
|
skipped: number;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
|
-
import { createMemory, Layer, USER_RULE_TAG } from '../../base/memory/entry.js';
|
|
3
|
+
import { createMemory, isUserMessage, Layer, USER_MESSAGE_TAG, USER_RULE_TAG, } from '../../base/memory/entry.js';
|
|
4
4
|
import { duplicateKey, storedTextKeys } from '../../base/util/same-text.js';
|
|
5
5
|
import { ruleReplaces } from '../../base/memory/rule-replaces.js';
|
|
6
6
|
import { isCertainReason, storedDefect } from '../../base/memory/memory-quality.js';
|
|
@@ -15,6 +15,7 @@ import { followUpRules, usageLine, withFollowUpRules } from './follow-up.js';
|
|
|
15
15
|
import { collectSessionTurns, sessionTail, resolveLastSessionTranscript, } from './transcript.js';
|
|
16
16
|
import { PRIVATE_DIR } from '../../host/private-dir.js';
|
|
17
17
|
import { evalClock } from '../../infra/eval-switches.js';
|
|
18
|
+
import { userMessageItems, userMessageRef } from './user-messages.js';
|
|
18
19
|
export async function runCapture(target, options) {
|
|
19
20
|
// Tee to a log file for the SessionEnd hook; APPEND, because `hippo sleep` runs first in that
|
|
20
21
|
// sequence and already wrote its header and body.
|
|
@@ -69,7 +70,7 @@ async function runCaptureCore(target, options) {
|
|
|
69
70
|
const read = readCaptureTexts(options);
|
|
70
71
|
if (read === null)
|
|
71
72
|
return;
|
|
72
|
-
const { texts, userTexts, firstFollowUp, followUps } = read;
|
|
73
|
+
const { texts, userTexts, firstFollowUp, followUps, turns } = read;
|
|
73
74
|
if (texts.every((text) => text.trim().length === 0)) {
|
|
74
75
|
console.log('No text to capture from.');
|
|
75
76
|
return;
|
|
@@ -81,7 +82,13 @@ async function runCaptureCore(target, options) {
|
|
|
81
82
|
const dedup = await captureDedup(target.store.folderFiles, options);
|
|
82
83
|
const rules = await modelRules(target, dedup, followUps);
|
|
83
84
|
const patterned = extractFromTexts(scrubbed, userTexts, firstFollowUp);
|
|
84
|
-
|
|
85
|
+
// A message dedupes only against messages, so a rule it states is kept as a rule too.
|
|
86
|
+
const keepsMessages = options.saveMessages === true && target.config.capture.userMessages;
|
|
87
|
+
const messages = keepsMessages ? userMessageItems(turns, scrub) : [];
|
|
88
|
+
const extracted = [
|
|
89
|
+
...scrubbedItems(withFollowUpRules(patterned, rules, scrub), scrub),
|
|
90
|
+
...messages,
|
|
91
|
+
];
|
|
85
92
|
if (extracted.length === 0) {
|
|
86
93
|
console.log('No actionable items found in the input.');
|
|
87
94
|
return;
|
|
@@ -109,6 +116,7 @@ async function captureFromCli({ store, config, embed }, options, dedup, extracte
|
|
|
109
116
|
const writeOpts = {
|
|
110
117
|
tenantId: options.global ? undefined : options.tenantId,
|
|
111
118
|
originProject: options.originProject,
|
|
119
|
+
sessionId: options.sessionId ?? transcriptSession(options.transcriptPath),
|
|
112
120
|
lean: false,
|
|
113
121
|
dedup,
|
|
114
122
|
now: evalClock()(),
|
|
@@ -118,23 +126,38 @@ async function captureFromCli({ store, config, embed }, options, dedup, extracte
|
|
|
118
126
|
}
|
|
119
127
|
return previewCapture({ store, config }, writeOpts, extracted, keys, ruleKeys);
|
|
120
128
|
}
|
|
129
|
+
/** A Claude Code or Codex transcript is named for its session. */
|
|
130
|
+
function transcriptSession(transcriptPath) {
|
|
131
|
+
return transcriptPath ? path.basename(transcriptPath, '.jsonl') : undefined;
|
|
132
|
+
}
|
|
121
133
|
/** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
|
|
122
134
|
* items one, and the first stays unless a later one is the person's rule. */
|
|
123
135
|
export function scrubbedItems(items, scrub) {
|
|
124
136
|
return oncePerText(items.map((item) => ({ ...item, content: scrub(item.content) })));
|
|
125
137
|
}
|
|
138
|
+
/** A message is the transcript and an item is what capture made of it, so each dedupes in its own
|
|
139
|
+
* lane: the same text can be both, and a held message never stops a rule. */
|
|
140
|
+
function laneKey(item) {
|
|
141
|
+
const key = duplicateKey(item.content);
|
|
142
|
+
return isUserMessage(item) ? `${USER_MESSAGE_TAG}\n${key}` : key;
|
|
143
|
+
}
|
|
126
144
|
/** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
|
|
127
|
-
* row: a pinned row, or one whose text is no certain defect. */
|
|
128
|
-
export function heldKeysOf(
|
|
145
|
+
* row: a pinned row, or one whose text is no certain defect. A held message adds its lane key. */
|
|
146
|
+
export function heldKeysOf(all, twins) {
|
|
147
|
+
const rows = all.filter((row) => !isUserMessage(row));
|
|
129
148
|
const keys = storedTextKeys(rows);
|
|
130
149
|
for (const key of twinKeys(rows, twins))
|
|
131
150
|
keys.add(key);
|
|
151
|
+
for (const row of all)
|
|
152
|
+
if (isUserMessage(row))
|
|
153
|
+
keys.add(laneKey(row));
|
|
132
154
|
return keys;
|
|
133
155
|
}
|
|
134
156
|
/** The held keys that also stop a stated rule written for `ruleProject` (its names): every copy
|
|
135
157
|
* but one the store's write replaces, which holds the rule's exact text, so a bold twin still
|
|
136
158
|
* stops it. */
|
|
137
|
-
export function ruleHeldKeysOf(
|
|
159
|
+
export function ruleHeldKeysOf(all, twins, ruleProject) {
|
|
160
|
+
const rows = all.filter((row) => !isUserMessage(row));
|
|
138
161
|
const keys = storedTextKeys(rows.filter((row) => !ruleReplaces(row, ruleProject)));
|
|
139
162
|
for (const key of twinKeys(rows, twins))
|
|
140
163
|
keys.add(key);
|
|
@@ -189,7 +212,7 @@ function readCaptureTexts(options) {
|
|
|
189
212
|
switch (options.source) {
|
|
190
213
|
case 'stdin': {
|
|
191
214
|
try {
|
|
192
|
-
return { texts: [fs.readFileSync(0, 'utf8')], userTexts: 0, followUps: [] };
|
|
215
|
+
return { texts: [fs.readFileSync(0, 'utf8')], userTexts: 0, followUps: [], turns: [] };
|
|
193
216
|
}
|
|
194
217
|
catch (err) {
|
|
195
218
|
log.warn(`reading stdin failed: ${errorMessage(err)}`, errorFields(err));
|
|
@@ -209,7 +232,12 @@ function readCaptureTexts(options) {
|
|
|
209
232
|
process.exitCode = 1;
|
|
210
233
|
return null;
|
|
211
234
|
}
|
|
212
|
-
return {
|
|
235
|
+
return {
|
|
236
|
+
texts: [fs.readFileSync(options.filePath, 'utf8')],
|
|
237
|
+
userTexts: 0,
|
|
238
|
+
followUps: [],
|
|
239
|
+
turns: [],
|
|
240
|
+
};
|
|
213
241
|
}
|
|
214
242
|
case 'last-session':
|
|
215
243
|
return readSessionTexts(options);
|
|
@@ -240,6 +268,7 @@ function readSessionTexts(options) {
|
|
|
240
268
|
userTexts: users.length,
|
|
241
269
|
firstFollowUp: opener,
|
|
242
270
|
followUps: userTurns.slice(opener),
|
|
271
|
+
turns,
|
|
243
272
|
};
|
|
244
273
|
}
|
|
245
274
|
/** One entry per new text: building can throw, so a held text or a repeat within the batch is
|
|
@@ -248,7 +277,7 @@ function planCapture(extracted, isHeld, build) {
|
|
|
248
277
|
const plan = { entries: [], keys: [], slots: [] };
|
|
249
278
|
const firstSlot = new Map();
|
|
250
279
|
for (const item of extracted) {
|
|
251
|
-
const key =
|
|
280
|
+
const key = laneKey(item);
|
|
252
281
|
let slot = isHeld(item, key) ? -1 : firstSlot.get(key);
|
|
253
282
|
if (slot === undefined) {
|
|
254
283
|
slot = plan.entries.push(build(item)) - 1;
|
|
@@ -290,14 +319,14 @@ export async function captureExtractedItems({ store, config, embed }, options, e
|
|
|
290
319
|
async function previewCapture({ store: { rejections }, config }, options, extracted, keys, ruleKeys) {
|
|
291
320
|
const tally = { captured: 0, skipped: 0, rejected: 0 };
|
|
292
321
|
const held = heldCheck(keys, ruleKeys);
|
|
293
|
-
const fresh = extracted.filter((item) => !held(item,
|
|
322
|
+
const fresh = extracted.filter((item) => !held(item, laneKey(item)));
|
|
294
323
|
const refused = await rejections.refusedAmong(fresh.map((item) => {
|
|
295
324
|
const entry = captureEntry(item, options, config.defaultHalfLifeDays);
|
|
296
325
|
return { tenantId: entry.tenantId, entryId: entry.id, content: entry.content };
|
|
297
326
|
}));
|
|
298
327
|
let next = 0;
|
|
299
328
|
for (const item of extracted) {
|
|
300
|
-
if (held(item,
|
|
329
|
+
if (held(item, laneKey(item))) {
|
|
301
330
|
tally.skipped++;
|
|
302
331
|
console.log(` [skip] (${item.category}) ${item.content.slice(0, 80)}`);
|
|
303
332
|
}
|
|
@@ -313,6 +342,8 @@ async function previewCapture({ store: { rejections }, config }, options, extrac
|
|
|
313
342
|
return tally;
|
|
314
343
|
}
|
|
315
344
|
function captureEntry(item, options, baseHalfLifeDays) {
|
|
345
|
+
if (isUserMessage(item))
|
|
346
|
+
return userMessageEntry(item, options, baseHalfLifeDays);
|
|
316
347
|
// kind stays 'distilled': these are curated items, not raw transcript (see MEMORY_ENVELOPE.md).
|
|
317
348
|
// The write tenant must match the dedup read's, or scoped dedup passes and the row lands in
|
|
318
349
|
// 'default'. A rule the person stated is pinned, so it injects even when a later prompt shares
|
|
@@ -333,4 +364,23 @@ function captureEntry(item, options, baseHalfLifeDays) {
|
|
|
333
364
|
return created;
|
|
334
365
|
return { ...created, originProject: projectId(options.originProject) };
|
|
335
366
|
}
|
|
367
|
+
/** A message the person typed is raw transcript, so it is stored as written (MEMORY_ENVELOPE.md)
|
|
368
|
+
* with its owner and source. */
|
|
369
|
+
function userMessageEntry(item, options, baseHalfLifeDays) {
|
|
370
|
+
const created = createMemory(item.content, {
|
|
371
|
+
kind: 'raw',
|
|
372
|
+
layer: Layer.Episodic,
|
|
373
|
+
tags: [USER_MESSAGE_TAG],
|
|
374
|
+
source: USER_MESSAGE_TAG,
|
|
375
|
+
confidence: 'observed',
|
|
376
|
+
owner: `user:${options.actor ?? 'local'}`,
|
|
377
|
+
artifactRef: userMessageRef(options.sessionId, item.content),
|
|
378
|
+
tenantId: options.tenantId,
|
|
379
|
+
sourceSessionId: options.sessionId,
|
|
380
|
+
baseHalfLifeDays,
|
|
381
|
+
}, options.now);
|
|
382
|
+
if (options.originProject === undefined)
|
|
383
|
+
return created;
|
|
384
|
+
return { ...created, originProject: projectId(options.originProject) };
|
|
385
|
+
}
|
|
336
386
|
//# sourceMappingURL=command.js.map
|
|
@@ -18,6 +18,8 @@ export type NoiseCheck = (line: string, sentenceOpen: boolean) => boolean;
|
|
|
18
18
|
export declare function proseOnly(text: string, isNoise?: NoiseCheck): string;
|
|
19
19
|
/** The text above has not ended its sentence; "e.g." at its end has not. */
|
|
20
20
|
export declare function leftOpen(previous: string, next: string): boolean;
|
|
21
|
+
/** `text` without its paired bold markers, which can split an address or key past the scrubber. */
|
|
22
|
+
export declare function withoutBold(text: string): string;
|
|
21
23
|
/** A sentence as capture stores it: its lead label and closing marks trimmed. */
|
|
22
24
|
export declare function statedText(sentence: string): string;
|
|
23
25
|
/** The sentences a pattern reads in `text`: wrapped lines joined, bold cut, and none that starts
|
|
@@ -146,6 +146,13 @@ function pairedBold(line) {
|
|
|
146
146
|
/** The segmenter splits after "**Label.**" and leaves the closing `**` on the next rule, so each
|
|
147
147
|
* real pair goes; a glob, `2**10` or `**kwargs` has no partner and stays as written. */
|
|
148
148
|
const boldPairs = (line) => pairedBold(line).sort((a, b) => a - b);
|
|
149
|
+
/** `text` without its paired bold markers, which can split an address or key past the scrubber. */
|
|
150
|
+
export function withoutBold(text) {
|
|
151
|
+
return text
|
|
152
|
+
.split('\n')
|
|
153
|
+
.map((line) => cutOut(line, boldPairs(line)))
|
|
154
|
+
.join('\n');
|
|
155
|
+
}
|
|
149
156
|
/** `line` without the two characters at each of the sorted `cuts`, built in one pass. */
|
|
150
157
|
function cutOut(line, cuts) {
|
|
151
158
|
let out = '';
|
|
@@ -63,6 +63,7 @@ export async function captureFailureForCaller(given, req) {
|
|
|
63
63
|
let logged = 'store-failed';
|
|
64
64
|
try {
|
|
65
65
|
const caller = { actor: key.owner, originProject: req.project.name, origins: key.project };
|
|
66
|
+
// No embed: a server process keeps the model off its request path, as lean capture does.
|
|
66
67
|
const outcome = failure.skip === null ? await storeLesson(ctx, failure.text, caller) : failure.skip;
|
|
67
68
|
logged = outcome;
|
|
68
69
|
return { outcome };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type ExtractedItem } from './extract.js';
|
|
2
|
+
import type { SessionTurn } from './transcript.js';
|
|
3
|
+
/** One item per user turn with enough prose, scrubbed and cut, in the order typed; scrubbed after
|
|
4
|
+
* the bold markers go, as capture's own items are. */
|
|
5
|
+
export declare function userMessageItems(turns: readonly SessionTurn[], scrub: (text: string) => string): ExtractedItem[];
|
|
6
|
+
/** Where a saved message came from: its session and a hash of its text, since a transcript line
|
|
7
|
+
* has no stable id across a turn close and the full close. */
|
|
8
|
+
export declare function userMessageRef(sessionId: string | undefined, text: string): string;
|
|
9
|
+
//# sourceMappingURL=user-messages.d.ts.map
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// The messages a person typed in a session, as capture items: each one whole, so a fact told in
|
|
2
|
+
// chat can be recalled later even when it states no rule.
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { USER_MESSAGE_TAG } from '../../base/memory/entry.js';
|
|
5
|
+
import { ID_SUFFIX_CHARS } from '../../base/util/token-text.js';
|
|
6
|
+
import { truncateCodePointSafe } from '../../base/util/code-point-cut.js';
|
|
7
|
+
import { proseOnly, withoutBold } from './extract.js';
|
|
8
|
+
const MAX_USER_MESSAGE_CHARS = 2000;
|
|
9
|
+
/** A reply such as "ok thanks" holds nothing to recall. */
|
|
10
|
+
const MIN_USER_MESSAGE_WORDS = 3;
|
|
11
|
+
const CUT_MARK = '…';
|
|
12
|
+
/** One item per user turn with enough prose, scrubbed and cut, in the order typed; scrubbed after
|
|
13
|
+
* the bold markers go, as capture's own items are. */
|
|
14
|
+
export function userMessageItems(turns, scrub) {
|
|
15
|
+
return turns
|
|
16
|
+
.filter((turn) => turn.role === 'user')
|
|
17
|
+
.map((turn) => scrub(withoutBold(turn.text)).trim())
|
|
18
|
+
.filter(isWorthKeeping)
|
|
19
|
+
.map((text) => ({ content: cut(text), category: USER_MESSAGE_TAG, tags: [USER_MESSAGE_TAG] }));
|
|
20
|
+
}
|
|
21
|
+
function isWorthKeeping(text) {
|
|
22
|
+
return proseOnly(text).split(/\s+/).filter(Boolean).length >= MIN_USER_MESSAGE_WORDS;
|
|
23
|
+
}
|
|
24
|
+
/** A long message is mostly a paste, so it keeps only the person's own words, then is cut. */
|
|
25
|
+
function cut(text) {
|
|
26
|
+
if (text.length <= MAX_USER_MESSAGE_CHARS)
|
|
27
|
+
return text;
|
|
28
|
+
const prose = proseOnly(text)
|
|
29
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
30
|
+
.trim();
|
|
31
|
+
if (prose.length <= MAX_USER_MESSAGE_CHARS)
|
|
32
|
+
return prose;
|
|
33
|
+
const room = MAX_USER_MESSAGE_CHARS - CUT_MARK.length;
|
|
34
|
+
return truncateCodePointSafe(prose, room).trimEnd() + CUT_MARK;
|
|
35
|
+
}
|
|
36
|
+
/** Where a saved message came from: its session and a hash of its text, since a transcript line
|
|
37
|
+
* has no stable id across a turn close and the full close. */
|
|
38
|
+
export function userMessageRef(sessionId, text) {
|
|
39
|
+
const hash = createHash('sha256').update(text).digest('hex').slice(0, ID_SUFFIX_CHARS);
|
|
40
|
+
return `session:${sessionId ?? 'unknown'}/${hash}`;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=user-messages.js.map
|