hippo-memory 3.2.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +10 -1
  2. package/dist/base/config/config-keys.js +3 -0
  3. package/dist/base/config/config.d.ts +5 -0
  4. package/dist/base/config/config.js +3 -0
  5. package/dist/base/memory/rule-replaces.d.ts +10 -0
  6. package/dist/base/memory/rule-replaces.js +14 -0
  7. package/dist/base/ports/llm.d.ts +8 -1
  8. package/dist/base/ports/store/types.d.ts +2 -2
  9. package/dist/base/util/same-text.d.ts +0 -7
  10. package/dist/base/util/same-text.js +0 -12
  11. package/dist/base/util/version.d.ts +1 -1
  12. package/dist/base/util/version.js +1 -1
  13. package/dist/infra/anthropic-llm.js +17 -3
  14. package/dist/store/sqlite/capture/captures.js +4 -3
  15. package/dist/store/sqlite/recall/candidates.js +1 -0
  16. package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
  17. package/dist/store/sqlite/sharing/copy-rows.js +1 -1
  18. package/dist/surface/capture/command.d.ts +5 -0
  19. package/dist/surface/capture/command.js +37 -9
  20. package/dist/surface/capture/extract.d.ts +16 -0
  21. package/dist/surface/capture/extract.js +56 -15
  22. package/dist/surface/capture/follow-up.d.ts +21 -0
  23. package/dist/surface/capture/follow-up.js +164 -0
  24. package/dist/surface/cli/hook-runtime.d.ts +4 -3
  25. package/dist/surface/cli/hook-runtime.js +11 -3
  26. package/dist/surface/cli/hooks/capture.js +1 -1
  27. package/dist/surface/cli/hooks/codex-session-end-worker.js +1 -1
  28. package/dist/surface/cli/hooks/session-end-worker.js +3 -3
  29. package/dist-ui/assets/index-CHXQD8Lk.css +1 -0
  30. package/dist-ui/assets/index-Q9SY-1Bj.js +1 -0
  31. package/dist-ui/assets/react-O0ULxr_i.js +9 -0
  32. package/dist-ui/index.html +3 -3
  33. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  34. package/extensions/openclaw-plugin/package.json +1 -1
  35. package/openclaw.plugin.json +1 -1
  36. package/package.json +1 -1
  37. package/dist-ui/assets/index-DvZmCHQ5.js +0 -33
  38. package/dist-ui/assets/index-dFloKRVr.css +0 -1
  39. package/dist-ui/assets/react-BpaniaUk.js +0 -17
package/README.md CHANGED
@@ -201,6 +201,15 @@ hippo capture --file conversation.md
201
201
  hippo capture --file conversation.md --dry-run
202
202
  ```
203
203
 
204
+ #### Rules from follow-up turns (opt-in, uses a model)
205
+
206
+ Word patterns miss a rule that has no rule word in it, such as "That file is generated, so changes belong in the template it comes from." With `{"capture": {"followUpRules": "model"}}` in `.hippo/config.json` and `ANTHROPIC_API_KEY` set, a session capture also asks a model which of your sentences state a lasting rule. Hippo stores those sentences, in your own words, as pinned rules.
207
+
208
+ - **What it sends:** your turns after the first one in the session, the last 20, each cut to 1,000 characters. Pasted blocks, fenced code and log lines are removed first, and secrets, email addresses and home paths are masked. The assistant's replies are never sent.
209
+ - **To whom:** Anthropic, with your `ANTHROPIC_API_KEY`, in one call per session capture. A failed call logs one line and capture keeps its pattern rules. A session with no project never makes the call.
210
+ - **On and off:** the default is `"off"`. Set `"model"` to turn it on; set `"off"` again, or unset the key, to turn it off.
211
+ - A capture after each reply (`--turn`, as VS Code's agent mode runs) never calls the model, so those runtimes gain little.
212
+
204
213
  ### Slack ingestion (E1.3)
205
214
 
206
215
  Hippo accepts Slack Events API webhooks at `POST /v1/connectors/slack/events`. Configure `SLACK_SIGNING_SECRET` (validated on every request) and point Slack at `https://<your-host>/v1/connectors/slack/events`. Messages land as `kind='raw'` memories with `slack://team/channel/ts` provenance and a `slack:public:Cxxx` or `slack:private:Cxxx` scope. Source deletions are honored (GDPR). To rotate the secret, put the new one in `SLACK_SIGNING_SECRET` and the old one in `SLACK_SIGNING_SECRET_PREVIOUS`, and set `SLACK_SIGNING_SECRET_PREVIOUS_UNTIL` to an ISO date (`2026-12-01` means 00:00 UTC that day) or timestamp. From then on requests signed with the old secret are refused, with one warning in the log. Left unset, the old secret works for as long as it stays in the environment.
@@ -836,7 +845,7 @@ This adds a `<!-- hippo:start -->` ... `<!-- hippo:end -->` block that tells the
836
845
  The block asks for nothing a hook already does, because each extra tool call re-reads the whole context. Re-running `hippo init` swaps a block an older hippo wrote for the current one, as long as nobody edited it. It leaves an edited block alone and says so, and never touches text outside the markers.
837
846
 
838
847
  For Claude Code, it also adds 7 hook entries to `~/.claude/settings.json`:
839
- - a `SessionEnd` hook that runs `hippo sleep` and then `hippo capture` when the session exits. Capture matches the last 20 user and 10 assistant messages of the transcript against word patterns for decisions, rules, errors and preferences. It uses no model and does not read earlier turns, so a correction older than that window needs a `hippo remember` by hand.
848
+ - a `SessionEnd` hook that runs `hippo sleep` and then `hippo capture` when the session exits. Capture matches the last 20 user and 10 assistant messages of the transcript against word patterns for decisions, rules, errors and preferences. It uses no model unless you turn on `capture.followUpRules` (see Conversation Capture) and does not read earlier turns, so a correction older than that window needs a `hippo remember` by hand.
840
849
  - a `SessionStart` hook that prints the previous session's consolidation log on stderr, where `claude --debug` shows it. When the last sleep failed or a compaction summary was set aside, it shows you one line that points to `hippo doctor`.
841
850
  - a `UserPromptSubmit` hook that runs `hippo context --pinned-only --include-recent 5 --format additional-context` every turn. It re-injects pinned memories (`hippo remember <text> --pin`) plus up to 5 memories that share words with your prompt. When nothing matches, it adds only the pinned ones. Since 1.55.0 this replaces the 5 newest memories, which cut the median block from 847 to 533 tokens in our eval. `{"pinnedInject":{"promptRecall":false}}` brings back the 5 newest, so fresh same-session lessons appear on the next prompt whatever you ask. The block is rendered without live strength percentages, so it stays byte-identical while its memories do not change, and it is sent only when it changed since the session's last prompt: an unchanged block is skipped, resent every 10 skips (`pinnedInject.refreshTurns`, `0` never resends) and resent after compaction. The prompt-matched memories go in a separate block that is never skipped, so they are sent on every prompt they match. `{"pinnedInject":{"skipUnchanged":false}}` sends the pinned block every turn as before. Opt out entirely with `{"pinnedInject":{"enabled":false}}` in `.hippo/config.json`.
842
851
  - a `PreCompact` hook that runs `hippo pre-compact` before the transcript gets summarized. It records the compaction in the store, saves a working-state snapshot (task/summary/next step) so mid-session compaction can't drop it, and asks the summariser to end its summary with a "Memories for hippo" list: the lessons, decisions and corrections from the session that should outlive it. The `SessionEnd` hook still owns extracting durable memories from the transcript.
@@ -128,6 +128,9 @@ const CONFIG_TABLE = [
128
128
  configKey('vectorGraph', block('an object like {"maxVectors": 200000}', D.vectorGraph, (read) => ({
129
129
  maxVectors: read('maxVectors', count(D.vectorGraph.maxVectors)),
130
130
  }))),
131
+ configKey('capture', block('an object like {"followUpRules": "model"}', D.capture, (read) => ({
132
+ followUpRules: read('followUpRules', choice(D.capture.followUpRules, 'off', 'model')),
133
+ }))),
131
134
  ];
132
135
  /** The config a parsed config.json gives: each key it sets over the defaults;
133
136
  * src/host/config-file.ts reads it. */
@@ -150,6 +150,11 @@ export interface HippoConfig {
150
150
  vectorGraph: {
151
151
  maxVectors: number;
152
152
  };
153
+ /** Session capture. `followUpRules: 'model'` sends the person's scrubbed turns after the first to
154
+ * Anthropic (ANTHROPIC_API_KEY), which picks the sentences that state a lasting rule. Default 'off'. */
155
+ capture: {
156
+ followUpRules: 'off' | 'model';
157
+ };
153
158
  /** What the composition root read from the environment for this command or request;
154
159
  * config.json never sets it. */
155
160
  runtime?: RuntimeSettings;
@@ -100,5 +100,8 @@ export const DEFAULT_CONFIG = {
100
100
  vectorGraph: {
101
101
  maxVectors: 200_000,
102
102
  },
103
+ capture: {
104
+ followUpRules: 'off',
105
+ },
103
106
  };
104
107
  //# sourceMappingURL=config.js.map
@@ -0,0 +1,10 @@
1
+ import { type MemoryEntry } from './entry.js';
2
+ type HeldCopy = Partial<Pick<MemoryEntry, 'source' | 'pinned' | 'tags' | 'originProject' | 'supersededBy'>> & {
3
+ confidence?: string;
4
+ };
5
+ /** Whether a stated rule replaces this held copy: a live, observed capture row of `ruleProject`
6
+ * (the rule's project names), unpinned or pinned without the user-rule tag. Any other copy stops
7
+ * the rule, so a replayed transcript revives nothing and no project loses its row. */
8
+ export declare function ruleReplaces(row: HeldCopy, ruleProject: readonly string[]): boolean;
9
+ export {};
10
+ //# sourceMappingURL=rule-replaces.d.ts.map
@@ -0,0 +1,14 @@
1
+ // Not in same-text because it reads the user-rule tag, and base/util may not import base/memory.
2
+ import { USER_RULE_TAG } from './entry.js';
3
+ /** Whether a stated rule replaces this held copy: a live, observed capture row of `ruleProject`
4
+ * (the rule's project names), unpinned or pinned without the user-rule tag. Any other copy stops
5
+ * the rule, so a replayed transcript revives nothing and no project loses its row. */
6
+ export function ruleReplaces(row, ruleProject) {
7
+ return ((row.pinned !== true || (row.tags !== undefined && !row.tags.includes(USER_RULE_TAG))) &&
8
+ row.source === 'capture' &&
9
+ Boolean(row.originProject) &&
10
+ ruleProject.includes(row.originProject ?? '') &&
11
+ row.supersededBy == null &&
12
+ row.confidence === 'observed');
13
+ }
14
+ //# sourceMappingURL=rule-replaces.js.map
@@ -22,12 +22,19 @@ export type LlmFailure = {
22
22
  } | {
23
23
  kind: 'empty';
24
24
  };
25
- export type LlmReply = {
25
+ /** What the provider billed for one call; an empty reply is billed too. */
26
+ export interface LlmUsage {
27
+ readonly inputTokens: number;
28
+ readonly outputTokens: number;
29
+ }
30
+ export type LlmReply = ({
26
31
  ok: true;
27
32
  text: string;
28
33
  } | {
29
34
  ok: false;
30
35
  failure: LlmFailure;
36
+ }) & {
37
+ readonly usage?: LlmUsage;
31
38
  };
32
39
  /** One user prompt in, the reply's first text block or a typed failure out. Never throws. */
33
40
  export interface Llm {
@@ -112,8 +112,8 @@ export interface AuditEvent {
112
112
  targetId: string | null;
113
113
  metadata: JsonObject;
114
114
  }
115
- /** A stored text the capture dedup reads; `pinned` is optional so an adapter that omits it still fits. */
116
- export type HeldText = Pick<MemoryEntry, 'content' | 'source' | 'originProject'> & Partial<Pick<MemoryEntry, 'pinned' | 'id' | 'supersededBy'>> & {
115
+ /** A stored text the capture dedup reads; `pinned` and `tags` are optional so an adapter that omits them still fits. */
116
+ export type HeldText = Pick<MemoryEntry, 'content' | 'source' | 'originProject'> & Partial<Pick<MemoryEntry, 'pinned' | 'tags' | 'id' | 'supersededBy'>> & {
117
117
  confidence?: string;
118
118
  };
119
119
  /** One project name, or every name a project's rows carry. */
@@ -15,13 +15,6 @@ export declare function mergedText(header: string, texts: readonly string[]): st
15
15
  * whole after its header.
16
16
  */
17
17
  export declare function heldTexts(entry: Text): string[];
18
- /** Whether a stated rule replaces this held copy rather than giving way to it: a live, unpinned,
19
- * observed capture row of one of `ruleProject`, the names of the rule's project. A superseded,
20
- * pinned, typed, verified, no-project or other-project copy stops the rule, so a replayed
21
- * transcript revives nothing and no project loses its row. */
22
- export declare function ruleReplaces(row: Text & Partial<Pick<MemoryEntry, 'originProject' | 'supersededBy'>> & {
23
- confidence?: string;
24
- }, ruleProject: readonly string[]): boolean;
25
18
  /** Keys of every text a row holds word for word: its own, plus each source text inside a
26
19
  * sleep-merged row. */
27
20
  export declare function heldTextKeys(entry: Text): string[];
@@ -38,18 +38,6 @@ export function heldTexts(entry) {
38
38
  .slice(1)
39
39
  .map((t) => t.replace(/\n {2}/g, '\n'));
40
40
  }
41
- /** Whether a stated rule replaces this held copy rather than giving way to it: a live, unpinned,
42
- * observed capture row of one of `ruleProject`, the names of the rule's project. A superseded,
43
- * pinned, typed, verified, no-project or other-project copy stops the rule, so a replayed
44
- * transcript revives nothing and no project loses its row. */
45
- export function ruleReplaces(row, ruleProject) {
46
- return (row.pinned !== true &&
47
- row.source === 'capture' &&
48
- Boolean(row.originProject) &&
49
- ruleProject.includes(row.originProject ?? '') &&
50
- row.supersededBy == null &&
51
- row.confidence === 'observed');
52
- }
53
41
  /** Keys of every text a row holds word for word: its own, plus each source text inside a
54
42
  * sleep-merged row. */
55
43
  export function heldTextKeys(entry) {
@@ -2,7 +2,7 @@
2
2
  * Single source of truth for the package version, written by scripts/sync-version.mjs; never
3
3
  * edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
4
4
  * package.json read: the published bundle may lack package.json on a path ESM can resolve. */
5
- export declare const PACKAGE_VERSION = "3.2.1";
5
+ export declare const PACKAGE_VERSION = "3.3.0";
6
6
  /** The floor a store takes on its first expiring key: the first release with schema v53, set by
7
7
  * hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
8
8
  export declare const EXPIRING_KEYS_MIN_BINARY = "1.64.0";
@@ -2,7 +2,7 @@
2
2
  * Single source of truth for the package version, written by scripts/sync-version.mjs; never
3
3
  * edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
4
4
  * package.json read: the published bundle may lack package.json on a path ESM can resolve. */
5
- export const PACKAGE_VERSION = '3.2.1';
5
+ export const PACKAGE_VERSION = '3.3.0';
6
6
  /** The floor a store takes on its first expiring key: the first release with schema v53, set by
7
7
  * hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
8
8
  export const EXPIRING_KEYS_MIN_BINARY = '1.64.0';
@@ -13,12 +13,24 @@ const DEFAULT_MODEL = 'claude-sonnet-4-6';
13
13
  // The error body names the cause (bad key, no credit, unknown model) in its first line or two.
14
14
  const ERROR_SNIPPET_CHARS = 256;
15
15
  const UTF8_MAX_BYTES_PER_CHAR = 4;
16
- /** The first content block's text, trimmed; empty when the reply has no such block. */
16
+ /** The first text block's text, trimmed; empty when the reply has no such block. A thinking
17
+ * model puts its thinking block first. */
17
18
  function firstText(reply) {
18
- const first = Array.isArray(reply.content) ? reply.content[0] : undefined;
19
+ const blocks = Array.isArray(reply.content) ? reply.content : [];
20
+ const first = blocks.find((block) => isJsonObject(block) && isJsonString(block.text));
19
21
  const text = isJsonObject(first) ? first.text : undefined;
20
22
  return isJsonString(text) ? text.trim() : '';
21
23
  }
24
+ /** The reply's billed tokens, or undefined when its usage block is missing or malformed. */
25
+ function usageOf(reply) {
26
+ const usage = reply.usage;
27
+ if (!isJsonObject(usage))
28
+ return undefined;
29
+ const { input_tokens: inputTokens, output_tokens: outputTokens } = usage;
30
+ if (!Number.isInteger(inputTokens) || !Number.isInteger(outputTokens))
31
+ return undefined;
32
+ return { inputTokens: Number(inputTokens), outputTokens: Number(outputTokens) };
33
+ }
22
34
  /** An Llm that calls the Anthropic Messages API with `apiKey`. */
23
35
  export function anthropicLlm(connection) {
24
36
  return { send: (request) => sendMessage(connection, request) };
@@ -62,7 +74,9 @@ async function sendMessage(connection, req) {
62
74
  };
63
75
  }
64
76
  const text = firstText(data);
65
- return text === '' ? { ok: false, failure: { kind: 'empty' } } : { ok: true, text };
77
+ const usage = usageOf(data);
78
+ const reply = text === '' ? { ok: false, failure: { kind: 'empty' } } : { ok: true, text };
79
+ return usage === undefined ? reply : { ...reply, usage };
66
80
  }
67
81
  catch (err) {
68
82
  return { ok: false, failure: { kind: 'unreadable', message: errorMessage(err), cause: err } };
@@ -2,7 +2,8 @@
2
2
  // the mirror folders.
3
3
  import { withWriteScope } from '../../../db/index.js';
4
4
  import { USER_RULE_TAG } from '../../../base/memory/entry.js';
5
- import { duplicateKey, heldTextKeys, ruleReplaces, storedTextKeys, } from '../../../base/util/same-text.js';
5
+ import { duplicateKey, heldTextKeys, storedTextKeys } from '../../../base/util/same-text.js';
6
+ import { ruleReplaces } from '../../../base/memory/rule-replaces.js';
6
7
  import { CLI_ACTOR } from '../../../base/scope/actor.js';
7
8
  import { supersedeInTx, writeEntryMirrors } from '../entries/entry-writes.js';
8
9
  import { stampOriginProject } from '../entries/entry-row.js';
@@ -39,8 +40,8 @@ function writeCapturedEntries(db, files, entries, opts) {
39
40
  const held = storedTextKeys(rows);
40
41
  return entries.map((entry) => {
41
42
  const stamped = stampOriginProject(files, entry);
42
- // A stated rule replaces an unpinned capture copy of its own project, which would hide it
43
- // from the hooks.
43
+ // A stated rule replaces an unpinned or untagged-pin capture copy of its own project, which
44
+ // would hide it from the hooks.
44
45
  const project = entry.tags.includes(USER_RULE_TAG)
45
46
  ? ruleProjectNames(stamped, opts.dedup)
46
47
  : null;
@@ -300,6 +300,7 @@ export function listTextsHolding(db, { tenantId, texts, project, admit = TEAM_VI
300
300
  source: row.source,
301
301
  originProject: row.origin_project,
302
302
  pinned: row.pinned === 1,
303
+ tags: parseJsonArray(row.tags_json),
303
304
  supersededBy: row.superseded_by,
304
305
  confidence: row.confidence,
305
306
  });
@@ -10,6 +10,7 @@ export type CopyDbRow = {
10
10
  source: string;
11
11
  origin_project: string | null;
12
12
  pinned: number;
13
+ tags_json: string | null;
13
14
  superseded_by: string | null;
14
15
  confidence: string;
15
16
  };
@@ -33,7 +33,7 @@ const MAY_HOLD_COPY_SQL = `(${bareTextSql('content')} IN (SELECT value FROM json
33
33
  function copyChunk(db, read, texts, phrases) {
34
34
  const tenant = tenantAnd(read.tenantId, phrases ? TENANT_IS : 'tenant_id = ?');
35
35
  const bares = JSON.stringify([...new Set(texts.map(bareText))]);
36
- return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project, pinned, superseded_by, confidence ` +
36
+ return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project, pinned, tags_json, superseded_by, confidence ` +
37
37
  `FROM memories ` +
38
38
  `WHERE ${phrases ? IN_PHRASE_HITS : '1'}` +
39
39
  `${tenant.sql}${read.filter.sql} AND ${MAY_HOLD_COPY_SQL}`).all(...(phrases ? [phrases.join(' OR ')] : []), ...tenant.params, ...read.filter.params, bares, bares);
@@ -6,6 +6,7 @@ import { type SessionTurn } from './transcript.js';
6
6
  import type { CaptureDedup } from '../../base/ports/store/capture.js';
7
7
  import type { HeldText } from '../../base/ports/store/types.js';
8
8
  import type { StoreGroups } from '../../base/ports/store.js';
9
+ import type { Llm } from '../../base/ports/llm.js';
9
10
  export interface CaptureOptions {
10
11
  source: 'stdin' | 'file' | 'last-session';
11
12
  filePath?: string;
@@ -43,6 +44,10 @@ export interface CaptureTarget {
43
44
  readonly store: Pick<StoreGroups, 'captures' | 'folderFiles' | 'rejections'>;
44
45
  /** Embeds each captured row, off the capture's path. */
45
46
  readonly embed: (entry: MemoryEntry) => void;
47
+ /** Picks the rules in the person's follow-up turns; without it capture reads word patterns only. */
48
+ readonly llm?: Llm;
49
+ /** Why no `llm` came when the config asked for one; printed inside the capture's log. */
50
+ readonly llmMissing?: string;
46
51
  }
47
52
  export declare function runCapture(target: CaptureTarget, options: CaptureOptions): Promise<void>;
48
53
  /** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
@@ -1,7 +1,8 @@
1
1
  import * as fs from 'node:fs';
2
2
  import * as path from 'node:path';
3
3
  import { createMemory, Layer, USER_RULE_TAG } from '../../base/memory/entry.js';
4
- import { duplicateKey, ruleReplaces, storedTextKeys } from '../../base/util/same-text.js';
4
+ import { duplicateKey, storedTextKeys } from '../../base/util/same-text.js';
5
+ import { ruleReplaces } from '../../base/memory/rule-replaces.js';
5
6
  import { isCertainReason, storedDefect } from '../../base/memory/memory-quality.js';
6
7
  import { maskEmails, redactSecretsStrict } from '../../base/util/secret-detect.js';
7
8
  import { projectId, projectNames } from '../../base/scope/project-identity.js';
@@ -10,6 +11,7 @@ import { errorMessage, log, errorFields } from '../../base/util/log.js';
10
11
  import { teeStdStreams } from '../../host/stream-tee.js';
11
12
  import { ALL_TENANTS } from '../../base/util/tenant.js';
12
13
  import { boldTwins, extractFromTexts, isStatedRule, oncePerText, } from './extract.js';
14
+ import { followUpRules, usageLine, withFollowUpRules } from './follow-up.js';
13
15
  import { collectSessionTurns, sessionTail, resolveLastSessionTranscript, } from './transcript.js';
14
16
  import { PRIVATE_DIR } from '../../host/private-dir.js';
15
17
  import { evalClock } from '../../infra/eval-switches.js';
@@ -67,7 +69,7 @@ async function runCaptureCore(target, options) {
67
69
  const read = readCaptureTexts(options);
68
70
  if (read === null)
69
71
  return;
70
- const { texts, userTexts, firstFollowUp } = read;
72
+ const { texts, userTexts, firstFollowUp, followUps } = read;
71
73
  if (texts.every((text) => text.trim().length === 0)) {
72
74
  console.log('No text to capture from.');
73
75
  return;
@@ -76,15 +78,33 @@ async function runCaptureCore(target, options) {
76
78
  // pass could not see (AGENTS.md: no secrets in memories).
77
79
  const scrub = (text) => maskEmails(redactSecretsStrict(text));
78
80
  const scrubbed = texts.map(scrub);
79
- const extracted = scrubbedItems(extractFromTexts(scrubbed, userTexts, firstFollowUp), scrub);
81
+ const dedup = await captureDedup(target.store.folderFiles, options);
82
+ const rules = await modelRules(target, dedup, followUps);
83
+ const patterned = extractFromTexts(scrubbed, userTexts, firstFollowUp);
84
+ const extracted = scrubbedItems(withFollowUpRules(patterned, rules, scrub), scrub);
80
85
  if (extracted.length === 0) {
81
86
  console.log('No actionable items found in the input.');
82
87
  return;
83
88
  }
84
- const dedup = await captureDedup(target.store.folderFiles, options);
85
89
  const held = await storedKeysInView(target.store, dedup, extracted, boldTwins(scrubbed, scrub));
86
90
  printCaptureTally(options, await captureFromCli(target, options, dedup, extracted, held));
87
91
  }
92
+ const warnLine = (line, opts) => log.warn(line, opts?.fields);
93
+ /** The rules the model finds in the person's later turns, when the target has one. */
94
+ async function modelRules({ llm, llmMissing }, { ruleProject = [] }, followUps) {
95
+ if (!llm) {
96
+ if (llmMissing)
97
+ warnLine(llmMissing);
98
+ return [];
99
+ }
100
+ // A rule with no project to belong to would pin into every project.
101
+ if (ruleProject.length === 0)
102
+ return [];
103
+ const { rules, usage } = await followUpRules(followUps, llm, warnLine);
104
+ if (usage)
105
+ console.log(usageLine(usage));
106
+ return rules;
107
+ }
88
108
  async function captureFromCli({ store, config, embed }, options, dedup, extracted, { keys, ruleKeys }) {
89
109
  const writeOpts = {
90
110
  tenantId: options.global ? undefined : options.tenantId,
@@ -162,11 +182,14 @@ function printCaptureTally(options, { captured, skipped, rejected }) {
162
182
  (rejected > 0 ? `, ${rejected} rejected` : '') +
163
183
  ')');
164
184
  }
185
+ /** The raw texts for the chosen source, one per session turn, the person's turns first; null after printing why a
186
+ * last-session capture has nothing.
187
+ * Only a session says who spoke, so a piped or file capture counts no user texts. */
165
188
  function readCaptureTexts(options) {
166
189
  switch (options.source) {
167
190
  case 'stdin': {
168
191
  try {
169
- return { texts: [fs.readFileSync(0, 'utf8')], userTexts: 0 };
192
+ return { texts: [fs.readFileSync(0, 'utf8')], userTexts: 0, followUps: [] };
170
193
  }
171
194
  catch (err) {
172
195
  log.warn(`reading stdin failed: ${errorMessage(err)}`, errorFields(err));
@@ -186,7 +209,7 @@ function readCaptureTexts(options) {
186
209
  process.exitCode = 1;
187
210
  return null;
188
211
  }
189
- return { texts: [fs.readFileSync(options.filePath, 'utf8')], userTexts: 0 };
212
+ return { texts: [fs.readFileSync(options.filePath, 'utf8')], userTexts: 0, followUps: [] };
190
213
  }
191
214
  case 'last-session':
192
215
  return readSessionTexts(options);
@@ -210,9 +233,14 @@ function readSessionTexts(options) {
210
233
  return null;
211
234
  }
212
235
  // The first user turn is the request; a tail that cut it off, or a later slice, holds only replies.
213
- const allUsers = turns.filter((t) => t.role === 'user').length;
214
- const opener = options.fromStart !== false && users.length === allUsers ? 1 : 0;
215
- return { texts: [...users, ...assistants], userTexts: users.length, firstFollowUp: opener };
236
+ const userTurns = turns.filter((t) => t.role === 'user').map((t) => t.text);
237
+ const opener = options.fromStart !== false && users.length === userTurns.length ? 1 : 0;
238
+ return {
239
+ texts: [...users, ...assistants],
240
+ userTexts: users.length,
241
+ firstFollowUp: opener,
242
+ followUps: userTurns.slice(opener),
243
+ };
216
244
  }
217
245
  /** One entry per new text: building can throw, so a held text or a repeat within the batch is
218
246
  * never built. */
@@ -10,6 +10,22 @@ export declare function isStatedRule(item: ExtractedItem): boolean;
10
10
  /** Each text once, where its first copy stood; a later copy that is the person's rule replaces a
11
11
  * first copy that is not, so a preference restated in a reply keeps its rule status. */
12
12
  export declare function oncePerText(items: readonly ExtractedItem[], admit?: (item: ExtractedItem) => boolean): ExtractedItem[];
13
+ /** Whether a line is program output, not the speaker's words. */
14
+ export type NoiseCheck = (line: string, sentenceOpen: boolean) => boolean;
15
+ /** CommonMark fences: a backtick run with a backtick after it is inline code, and only a bare
16
+ * run as long closes. A line `isNoise` drops also ends the paragraph above it; it is told when
17
+ * the prose above left its sentence open, so it can keep a condition with few letters. */
18
+ export declare function proseOnly(text: string, isNoise?: NoiseCheck): string;
19
+ /** The text above has not ended its sentence; "e.g." at its end has not. */
20
+ export declare function leftOpen(previous: string, next: string): boolean;
21
+ /** A sentence as capture stores it: its lead label and closing marks trimmed. */
22
+ export declare function statedText(sentence: string): string;
23
+ /** The sentences a pattern reads in `text`: wrapped lines joined, bold cut, and none that starts
24
+ * inside a cut aside. */
25
+ export declare function statedSentences(text: string, isNoise?: NoiseCheck): string[];
26
+ /** Picked sentences of one turn through the gates a pattern sentence meets, or null; one sentence
27
+ * gives its pattern item's text, so held rows and bold twins match it. */
28
+ export declare function pickedText(sentences: readonly string[]): string | null;
13
29
  /** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
14
30
  * The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`,
15
31
  * and a copy that is their rule wins over one that is not.
@@ -55,23 +55,30 @@ const CATEGORIES = [
55
55
  const MAX_CHARS = 500;
56
56
  const BULLET = /^(?:[-*]|\d+\.)\s+(?:\[[ xX]\]\s+)?(.+)/;
57
57
  const LABEL = /^(?:decision|rule|error|bug|gotcha|important|critical|remember|plan):/i;
58
+ const CONTINUED_LEAD = /^(?:[a-z]|\((?!\w{1,2}\)))/;
58
59
  const SENTENCE_END = /[.!?]["')\]]*$/;
59
60
  const SPEC_HEADING_PATTERNS = [
60
61
  new RegExp(String.raw `^#+\s*(?:features?|requirements?|specs?|specifications?|plan|design|architecture|interface|api|todo|` +
61
62
  String.raw `tasks?|implementation|notes?)(?:\s|:|$)`, 'i'),
62
63
  /^(?:features?|requirements?|specs?|specifications?|plan|design|tasks?|implementation)(?:\s*:|$)/i,
63
64
  ];
64
- // A list item may open a fence on its own line: "- ```".
65
- const FENCE_OPEN = /^(?:(?:[-*]|\d+\.)\s+)?(`{3,}|~{3,})(.*)$/;
65
+ // A list item may open a fence on its own line: "- ```", "+ ```" or "1) ```" (CommonMark markers).
66
+ const FENCE_OPEN = /^(?:(?:[-*+]|\d+[.)])\s+)?(`{3,}|~{3,})(.*)$/;
67
+ const INDENTED_CODE = /^(?: {4}|\t)(?!\s*(?:[-*+]|\d+[.)])\s)/;
66
68
  const FENCE_CLOSE = /^(`{3,}|~{3,})\s*$/;
67
69
  // Inline code and abbreviation dots never end a sentence; "etc." does when a capital follows.
68
70
  const NO_BREAK = /(`+)[^`\n]+\1|\b(?:[Ee]\.g|[Ii]\.e|[Vv]s|[Cc]f)\.|\betc\.(?=\s+[a-z])/g;
71
+ // Pasted text is not the speaker's own; an unclosed block runs to the end.
72
+ const PASTED_BLOCK = /<pasted_content\b[^>]*>[\s\S]*?(?:<\/pasted_content>|$)/g;
69
73
  /** CommonMark fences: a backtick run with a backtick after it is inline code, and only a bare
70
- * run as long closes. */
71
- function proseOnly(text) {
74
+ * run as long closes. A line `isNoise` drops also ends the paragraph above it; it is told when
75
+ * the prose above left its sentence open, so it can keep a condition with few letters. */
76
+ export function proseOnly(text, isNoise = () => false) {
72
77
  let fence = null;
73
78
  let afterProse = false;
79
+ let previous;
74
80
  return text
81
+ .replace(PASTED_BLOCK, '\n')
75
82
  .split('\n')
76
83
  .map((line) => {
77
84
  const trimmed = line.trimStart();
@@ -90,8 +97,16 @@ function proseOnly(text) {
90
97
  }
91
98
  // Quotes, table rows and indented code are not the speaker's own; indented code cannot
92
99
  // interrupt a paragraph or list item.
93
- const dropped = /^\s*[>|]/.test(line) || (!afterProse && /^(?: {4}|\t)(?!\s*(?:[-*]|\d+\.)\s)/.test(line));
100
+ // A colon introduces what follows, so output shown after one stays noise.
101
+ const sentenceOpen = afterProse &&
102
+ previous !== undefined &&
103
+ !previous.endsWith(':') &&
104
+ leftOpen(previous, trimmed);
105
+ const dropped = isNoise(line, sentenceOpen) ||
106
+ /^\s*[>|]/.test(line) ||
107
+ (!afterProse && INDENTED_CODE.test(line));
94
108
  afterProse = !dropped && trimmed !== '' && !/^#{1,6}(?:\s|$)/.test(trimmed);
109
+ previous = trimmed.trimEnd();
95
110
  return dropped ? '' : line;
96
111
  })
97
112
  .join('\n');
@@ -99,14 +114,16 @@ function proseOnly(text) {
99
114
  function masked(text) {
100
115
  return text.replace(NO_BREAK, (span) => 'x'.repeat(span.length));
101
116
  }
102
- /**
103
- * A lowercase line continues the line above only when that line has not ended its sentence; "e.g."
104
- * at a line end has not.
105
- */
117
+ /** The text above has not ended its sentence; "e.g." at its end has not. */
118
+ export function leftOpen(previous, next) {
119
+ return !SENTENCE_END.test(masked(`${previous} ${next}`).slice(0, previous.length));
120
+ }
121
+ /** A lowercase or bracketed line, but not an enumerator like "(1)", continues the line above
122
+ * only when that line left its sentence open. */
106
123
  function continues(line, previous) {
107
- if (previous === undefined || !/^[a-z]/.test(line) || LABEL.test(line))
124
+ if (previous === undefined || !CONTINUED_LEAD.test(line) || LABEL.test(line))
108
125
  return false;
109
- return !SENTENCE_END.test(masked(`${previous} ${line}`).slice(0, previous.length));
126
+ return leftOpen(previous, line);
110
127
  }
111
128
  /** What capture read before it removed bold markers. */
112
129
  const keepAll = () => [];
@@ -251,12 +268,36 @@ function extractFromPatterns(sentence, patterns, category) {
251
268
  const keyed = patterns.some((pattern) => [...sentence.matchAll(new RegExp(pattern.source, `${pattern.flags}g`))].some((match) => !insideBrackets(sentence.slice(0, match.index))));
252
269
  if (!keyed)
253
270
  return null;
254
- const content = sentence
255
- .replace(new RegExp(String.raw `^(?:decision|plan|rule|important|critical|remember|error|bug|gotcha|watch out|careful|warning|` +
256
- String.raw `caveat|trap|don't forget|easy to miss):\s*`, 'i'), '')
271
+ const content = statedText(sentence);
272
+ return content.length <= MAX_CHARS ? { content, category, tags: [category, 'captured'] } : null;
273
+ }
274
+ const LEAD_LABEL = new RegExp(String.raw `^(?:decision|plan|rule|important|critical|remember|error|bug|gotcha|watch out|careful|warning|` +
275
+ String.raw `caveat|trap|don't forget|easy to miss):\s*`, 'i');
276
+ /** A sentence as capture stores it: its lead label and closing marks trimmed. */
277
+ export function statedText(sentence) {
278
+ return sentence
279
+ .replace(LEAD_LABEL, '')
257
280
  .replace(/[.!?\s]+$/, '')
258
281
  .trim();
259
- return content.length <= MAX_CHARS ? { content, category, tags: [category, 'captured'] } : null;
282
+ }
283
+ /** The sentences a pattern reads in `text`: wrapped lines joined, bold cut, and none that starts
284
+ * inside a cut aside. */
285
+ export function statedSentences(text, isNoise) {
286
+ return proseBlocks(proseOnly(text, isNoise), boldPairs).flatMap((block) => {
287
+ const cutByAside = asideCutter(block.text);
288
+ return completeSentences(block)
289
+ .filter(({ sentence, at }) => !cutByAside(sentence, at))
290
+ .map(({ sentence }) => sentence);
291
+ });
292
+ }
293
+ /** Picked sentences of one turn through the gates a pattern sentence meets, or null; one sentence
294
+ * gives its pattern item's text, so held rows and bold twins match it. */
295
+ export function pickedText(sentences) {
296
+ const kept = sentences.filter((sentence) => !sentence.endsWith('?'));
297
+ if (kept.length === 0)
298
+ return null;
299
+ const content = statedText(kept.join(' '));
300
+ return content.length <= MAX_CHARS && assessAutomaticMemory(content).accepted ? content : null;
260
301
  }
261
302
  function extractSpecSections(prose, cuts) {
262
303
  const bullets = [];
@@ -0,0 +1,21 @@
1
+ import { type Llm, type LlmUsage } from '../../base/ports/llm.js';
2
+ import { type LineLog } from '../../base/util/log.js';
3
+ import { type ExtractedItem } from './extract.js';
4
+ /** A rule the model picked, and the whole sentences it came from. */
5
+ export interface FollowUpRule extends ExtractedItem {
6
+ readonly sentences: readonly string[];
7
+ }
8
+ export interface FollowUpRules {
9
+ readonly rules: FollowUpRule[];
10
+ /** What the model call was billed; undefined when no call ran or the reply named none. */
11
+ readonly usage: LlmUsage | undefined;
12
+ }
13
+ /** The rules the model finds in the person's turns after the session's first, the last 20 of
14
+ * them; a failed call or an unreadable reply logs one line and gives none. Never throws. */
15
+ export declare function followUpRules(turns: readonly string[], llm: Llm, log: LineLog): Promise<FollowUpRules>;
16
+ /** One line per model call, in key=value form, so a harness can price it from the capture log. */
17
+ export declare function usageLine(usage: LlmUsage): string;
18
+ /** The pattern items but each one that is a whole sentence of a rule, then the rules; both read
19
+ * after `scrub`, so the comparison sees what is stored. */
20
+ export declare function withFollowUpRules(items: readonly ExtractedItem[], rules: readonly FollowUpRule[], scrub: (text: string) => string): ExtractedItem[];
21
+ //# sourceMappingURL=follow-up.d.ts.map