hippo-memory 3.2.0 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/base/ports/store/capture.d.ts +3 -0
- package/dist/base/ports/store/types.d.ts +3 -1
- package/dist/base/util/same-text.d.ts +7 -0
- package/dist/base/util/same-text.js +12 -0
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/store/sqlite/capture/captures.js +33 -8
- package/dist/store/sqlite/entries/entry-writes.d.ts +5 -0
- package/dist/store/sqlite/entries/entry-writes.js +13 -0
- package/dist/store/sqlite/recall/candidates.js +3 -0
- package/dist/store/sqlite/sharing/copy-rows.d.ts +2 -0
- package/dist/store/sqlite/sharing/copy-rows.js +2 -1
- package/dist/surface/capture/command.d.ts +6 -2
- package/dist/surface/capture/command.js +57 -39
- package/dist/surface/capture/extract.d.ts +7 -1
- package/dist/surface/capture/extract.js +26 -11
- package/dist-ui/assets/{index-DWo8WuPW.js → index-DvZmCHQ5.js} +2 -2
- package/dist-ui/assets/{react-BZPdts19.js → react-BpaniaUk.js} +2 -2
- package/dist-ui/index.html +2 -2
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -2
|
@@ -38,6 +38,9 @@ export interface CaptureDedup {
|
|
|
38
38
|
readonly tenantId: TenantId;
|
|
39
39
|
readonly project?: readonly string[];
|
|
40
40
|
readonly everyScope?: boolean;
|
|
41
|
+
/** The names of the project a stated rule is written for: a held copy of one of them gives way
|
|
42
|
+
* to the rule. None replaces nothing. */
|
|
43
|
+
readonly ruleProject?: readonly string[];
|
|
41
44
|
}
|
|
42
45
|
/** Each method opens hippo.db with the hook's wait, so a busy store costs the hook a bounded
|
|
43
46
|
* delay. */
|
|
@@ -113,7 +113,9 @@ export interface AuditEvent {
|
|
|
113
113
|
metadata: JsonObject;
|
|
114
114
|
}
|
|
115
115
|
/** A stored text the capture dedup reads; `pinned` is optional so an adapter that omits it still fits. */
|
|
116
|
-
export type HeldText = Pick<MemoryEntry, 'content' | 'source' | 'originProject'> & Partial<Pick<MemoryEntry, 'pinned'
|
|
116
|
+
export type HeldText = Pick<MemoryEntry, 'content' | 'source' | 'originProject'> & Partial<Pick<MemoryEntry, 'pinned' | 'id' | 'supersededBy'>> & {
|
|
117
|
+
confidence?: string;
|
|
118
|
+
};
|
|
117
119
|
/** One project name, or every name a project's rows carry. */
|
|
118
120
|
export type OriginFilter = string | readonly string[];
|
|
119
121
|
/** The api_keys fields a tenant, revocation, rank or self-service ownership check reads. */
|
|
@@ -15,6 +15,13 @@ export declare function mergedText(header: string, texts: readonly string[]): st
|
|
|
15
15
|
* whole after its header.
|
|
16
16
|
*/
|
|
17
17
|
export declare function heldTexts(entry: Text): string[];
|
|
18
|
+
/** Whether a stated rule replaces this held copy rather than giving way to it: a live, unpinned,
|
|
19
|
+
* observed capture row of one of `ruleProject`, the names of the rule's project. A superseded,
|
|
20
|
+
* pinned, typed, verified, no-project or other-project copy stops the rule, so a replayed
|
|
21
|
+
* transcript revives nothing and no project loses its row. */
|
|
22
|
+
export declare function ruleReplaces(row: Text & Partial<Pick<MemoryEntry, 'originProject' | 'supersededBy'>> & {
|
|
23
|
+
confidence?: string;
|
|
24
|
+
}, ruleProject: readonly string[]): boolean;
|
|
18
25
|
/** Keys of every text a row holds word for word: its own, plus each source text inside a
|
|
19
26
|
* sleep-merged row. */
|
|
20
27
|
export declare function heldTextKeys(entry: Text): string[];
|
|
@@ -38,6 +38,18 @@ export function heldTexts(entry) {
|
|
|
38
38
|
.slice(1)
|
|
39
39
|
.map((t) => t.replace(/\n {2}/g, '\n'));
|
|
40
40
|
}
|
|
41
|
+
/** Whether a stated rule replaces this held copy rather than giving way to it: a live, unpinned,
|
|
42
|
+
* observed capture row of one of `ruleProject`, the names of the rule's project. A superseded,
|
|
43
|
+
* pinned, typed, verified, no-project or other-project copy stops the rule, so a replayed
|
|
44
|
+
* transcript revives nothing and no project loses its row. */
|
|
45
|
+
export function ruleReplaces(row, ruleProject) {
|
|
46
|
+
return (row.pinned !== true &&
|
|
47
|
+
row.source === 'capture' &&
|
|
48
|
+
Boolean(row.originProject) &&
|
|
49
|
+
ruleProject.includes(row.originProject ?? '') &&
|
|
50
|
+
row.supersededBy == null &&
|
|
51
|
+
row.confidence === 'observed');
|
|
52
|
+
}
|
|
41
53
|
/** Keys of every text a row holds word for word: its own, plus each source text inside a
|
|
42
54
|
* sleep-merged row. */
|
|
43
55
|
export function heldTextKeys(entry) {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export declare const PACKAGE_VERSION = "3.2.
|
|
5
|
+
export declare const PACKAGE_VERSION = "3.2.1";
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export declare const EXPIRING_KEYS_MIN_BINARY = "1.64.0";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export const PACKAGE_VERSION = '3.2.
|
|
5
|
+
export const PACKAGE_VERSION = '3.2.1';
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export const EXPIRING_KEYS_MIN_BINARY = '1.64.0';
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
// hippo.db's Captures: the whole batch on one handle, opened with openStore since a write sets up
|
|
2
2
|
// the mirror folders.
|
|
3
3
|
import { withWriteScope } from '../../../db/index.js';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { USER_RULE_TAG } from '../../../base/memory/entry.js';
|
|
5
|
+
import { duplicateKey, heldTextKeys, ruleReplaces, storedTextKeys, } from '../../../base/util/same-text.js';
|
|
6
|
+
import { CLI_ACTOR } from '../../../base/scope/actor.js';
|
|
7
|
+
import { supersedeInTx, writeEntryMirrors } from '../entries/entry-writes.js';
|
|
6
8
|
import { stampOriginProject } from '../entries/entry-row.js';
|
|
7
9
|
import { gatedWrite } from './gated-write.js';
|
|
8
10
|
import { updateStatsOn } from '../entries/index-and-stats.js';
|
|
@@ -10,29 +12,52 @@ import { listTextsHolding } from '../recall/candidates.js';
|
|
|
10
12
|
import { EVERY_SCOPE } from '../runtime/rule-sql.js';
|
|
11
13
|
import { openStore } from '../runtime/handles.js';
|
|
12
14
|
import { bound, skipEmpty } from '../runtime/bound.js';
|
|
13
|
-
/** The
|
|
14
|
-
function
|
|
15
|
+
/** The rows under `dedup` that may hold an entry's text. */
|
|
16
|
+
function heldRows(db, entries, dedup) {
|
|
15
17
|
const texts = entries.map((entry) => entry.content);
|
|
16
18
|
const admit = dedup.everyScope ? EVERY_SCOPE : undefined;
|
|
17
|
-
return
|
|
19
|
+
return listTextsHolding(db, { tenantId: dedup.tenantId, texts, project: dedup.project, admit });
|
|
20
|
+
}
|
|
21
|
+
/** The caller's rule project names when the stamped rule carries one of them, else none. */
|
|
22
|
+
function ruleProjectNames(rule, dedup) {
|
|
23
|
+
const origin = rule.originProject;
|
|
24
|
+
return origin && dedup.ruleProject?.includes(origin) ? dedup.ruleProject : [];
|
|
25
|
+
}
|
|
26
|
+
/** The ids of the held copies of the stamped rule's text that the rule replaces. */
|
|
27
|
+
function copiesTheRuleReplaces(rows, rule, project) {
|
|
28
|
+
const key = duplicateKey(rule.content);
|
|
29
|
+
return rows.flatMap(({ id, ...row }) => id !== undefined && ruleReplaces(row, project) && heldTextKeys(row).includes(key) ? [id] : []);
|
|
18
30
|
}
|
|
19
31
|
/** One outcome per entry, in order, in one transaction that reads the caller's dedup again
|
|
20
32
|
* first, so a text another writer stored since that read is skipped; mirrors and stats follow
|
|
21
33
|
* the commit. */
|
|
22
34
|
function writeCapturedEntries(db, files, entries, opts) {
|
|
23
35
|
const written = [];
|
|
36
|
+
const actor = opts.actor ?? CLI_ACTOR;
|
|
24
37
|
const outcomes = withWriteScope(db, 'capture_write', () => {
|
|
25
|
-
const
|
|
38
|
+
const rows = heldRows(db, entries, opts.dedup);
|
|
39
|
+
const held = storedTextKeys(rows);
|
|
26
40
|
return entries.map((entry) => {
|
|
27
|
-
|
|
41
|
+
const stamped = stampOriginProject(files, entry);
|
|
42
|
+
// A stated rule replaces an unpinned capture copy of its own project, which would hide it
|
|
43
|
+
// from the hooks.
|
|
44
|
+
const project = entry.tags.includes(USER_RULE_TAG)
|
|
45
|
+
? ruleProjectNames(stamped, opts.dedup)
|
|
46
|
+
: null;
|
|
47
|
+
const stops = project ? storedTextKeys(rows.filter((r) => !ruleReplaces(r, project))) : held;
|
|
48
|
+
if (stops.has(duplicateKey(entry.content)))
|
|
28
49
|
return 'skipped';
|
|
29
50
|
// One rejected entry must not abort the rest of this capture's entries.
|
|
30
|
-
const stamped = stampOriginProject(files, entry);
|
|
31
51
|
const outcome = gatedWrite(db, files, stamped, { actor: opts.actor });
|
|
32
52
|
if (outcome === 'skipped:rejected')
|
|
33
53
|
return 'rejected';
|
|
34
54
|
if (outcome !== 'written')
|
|
35
55
|
return 'skipped';
|
|
56
|
+
if (project) {
|
|
57
|
+
for (const id of copiesTheRuleReplaces(rows, stamped, project)) {
|
|
58
|
+
supersedeInTx(db, { tenantId: stamped.tenantId, id, newId: stamped.id, actor });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
36
61
|
written.push(stamped);
|
|
37
62
|
return 'captured';
|
|
38
63
|
});
|
|
@@ -106,6 +106,11 @@ interface SupersedeEntryOptions {
|
|
|
106
106
|
}
|
|
107
107
|
/** Marks `id` superseded by `newId`; false when another writer superseded it first. */
|
|
108
108
|
export declare function supersedeEntry(db: DatabaseSyncLike, { tenantId, id, newId }: SupersedeEntryOptions): boolean;
|
|
109
|
+
/** Marks `id` superseded by `newId` in the caller's transaction, with the supersede audit row and
|
|
110
|
+
* its summary parent's dirty mark; false when the row is gone or another writer got there first. */
|
|
111
|
+
export declare function supersedeInTx(db: DatabaseSyncLike, { tenantId, id, newId, actor }: SupersedeEntryOptions & {
|
|
112
|
+
actor: string;
|
|
113
|
+
}): boolean;
|
|
109
114
|
/** Each call on its own handle; mirrors and the forgotten counter follow the commit, so a
|
|
110
115
|
* rolled-back write leaves neither. */
|
|
111
116
|
export declare function sqliteEntryWrites({ on, files }: Handles): EntryWrites;
|
|
@@ -294,6 +294,19 @@ export function supersedeEntry(db, { tenantId, id, newId }) {
|
|
|
294
294
|
const result = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND superseded_by IS NULL`).run(newId, id, tenantId);
|
|
295
295
|
return Number(result.changes ?? 0) !== 0;
|
|
296
296
|
}
|
|
297
|
+
/** Marks `id` superseded by `newId` in the caller's transaction, with the supersede audit row and
|
|
298
|
+
* its summary parent's dirty mark; false when the row is gone or another writer got there first. */
|
|
299
|
+
export function supersedeInTx(db, { tenantId, id, newId, actor }) {
|
|
300
|
+
const won = cachedStatement(db, 'UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND superseded_by IS NULL ' +
|
|
301
|
+
'RETURNING dag_parent_id').get(newId, id, tenantId);
|
|
302
|
+
if (!won)
|
|
303
|
+
return false;
|
|
304
|
+
if (won.dag_parent_id) {
|
|
305
|
+
markSummaryDirtyInTx(db, { summaryId: won.dag_parent_id, tenantId, actor });
|
|
306
|
+
}
|
|
307
|
+
appendAuditEvents(db, [{ tenantId, actor, op: 'supersede', targetId: id, metadata: { newId } }]);
|
|
308
|
+
return true;
|
|
309
|
+
}
|
|
297
310
|
/** Each call on its own handle; mirrors and the forgotten counter follow the commit, so a
|
|
298
311
|
* rolled-back write leaves neither. */
|
|
299
312
|
export function sqliteEntryWrites({ on, files }) {
|
|
@@ -295,10 +295,13 @@ export function listTextsHolding(db, { tenantId, texts, project, admit = TEAM_VI
|
|
|
295
295
|
const byId = new Map();
|
|
296
296
|
for (const row of rowsHoldingTexts(db, { tenantId, filter }, texts)) {
|
|
297
297
|
byId.set(row.id, {
|
|
298
|
+
id: row.id,
|
|
298
299
|
content: row.content,
|
|
299
300
|
source: row.source,
|
|
300
301
|
originProject: row.origin_project,
|
|
301
302
|
pinned: row.pinned === 1,
|
|
303
|
+
supersededBy: row.superseded_by,
|
|
304
|
+
confidence: row.confidence,
|
|
302
305
|
});
|
|
303
306
|
}
|
|
304
307
|
return [...byId.values()];
|
|
@@ -10,6 +10,8 @@ export type CopyDbRow = {
|
|
|
10
10
|
source: string;
|
|
11
11
|
origin_project: string | null;
|
|
12
12
|
pinned: number;
|
|
13
|
+
superseded_by: string | null;
|
|
14
|
+
confidence: string;
|
|
13
15
|
};
|
|
14
16
|
/** Which rows of the tenant a copy read looks at: `filter` adds predicates, each written ` AND
|
|
15
17
|
* ...`. */
|
|
@@ -33,7 +33,8 @@ const MAY_HOLD_COPY_SQL = `(${bareTextSql('content')} IN (SELECT value FROM json
|
|
|
33
33
|
function copyChunk(db, read, texts, phrases) {
|
|
34
34
|
const tenant = tenantAnd(read.tenantId, phrases ? TENANT_IS : 'tenant_id = ?');
|
|
35
35
|
const bares = JSON.stringify([...new Set(texts.map(bareText))]);
|
|
36
|
-
return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project, pinned
|
|
36
|
+
return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project, pinned, superseded_by, confidence ` +
|
|
37
|
+
`FROM memories ` +
|
|
37
38
|
`WHERE ${phrases ? IN_PHRASE_HITS : '1'}` +
|
|
38
39
|
`${tenant.sql}${read.filter.sql} AND ${MAY_HOLD_COPY_SQL}`).all(...(phrases ? [phrases.join(' OR ')] : []), ...tenant.params, ...read.filter.params, bares, bares);
|
|
39
40
|
}
|
|
@@ -46,11 +46,15 @@ export interface CaptureTarget {
|
|
|
46
46
|
}
|
|
47
47
|
export declare function runCapture(target: CaptureTarget, options: CaptureOptions): Promise<void>;
|
|
48
48
|
/** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
|
|
49
|
-
* items one, and the first stays. */
|
|
49
|
+
* items one, and the first stays unless a later one is the person's rule. */
|
|
50
50
|
export declare function scrubbedItems(items: readonly ExtractedItem[], scrub: (text: string) => string): ExtractedItem[];
|
|
51
51
|
/** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
|
|
52
52
|
* row: a pinned row, or one whose text is no certain defect. */
|
|
53
53
|
export declare function heldKeysOf(rows: readonly HeldText[], twins: readonly BoldTwin[]): Set<string>;
|
|
54
|
+
/** The held keys that also stop a stated rule written for `ruleProject` (its names): every copy
|
|
55
|
+
* but one the store's write replaces, which holds the rule's exact text, so a bold twin still
|
|
56
|
+
* stops it. */
|
|
57
|
+
export declare function ruleHeldKeysOf(rows: readonly HeldText[], twins: readonly BoldTwin[], ruleProject: readonly string[]): Set<string>;
|
|
54
58
|
export interface CaptureTally {
|
|
55
59
|
captured: number;
|
|
56
60
|
skipped: number;
|
|
@@ -80,5 +84,5 @@ export interface ConfiguredStore {
|
|
|
80
84
|
}
|
|
81
85
|
/** Writes each new text once; a repeat of a captured text in the same batch counts as skipped,
|
|
82
86
|
* as a held one does. */
|
|
83
|
-
export declare function captureExtractedItems({ store, config, embed }: ConfiguredStore, options: CaptureWriteOptions, extracted: readonly ExtractedItem[], keys: Set<string>): Promise<CaptureTally>;
|
|
87
|
+
export declare function captureExtractedItems({ store, config, embed }: ConfiguredStore, options: CaptureWriteOptions, extracted: readonly ExtractedItem[], keys: Set<string>, ruleKeys?: ReadonlySet<string>): Promise<CaptureTally>;
|
|
84
88
|
//# sourceMappingURL=command.d.ts.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import { createMemory, Layer, USER_RULE_TAG } from '../../base/memory/entry.js';
|
|
4
|
-
import { duplicateKey, storedTextKeys } from '../../base/util/same-text.js';
|
|
4
|
+
import { duplicateKey, ruleReplaces, storedTextKeys } from '../../base/util/same-text.js';
|
|
5
5
|
import { isCertainReason, storedDefect } from '../../base/memory/memory-quality.js';
|
|
6
6
|
import { maskEmails, redactSecretsStrict } from '../../base/util/secret-detect.js';
|
|
7
7
|
import { projectId, projectNames } from '../../base/scope/project-identity.js';
|
|
@@ -9,7 +9,7 @@ import { isInitialized } from '../../host/project-identity.js';
|
|
|
9
9
|
import { errorMessage, log, errorFields } from '../../base/util/log.js';
|
|
10
10
|
import { teeStdStreams } from '../../host/stream-tee.js';
|
|
11
11
|
import { ALL_TENANTS } from '../../base/util/tenant.js';
|
|
12
|
-
import { boldTwins, extractFromTexts } from './extract.js';
|
|
12
|
+
import { boldTwins, extractFromTexts, isStatedRule, oncePerText, } from './extract.js';
|
|
13
13
|
import { collectSessionTurns, sessionTail, resolveLastSessionTranscript, } from './transcript.js';
|
|
14
14
|
import { PRIVATE_DIR } from '../../host/private-dir.js';
|
|
15
15
|
import { evalClock } from '../../infra/eval-switches.js';
|
|
@@ -81,66 +81,79 @@ async function runCaptureCore(target, options) {
|
|
|
81
81
|
console.log('No actionable items found in the input.');
|
|
82
82
|
return;
|
|
83
83
|
}
|
|
84
|
-
const
|
|
85
|
-
|
|
84
|
+
const dedup = await captureDedup(target.store.folderFiles, options);
|
|
85
|
+
const held = await storedKeysInView(target.store, dedup, extracted, boldTwins(scrubbed, scrub));
|
|
86
|
+
printCaptureTally(options, await captureFromCli(target, options, dedup, extracted, held));
|
|
86
87
|
}
|
|
87
|
-
async function captureFromCli({ store, config, embed }, options, extracted, keys) {
|
|
88
|
+
async function captureFromCli({ store, config, embed }, options, dedup, extracted, { keys, ruleKeys }) {
|
|
88
89
|
const writeOpts = {
|
|
89
90
|
tenantId: options.global ? undefined : options.tenantId,
|
|
90
91
|
originProject: options.originProject,
|
|
91
92
|
lean: false,
|
|
92
|
-
dedup
|
|
93
|
+
dedup,
|
|
93
94
|
now: evalClock()(),
|
|
94
95
|
};
|
|
95
96
|
if (!options.dryRun) {
|
|
96
|
-
return captureExtractedItems({ store, config, embed }, writeOpts, extracted, keys);
|
|
97
|
+
return captureExtractedItems({ store, config, embed }, writeOpts, extracted, keys, ruleKeys);
|
|
97
98
|
}
|
|
98
|
-
return previewCapture({ store, config }, writeOpts, extracted, keys);
|
|
99
|
+
return previewCapture({ store, config }, writeOpts, extracted, keys, ruleKeys);
|
|
99
100
|
}
|
|
100
101
|
/** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
|
|
101
|
-
* items one, and the first stays. */
|
|
102
|
+
* items one, and the first stays unless a later one is the person's rule. */
|
|
102
103
|
export function scrubbedItems(items, scrub) {
|
|
103
|
-
|
|
104
|
-
return items
|
|
105
|
-
.map((item) => ({ ...item, content: scrub(item.content) }))
|
|
106
|
-
.filter((item) => {
|
|
107
|
-
const key = duplicateKey(item.content);
|
|
108
|
-
if (seen.has(key))
|
|
109
|
-
return false;
|
|
110
|
-
seen.add(key);
|
|
111
|
-
return true;
|
|
112
|
-
});
|
|
104
|
+
return oncePerText(items.map((item) => ({ ...item, content: scrub(item.content) })));
|
|
113
105
|
}
|
|
114
106
|
/** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
|
|
115
107
|
* row: a pinned row, or one whose text is no certain defect. */
|
|
116
108
|
export function heldKeysOf(rows, twins) {
|
|
109
|
+
const keys = storedTextKeys(rows);
|
|
110
|
+
for (const key of twinKeys(rows, twins))
|
|
111
|
+
keys.add(key);
|
|
112
|
+
return keys;
|
|
113
|
+
}
|
|
114
|
+
/** The held keys that also stop a stated rule written for `ruleProject` (its names): every copy
|
|
115
|
+
* but one the store's write replaces, which holds the rule's exact text, so a bold twin still
|
|
116
|
+
* stops it. */
|
|
117
|
+
export function ruleHeldKeysOf(rows, twins, ruleProject) {
|
|
118
|
+
const keys = storedTextKeys(rows.filter((row) => !ruleReplaces(row, ruleProject)));
|
|
119
|
+
for (const key of twinKeys(rows, twins))
|
|
120
|
+
keys.add(key);
|
|
121
|
+
return keys;
|
|
122
|
+
}
|
|
123
|
+
function twinKeys(rows, twins) {
|
|
117
124
|
const keys = storedTextKeys(rows);
|
|
118
125
|
const pinned = storedTextKeys(rows.filter((row) => row.pinned));
|
|
119
|
-
|
|
126
|
+
return twins
|
|
120
127
|
.filter(({ old }) => (isCertainReason(storedDefect(old)) ? pinned : keys).has(duplicateKey(old)))
|
|
121
128
|
.map((twin) => duplicateKey(twin.clean));
|
|
122
|
-
for (const key of clean)
|
|
123
|
-
keys.add(key);
|
|
124
|
-
return keys;
|
|
125
129
|
}
|
|
126
130
|
// Dedup only against rows this capture's reader sees: another tenant's or project's rows are
|
|
127
131
|
// hidden, so they must not stop its own copy.
|
|
128
|
-
async function storedKeysInView(store,
|
|
129
|
-
const { tenantId, ...filter } = captureDedup(options);
|
|
132
|
+
async function storedKeysInView(store, { tenantId, project, everyScope, ruleProject = [] }, extracted, twins) {
|
|
130
133
|
// The lookup matches stored text exactly, so it also asks for the lines as capture once stored them.
|
|
131
134
|
const texts = [...extracted.map((item) => item.content), ...twins.map((twin) => twin.old)];
|
|
132
|
-
|
|
135
|
+
const rows = await store.captures.textsHolding(tenantId, texts, { project, everyScope });
|
|
136
|
+
return { keys: heldKeysOf(rows, twins), ruleKeys: ruleHeldKeysOf(rows, twins, ruleProject) };
|
|
133
137
|
}
|
|
134
138
|
/** The rows a capture's dedup reads, as data, so the store's write can read them again in its
|
|
135
|
-
* own transaction. */
|
|
136
|
-
function captureDedup(options) {
|
|
139
|
+
* own transaction, and the project a stated rule is written for. */
|
|
140
|
+
async function captureDedup(folderFiles, options) {
|
|
137
141
|
const tenantId = options.global ? ALL_TENANTS : options.tenantId;
|
|
138
142
|
const origin = options.originProject;
|
|
139
143
|
// classifyOriginProject in SQL: a project with no id sees every row, any other its own names and
|
|
140
144
|
// user-global rows.
|
|
141
|
-
if (origin
|
|
142
|
-
|
|
143
|
-
|
|
145
|
+
if (origin !== undefined && projectId(origin) !== '') {
|
|
146
|
+
const names = projectNames(origin);
|
|
147
|
+
return { tenantId, everyScope: true, project: names, ruleProject: names };
|
|
148
|
+
}
|
|
149
|
+
const ruleProject = origin === undefined ? await stampedProject(folderFiles) : [];
|
|
150
|
+
return { tenantId, everyScope: true, ruleProject };
|
|
151
|
+
}
|
|
152
|
+
/** The names of the project the store stamps on a row that names none: none for a shared or a
|
|
153
|
+
* user-global store. */
|
|
154
|
+
async function stampedProject(folderFiles) {
|
|
155
|
+
const { shared, project } = await folderFiles.home();
|
|
156
|
+
return shared || projectId(project) === '' ? [] : projectNames(project);
|
|
144
157
|
}
|
|
145
158
|
function printCaptureTally(options, { captured, skipped, rejected }) {
|
|
146
159
|
const prefix = options.dryRun ? '[dry-run] ' : '';
|
|
@@ -203,12 +216,12 @@ function readSessionTexts(options) {
|
|
|
203
216
|
}
|
|
204
217
|
/** One entry per new text: building can throw, so a held text or a repeat within the batch is
|
|
205
218
|
* never built. */
|
|
206
|
-
function planCapture(extracted,
|
|
219
|
+
function planCapture(extracted, isHeld, build) {
|
|
207
220
|
const plan = { entries: [], keys: [], slots: [] };
|
|
208
221
|
const firstSlot = new Map();
|
|
209
222
|
for (const item of extracted) {
|
|
210
223
|
const key = duplicateKey(item.content);
|
|
211
|
-
let slot =
|
|
224
|
+
let slot = isHeld(item, key) ? -1 : firstSlot.get(key);
|
|
212
225
|
if (slot === undefined) {
|
|
213
226
|
slot = plan.entries.push(build(item)) - 1;
|
|
214
227
|
plan.keys.push(key);
|
|
@@ -218,10 +231,14 @@ function planCapture(extracted, held, build) {
|
|
|
218
231
|
}
|
|
219
232
|
return plan;
|
|
220
233
|
}
|
|
234
|
+
/** Whether a held text stops `item`: any copy stops a note, only a `ruleKeys` copy a stated rule. */
|
|
235
|
+
function heldCheck(keys, ruleKeys) {
|
|
236
|
+
return (item, key) => (isStatedRule(item) ? ruleKeys : keys).has(key);
|
|
237
|
+
}
|
|
221
238
|
/** Writes each new text once; a repeat of a captured text in the same batch counts as skipped,
|
|
222
239
|
* as a held one does. */
|
|
223
|
-
export async function captureExtractedItems({ store, config, embed }, options, extracted, keys) {
|
|
224
|
-
const plan = planCapture(extracted, keys, (item) => captureEntry(item, options, config.defaultHalfLifeDays));
|
|
240
|
+
export async function captureExtractedItems({ store, config, embed }, options, extracted, keys, ruleKeys = keys) {
|
|
241
|
+
const plan = planCapture(extracted, heldCheck(keys, ruleKeys), (item) => captureEntry(item, options, config.defaultHalfLifeDays));
|
|
225
242
|
const { actor, lean, dedup } = options;
|
|
226
243
|
const outcomes = await store.captures.write(plan.entries, { actor, lean, dedup });
|
|
227
244
|
const tally = { captured: 0, skipped: 0, rejected: 0 };
|
|
@@ -242,16 +259,17 @@ export async function captureExtractedItems({ store, config, embed }, options, e
|
|
|
242
259
|
}
|
|
243
260
|
/** A dry run asks the store the same rejected-value question the write's gate does, so a
|
|
244
261
|
* tombstoned item previews as rejected rather than captured. */
|
|
245
|
-
async function previewCapture({ store: { rejections }, config }, options, extracted, keys) {
|
|
262
|
+
async function previewCapture({ store: { rejections }, config }, options, extracted, keys, ruleKeys) {
|
|
246
263
|
const tally = { captured: 0, skipped: 0, rejected: 0 };
|
|
247
|
-
const
|
|
264
|
+
const held = heldCheck(keys, ruleKeys);
|
|
265
|
+
const fresh = extracted.filter((item) => !held(item, duplicateKey(item.content)));
|
|
248
266
|
const refused = await rejections.refusedAmong(fresh.map((item) => {
|
|
249
267
|
const entry = captureEntry(item, options, config.defaultHalfLifeDays);
|
|
250
268
|
return { tenantId: entry.tenantId, entryId: entry.id, content: entry.content };
|
|
251
269
|
}));
|
|
252
270
|
let next = 0;
|
|
253
271
|
for (const item of extracted) {
|
|
254
|
-
if (
|
|
272
|
+
if (held(item, duplicateKey(item.content))) {
|
|
255
273
|
tally.skipped++;
|
|
256
274
|
console.log(` [skip] (${item.category}) ${item.content.slice(0, 80)}`);
|
|
257
275
|
}
|
|
@@ -272,7 +290,7 @@ function captureEntry(item, options, baseHalfLifeDays) {
|
|
|
272
290
|
// 'default'. A rule the person stated is pinned, so it injects even when a later prompt shares
|
|
273
291
|
// none of its words. SHORTCUT: every stated rule pins; cap or age these pins if a store's pins
|
|
274
292
|
// outgrow the hook budget. The from-user tag marks the same pin as the person's rule for the hooks.
|
|
275
|
-
const stated = item
|
|
293
|
+
const stated = isStatedRule(item);
|
|
276
294
|
const created = createMemory(item.content, {
|
|
277
295
|
layer: Layer.Episodic,
|
|
278
296
|
tags: stated ? [...item.tags, USER_RULE_TAG] : item.tags,
|
|
@@ -5,8 +5,14 @@ export interface ExtractedItem {
|
|
|
5
5
|
/** A person said it, not the agent: capture pins a rule that carries this. */
|
|
6
6
|
fromUser?: boolean;
|
|
7
7
|
}
|
|
8
|
+
/** A rule the person stated: capture pins it and tags it as theirs. */
|
|
9
|
+
export declare function isStatedRule(item: ExtractedItem): boolean;
|
|
10
|
+
/** Each text once, where its first copy stood; a later copy that is the person's rule replaces a
|
|
11
|
+
* first copy that is not, so a preference restated in a reply keeps its rule status. */
|
|
12
|
+
export declare function oncePerText(items: readonly ExtractedItem[], admit?: (item: ExtractedItem) => boolean): ExtractedItem[];
|
|
8
13
|
/** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
|
|
9
|
-
* The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser
|
|
14
|
+
* The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`,
|
|
15
|
+
* and a copy that is their rule wins over one that is not.
|
|
10
16
|
* From `firstFollowUp` on, those words answer the agent's work, so a statement in them can be the person's rule. */
|
|
11
17
|
export declare function extractFromTexts(texts: readonly string[], userTexts?: number, firstFollowUp?: number): ExtractedItem[];
|
|
12
18
|
/** A text an older capture, which kept every bold marker, stored for a line, and what capture
|
|
@@ -2,6 +2,28 @@ import { duplicateKey } from '../../base/util/same-text.js';
|
|
|
2
2
|
import { assessAutomaticMemory, asideCloseOffset, QUOTE_MARK, } from '../../base/memory/memory-quality.js';
|
|
3
3
|
import { ERROR_TAG } from '../../base/memory/entry.js';
|
|
4
4
|
import { blankCodeSpans, strongMarkers } from '../../base/memory/markdown-delimiters.js';
|
|
5
|
+
/** A rule the person stated: capture pins it and tags it as theirs. */
|
|
6
|
+
export function isStatedRule(item) {
|
|
7
|
+
return item.fromUser === true && item.category === 'rule';
|
|
8
|
+
}
|
|
9
|
+
/** Each text once, where its first copy stood; a later copy that is the person's rule replaces a
|
|
10
|
+
* first copy that is not, so a preference restated in a reply keeps its rule status. */
|
|
11
|
+
export function oncePerText(items, admit = () => true) {
|
|
12
|
+
const kept = [];
|
|
13
|
+
const slot = new Map();
|
|
14
|
+
for (const item of items) {
|
|
15
|
+
const key = duplicateKey(item.content);
|
|
16
|
+
const at = slot.get(key);
|
|
17
|
+
if (at === undefined) {
|
|
18
|
+
if (admit(item))
|
|
19
|
+
slot.set(key, kept.push(item) - 1);
|
|
20
|
+
}
|
|
21
|
+
else if (isStatedRule(item) && !isStatedRule(kept[at]) && admit(item)) {
|
|
22
|
+
kept[at] = item;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return kept;
|
|
26
|
+
}
|
|
5
27
|
const DECISION_PATTERNS = [
|
|
6
28
|
/\b(?:we(?:'ve| have)?|i(?:'ve| have)?|let's)\s+decid(?:ed|e)\b/i,
|
|
7
29
|
/\b(?:let's|we(?:'ll| will| should)?)\s+(?:go with|do|use|try|build|implement|switch to)\b/i,
|
|
@@ -348,19 +370,12 @@ function extractSourced(text, cuts, followUp = false) {
|
|
|
348
370
|
return found;
|
|
349
371
|
}
|
|
350
372
|
/** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
|
|
351
|
-
* The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser
|
|
373
|
+
* The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`,
|
|
374
|
+
* and a copy that is their rule wins over one that is not.
|
|
352
375
|
* From `firstFollowUp` on, those words answer the agent's work, so a statement in them can be the person's rule. */
|
|
353
376
|
export function extractFromTexts(texts, userTexts = 0, firstFollowUp = userTexts) {
|
|
354
|
-
const
|
|
355
|
-
return
|
|
356
|
-
.flatMap((text, i) => extractSourced(text, boldPairs, i >= firstFollowUp && i < userTexts).map(({ item }) => i < userTexts ? { ...item, fromUser: true } : item))
|
|
357
|
-
.filter((item) => {
|
|
358
|
-
const key = duplicateKey(item.content);
|
|
359
|
-
if (seen.has(key) || !assessAutomaticMemory(item.content).accepted)
|
|
360
|
-
return false;
|
|
361
|
-
seen.add(key);
|
|
362
|
-
return true;
|
|
363
|
-
});
|
|
377
|
+
const items = texts.flatMap((text, i) => extractSourced(text, boldPairs, i >= firstFollowUp && i < userTexts).map(({ item }) => i < userTexts ? { ...item, fromUser: true } : item));
|
|
378
|
+
return oncePerText(items, (item) => assessAutomaticMemory(item.content).accepted);
|
|
364
379
|
}
|
|
365
380
|
function trackOf(items) {
|
|
366
381
|
return { items, ends: items.map(({ to }) => to) };
|