hippo-memory 3.3.2 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/context/context-pools.js +2 -1
- package/dist/api/context/user-rule.js +2 -1
- package/dist/api/dashboard/detail.js +2 -1
- package/dist/api/dashboard/live.d.ts +0 -4
- package/dist/api/dashboard/live.js +0 -8
- package/dist/api/memories/conflicts.d.ts +2 -1
- package/dist/api/memories/conflicts.js +3 -8
- package/dist/api/memories/promote.js +6 -2
- package/dist/api/recall/compose.js +2 -1
- package/dist/api/recall/rank-stages.js +2 -1
- package/dist/api/recall/rows.js +2 -1
- package/dist/api/sleep/sleep-run.js +2 -1
- package/dist/api/summary-parents.js +2 -1
- package/dist/base/memory/ambient.js +2 -1
- package/dist/base/memory/entry.d.ts +4 -1
- package/dist/base/memory/entry.js +7 -1
- package/dist/base/memory/half-life-plan.js +6 -4
- package/dist/base/memory/keep-rules.js +2 -1
- package/dist/base/memory/liveness.d.ts +7 -0
- package/dist/base/memory/liveness.js +6 -0
- package/dist/base/memory/markdown.js +15 -5
- package/dist/base/memory/memory-audit.js +2 -1
- package/dist/base/memory/memory-facts.d.ts +0 -2
- package/dist/base/memory/memory-facts.js +0 -2
- package/dist/base/memory/merged-row.js +3 -2
- package/dist/base/memory/rule-replaces.js +2 -1
- package/dist/base/memory/stale-mark.d.ts +9 -0
- package/dist/base/memory/stale-mark.js +18 -0
- package/dist/base/ports/store/conflicts.d.ts +5 -1
- package/dist/base/ports/store/conflicts.js +2 -0
- package/dist/base/ports/store/consolidation.d.ts +51 -2
- package/dist/base/ports/store/consolidation.js +3 -0
- package/dist/base/ports/store/sessions.d.ts +8 -2
- package/dist/base/ports/store/sessions.js +6 -0
- package/dist/base/ports/store/store-effects.d.ts +3 -0
- package/dist/base/scope/live-memory.d.ts +7 -0
- package/dist/base/scope/live-memory.js +10 -0
- package/dist/base/scope/read-admission.js +2 -1
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/db/continuity-tables.d.ts +6 -1
- package/dist/db/continuity-tables.js +27 -15
- package/dist/db/migrate.js +8 -4
- package/dist/db/migrations/index.d.ts +1 -1
- package/dist/db/migrations/index.js +3 -1
- package/dist/db/migrations/v61.js +3 -3
- package/dist/db/migrations/v62.d.ts +3 -0
- package/dist/db/migrations/v62.js +43 -0
- package/dist/db/required-objects.js +2 -1
- package/dist/db/sql-predicates.d.ts +22 -2
- package/dist/db/sql-predicates.js +27 -2
- package/dist/domain/agent-memories/apply.js +2 -1
- package/dist/domain/agent-memories/container-view.js +2 -1
- package/dist/domain/consolidate/conflicts.js +2 -1
- package/dist/domain/consolidate/dag-summary.d.ts +2 -0
- package/dist/domain/consolidate/dag.d.ts +3 -0
- package/dist/domain/consolidate/dag.js +53 -8
- package/dist/domain/consolidate/dedupe.js +2 -1
- package/dist/domain/consolidate/events.d.ts +2 -0
- package/dist/domain/consolidate/llm-passes.js +94 -20
- package/dist/domain/consolidate/merge.js +5 -4
- package/dist/domain/consolidate/retry-holds.d.ts +14 -0
- package/dist/domain/consolidate/retry-holds.js +26 -0
- package/dist/domain/consolidate/run.d.ts +8 -0
- package/dist/domain/consolidate/run.js +2 -0
- package/dist/domain/graph/recall.js +3 -2
- package/dist/domain/learn/churn-stale.js +2 -1
- package/dist/domain/learn/extract.d.ts +10 -0
- package/dist/domain/learn/extract.js +10 -5
- package/dist/domain/learn/invalidation.d.ts +1 -1
- package/dist/domain/learn/invalidation.js +4 -11
- package/dist/domain/objects/decisions.d.ts +2 -2
- package/dist/domain/objects/decisions.js +4 -9
- package/dist/domain/search/as-of.js +3 -2
- package/dist/store/sqlite/capture/compaction-rows.js +9 -2
- package/dist/store/sqlite/connectors/quarantine.js +13 -4
- package/dist/store/sqlite/consolidation/consolidation-summaries.d.ts +4 -5
- package/dist/store/sqlite/consolidation/consolidation-summaries.js +46 -26
- package/dist/store/sqlite/consolidation/consolidation.js +5 -0
- package/dist/store/sqlite/consolidation/empty-extractions.d.ts +14 -0
- package/dist/store/sqlite/consolidation/empty-extractions.js +57 -0
- package/dist/store/sqlite/consolidation/rebuild-holds.d.ts +21 -0
- package/dist/store/sqlite/consolidation/rebuild-holds.js +68 -0
- package/dist/store/sqlite/entries/entry-filter.js +2 -1
- package/dist/store/sqlite/entries/entry-reads.js +2 -1
- package/dist/store/sqlite/entries/entry-writes.js +6 -5
- package/dist/store/sqlite/maintenance/agent-import-steps.js +7 -2
- package/dist/store/sqlite/maintenance/project-tags.js +4 -3
- package/dist/store/sqlite/recall/candidates.js +10 -4
- package/dist/store/sqlite/recall/search-rows.js +4 -3
- package/dist/store/sqlite/reports/dashboard-lists.js +3 -2
- package/dist/store/sqlite/reports/dashboard-reads.js +5 -3
- package/dist/store/sqlite/reports/dashboard-sql.d.ts +0 -3
- package/dist/store/sqlite/reports/dashboard-sql.js +2 -9
- package/dist/store/sqlite/reports/reports.js +2 -1
- package/dist/store/sqlite/review/conflicts.js +7 -1
- package/dist/store/sqlite/runtime/rule-sql.js +2 -1
- package/dist/store/sqlite/runtime/status-transitions.d.ts +98 -0
- package/dist/store/sqlite/runtime/status-transitions.js +105 -0
- package/dist/store/sqlite/sessions/goals.js +11 -12
- package/dist/store/sqlite/sessions/sessions.d.ts +1 -1
- package/dist/store/sqlite/sessions/sessions.js +20 -8
- package/dist/surface/cli/conflicts.js +12 -1
- package/dist/surface/cli/continuity.js +4 -0
- package/dist/surface/cli/print.js +2 -1
- package/dist/surface/cli/verbs/objects.js +2 -2
- package/dist/surface/dashboard/dashboard-actions.js +1 -1
- package/dist/surface/render/sleep-events.js +2 -0
- package/dist-ui/assets/index-Cst0ubfs.js +1 -0
- package/dist-ui/index.html +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/dist-ui/assets/index-Q9SY-1Bj.js +0 -1
|
@@ -8,11 +8,10 @@ export declare function listL2SummariesPage(db: DatabaseSyncLike, page: {
|
|
|
8
8
|
readonly after?: EntryCursor;
|
|
9
9
|
}): EntryPage;
|
|
10
10
|
/** The `limit` dirty summaries of every tenant changed last (latest_at DESC NULLS LAST, id ASC),
|
|
11
|
-
* each with its live children; one row past the limit tells whether more
|
|
12
|
-
*
|
|
13
|
-
export declare function dirtySummaries(db: DatabaseSyncLike, limit: number): DirtySummaries;
|
|
14
|
-
/** Lands the settle
|
|
15
|
-
* clean is one UPDATE. */
|
|
11
|
+
* each with its live children and stored failure count; one row past the limit tells whether more
|
|
12
|
+
* were waiting. With `at`, a summary held past it is skipped and counted in `held`. */
|
|
13
|
+
export declare function dirtySummaries(db: DatabaseSyncLike, limit: number, at?: string): DirtySummaries;
|
|
14
|
+
/** Lands the settle and ends the summary's rebuild hold in one write scope when the row changed. */
|
|
16
15
|
export declare function settleSummary(db: DatabaseSyncLike, settle: SummarySettle): SummarySettled;
|
|
17
16
|
/** Each row its own write, a failed one logged: an audit row never undoes the change it records. */
|
|
18
17
|
export declare function auditRows(db: DatabaseSyncLike, events: readonly AppendAuditOptions[]): number;
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
// hippo.db's DAG summary reads and the one write that settles a dirty summary; what a summary
|
|
2
2
|
// settles to is decided in src/domain/consolidate/dag.ts.
|
|
3
3
|
import { DAG_LEVEL } from '../../../base/memory/entry.js';
|
|
4
|
-
import { withWriteScope } from '../../../db/index.js';
|
|
4
|
+
import { withReadSnapshot, withWriteScope } from '../../../db/index.js';
|
|
5
5
|
import { assertTenantId } from '../../../base/scope/tenant-id.js';
|
|
6
6
|
import { rejectionDigest } from '../../../base/records/rejection-digest.js';
|
|
7
7
|
import { readRejectedValue } from '../review/rejection.js';
|
|
8
|
-
import { MEMORY_SELECT_COLUMNS, parseJsonArray, rowToEntry, } from '../runtime/rows.js';
|
|
8
|
+
import { MEMORY_SEARCH_COLUMNS, MEMORY_SELECT_COLUMNS, parseJsonArray, rowToEntry, } from '../runtime/rows.js';
|
|
9
|
+
import { clearRebuildHold, heldSummaryCount, holdFailuresSql, holdRetryAfterSql, REBUILD_HOLD_KEY, } from './rebuild-holds.js';
|
|
9
10
|
import { refusalAudit } from '../../rules/audit-rows.js';
|
|
10
11
|
import { syncFtsRow } from '../entries/entry-row.js';
|
|
11
12
|
import { chunked } from '../../../base/util/chunked.js';
|
|
@@ -13,6 +14,7 @@ import { keysetPage } from '../../../base/util/keyset.js';
|
|
|
13
14
|
import { recordAuditEventsRowByRow } from '../audit/audit.js';
|
|
14
15
|
import { reportAuditWriteFailure } from '../../../base/records/audit-report.js';
|
|
15
16
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
17
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
16
18
|
/** One keyset page of the L2 topic summaries without an L3 parent, across all tenants; a SQL
|
|
17
19
|
* filter, since sleep's survivors lack the L2s its own DAG pass just wrote. */
|
|
18
20
|
export function listL2SummariesPage(db, page) {
|
|
@@ -24,7 +26,7 @@ export function listL2SummariesPage(db, page) {
|
|
|
24
26
|
WHERE dag_level = ${DAG_LEVEL.topic}
|
|
25
27
|
AND dag_parent_id IS NULL
|
|
26
28
|
AND kind != 'archived'
|
|
27
|
-
AND
|
|
29
|
+
AND ${notSupersededIn()}
|
|
28
30
|
${range}
|
|
29
31
|
ORDER BY created ASC, id ASC
|
|
30
32
|
LIMIT ?
|
|
@@ -33,23 +35,37 @@ export function listL2SummariesPage(db, page) {
|
|
|
33
35
|
return { entries: kept.rows.map(rowToEntry), next: kept.next };
|
|
34
36
|
}
|
|
35
37
|
/** The `limit` dirty summaries of every tenant changed last (latest_at DESC NULLS LAST, id ASC),
|
|
36
|
-
* each with its live children; one row past the limit tells whether more
|
|
37
|
-
*
|
|
38
|
-
export function dirtySummaries(db, limit) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
38
|
+
* each with its live children and stored failure count; one row past the limit tells whether more
|
|
39
|
+
* were waiting. With `at`, a summary held past it is skipped and counted in `held`. */
|
|
40
|
+
export function dirtySummaries(db, limit, at) {
|
|
41
|
+
return withReadSnapshot(db, () => {
|
|
42
|
+
const retry = holdRetryAfterSql('h');
|
|
43
|
+
const unheld = at === undefined ? '' : `AND (${retry} IS NULL OR ${retry} <= ?)`;
|
|
44
|
+
// SAFETY: MEMORY_SEARCH_COLUMNS selects every MemoryRow key and hold_failures is named here.
|
|
45
|
+
// The literal summary_dirty = 1 lets idx_memories_summary_dirty drive the order and the limit.
|
|
46
|
+
const rows = cachedStatement(db, `
|
|
47
|
+
SELECT ${MEMORY_SEARCH_COLUMNS}, ${holdFailuresSql('h')} AS hold_failures
|
|
48
|
+
FROM memories m
|
|
49
|
+
LEFT JOIN meta h ON h.key = '${REBUILD_HOLD_KEY}' || m.id
|
|
50
|
+
WHERE m.summary_dirty = 1
|
|
51
|
+
AND m.kind != 'archived'
|
|
52
|
+
${unheld}
|
|
53
|
+
ORDER BY m.latest_at DESC NULLS LAST, m.id ASC
|
|
54
|
+
LIMIT ?
|
|
55
|
+
`).all(...(at === undefined ? [] : [at]), limit + 1);
|
|
56
|
+
const kept = rows.slice(0, limit);
|
|
57
|
+
const summaries = kept.map(rowToEntry);
|
|
58
|
+
const children = childrenOfSummaries(db, summaries);
|
|
59
|
+
return {
|
|
60
|
+
summaries: summaries.map((summary, i) => ({
|
|
61
|
+
summary,
|
|
62
|
+
children: children[i],
|
|
63
|
+
failures: kept[i].hold_failures,
|
|
64
|
+
})),
|
|
65
|
+
more: rows.length > limit,
|
|
66
|
+
held: at === undefined ? undefined : heldSummaryCount(db, at),
|
|
67
|
+
};
|
|
68
|
+
});
|
|
53
69
|
}
|
|
54
70
|
/** The live children of each summary in one IN query per tenant chunk: one list per summary in
|
|
55
71
|
* the order given, each by created then id; tenant-scoped as defence in depth. */
|
|
@@ -70,7 +86,7 @@ function childrenOfSummaries(db, summaries) {
|
|
|
70
86
|
WHERE dag_parent_id IN (SELECT value FROM json_each(?))
|
|
71
87
|
AND tenant_id = ?
|
|
72
88
|
AND kind != 'archived'
|
|
73
|
-
AND
|
|
89
|
+
AND ${notSupersededIn()}
|
|
74
90
|
ORDER BY created ASC, id ASC
|
|
75
91
|
`).all(JSON.stringify(slice), tenantId);
|
|
76
92
|
for (const entry of rows.map(rowToEntry)) {
|
|
@@ -112,13 +128,17 @@ const SETTLE_FIELDS_SQL = `UPDATE memories
|
|
|
112
128
|
summary_dirty = 0
|
|
113
129
|
${STILL_DIRTY}`;
|
|
114
130
|
const SETTLE_CLEAN_SQL = `UPDATE memories SET summary_dirty = 0 ${STILL_DIRTY}`;
|
|
115
|
-
/** Lands the settle
|
|
116
|
-
* clean is one UPDATE. */
|
|
131
|
+
/** Lands the settle and ends the summary's rebuild hold in one write scope when the row changed. */
|
|
117
132
|
export function settleSummary(db, settle) {
|
|
118
133
|
assertTenantId('settleSummary', settle.tenantId);
|
|
119
|
-
return
|
|
120
|
-
|
|
121
|
-
|
|
134
|
+
return withWriteScope(db, 'rebuild_summary', () => {
|
|
135
|
+
const settled = settle.fields
|
|
136
|
+
? settleFields(db, settle, settle.fields)
|
|
137
|
+
: settleClean(db, settle);
|
|
138
|
+
if (settled.changed)
|
|
139
|
+
clearRebuildHold(db, settle.id);
|
|
140
|
+
return settled;
|
|
141
|
+
});
|
|
122
142
|
}
|
|
123
143
|
function settleClean(db, settle) {
|
|
124
144
|
const row = cachedStatement(db, SETTLE_CLEAN_SQL).get(settle.id, settle.tenantId);
|
|
@@ -22,6 +22,8 @@ import { bound } from '../runtime/bound.js';
|
|
|
22
22
|
import { rethrowInPortTerms } from '../runtime/port-errors.js';
|
|
23
23
|
import { auditRows, dirtySummaries, listL2SummariesPage, settleSummary, } from './consolidation-summaries.js';
|
|
24
24
|
import { META_KEY } from '../../../db/meta.js';
|
|
25
|
+
import { noteRebuildFailures } from './rebuild-holds.js';
|
|
26
|
+
import { emptyExtractions, noteEmptyExtractions } from './empty-extractions.js';
|
|
25
27
|
// A half-life move reads, writes, audits and records its base on one handle, so it commits whole or
|
|
26
28
|
// not at all.
|
|
27
29
|
function listHalfLifeRows(db, from, typedPending) {
|
|
@@ -124,6 +126,9 @@ const bareBodies = {
|
|
|
124
126
|
particlesByRecall: storedParticlesByRecall,
|
|
125
127
|
saveParticles: saveStoredParticles,
|
|
126
128
|
auditEachRow: auditRows,
|
|
129
|
+
noteRebuildFailures,
|
|
130
|
+
emptyExtractions,
|
|
131
|
+
noteEmptyExtractions,
|
|
127
132
|
dirtyGraphTenants: listPendingExtractionTenants,
|
|
128
133
|
markGraphDrained: markPendingProcessedUpTo,
|
|
129
134
|
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type DatabaseSyncLike } from '../../../db/index.js';
|
|
2
|
+
import type { Consolidation } from '../../../base/ports/store/consolidation.js';
|
|
3
|
+
type Page = Awaited<ReturnType<NonNullable<Consolidation['emptyExtractions']>>>;
|
|
4
|
+
type Note = Parameters<NonNullable<Consolidation['noteEmptyExtractions']>>[0];
|
|
5
|
+
/** One page of marks by source id, after the id `after`; a row that is not a mark is skipped. */
|
|
6
|
+
export declare function emptyExtractions(db: DatabaseSyncLike, page: {
|
|
7
|
+
readonly limit: number;
|
|
8
|
+
readonly after?: string;
|
|
9
|
+
}): Page;
|
|
10
|
+
/** Stores the marks and drops the cleared ones in one write, then drops the marks of sources that
|
|
11
|
+
* are no longer live. */
|
|
12
|
+
export declare function noteEmptyExtractions(db: DatabaseSyncLike, { mark, clear }: Note): void;
|
|
13
|
+
export {};
|
|
14
|
+
//# sourceMappingURL=empty-extractions.d.ts.map
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// A source's empty extraction mark, one `meta` row per source: {"digest": hash of its text, "at": ISO}.
|
|
2
|
+
// When a mark stands is decided in src/domain/consolidate/retry-holds.ts.
|
|
3
|
+
import { withWriteScope } from '../../../db/index.js';
|
|
4
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
5
|
+
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
6
|
+
const MARK_KEY = 'empty_extraction:';
|
|
7
|
+
// The first key past every mark key, as ';' follows ':'.
|
|
8
|
+
const MARK_END = 'empty_extraction;';
|
|
9
|
+
// substr is 1-based, so this is the first character of the id.
|
|
10
|
+
const ID_FROM = MARK_KEY.length + 1;
|
|
11
|
+
// A value that is not JSON with text digest and at reads as no mark; CASE keeps json_type off bad
|
|
12
|
+
// JSON, which AND would not.
|
|
13
|
+
const PAGE_SQL = `SELECT substr(key, ${ID_FROM}) AS id,
|
|
14
|
+
json_extract(value, '$.digest') AS digest, json_extract(value, '$.at') AS at
|
|
15
|
+
FROM meta
|
|
16
|
+
WHERE key > ? AND key < '${MARK_END}'
|
|
17
|
+
AND CASE WHEN json_valid(value)
|
|
18
|
+
THEN json_type(value, '$.digest') = 'text' AND json_type(value, '$.at') = 'text'
|
|
19
|
+
ELSE 0 END
|
|
20
|
+
ORDER BY key
|
|
21
|
+
LIMIT ?`;
|
|
22
|
+
/** One page of marks by source id, after the id `after`; a row that is not a mark is skipped. */
|
|
23
|
+
export function emptyExtractions(db, page) {
|
|
24
|
+
// SAFETY: the SQL names id, digest and at, and its WHERE keeps digest and at text.
|
|
25
|
+
const rows = cachedStatement(db, PAGE_SQL).all(`${MARK_KEY}${page.after ?? ''}`, page.limit + 1);
|
|
26
|
+
const marks = rows.slice(0, page.limit);
|
|
27
|
+
return { marks, next: rows.length > page.limit ? (marks.at(-1)?.id ?? null) : null };
|
|
28
|
+
}
|
|
29
|
+
// Writes only for a source row that is live, as the sleep that offers it reads it.
|
|
30
|
+
const MARK_SQL = `INSERT INTO meta(key, value)
|
|
31
|
+
SELECT ?, ?
|
|
32
|
+
WHERE EXISTS (SELECT 1 FROM memories
|
|
33
|
+
WHERE id = ? AND kind != 'archived' AND ${notSupersededIn()})
|
|
34
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`;
|
|
35
|
+
// A mark left for a source that is gone, archived or superseded would never be read again.
|
|
36
|
+
const SWEEP_SQL = `DELETE FROM meta
|
|
37
|
+
WHERE key > '${MARK_KEY}' AND key < '${MARK_END}'
|
|
38
|
+
AND NOT EXISTS (SELECT 1 FROM memories m
|
|
39
|
+
WHERE m.id = substr(meta.key, ${ID_FROM})
|
|
40
|
+
AND m.kind != 'archived' AND ${notSupersededIn('m.')})`;
|
|
41
|
+
/** Stores the marks and drops the cleared ones in one write, then drops the marks of sources that
|
|
42
|
+
* are no longer live. */
|
|
43
|
+
export function noteEmptyExtractions(db, { mark, clear }) {
|
|
44
|
+
if (mark.length === 0 && clear.length === 0)
|
|
45
|
+
return;
|
|
46
|
+
withWriteScope(db, 'empty_extraction', () => {
|
|
47
|
+
const put = cachedStatement(db, MARK_SQL);
|
|
48
|
+
for (const { id, digest, at } of mark) {
|
|
49
|
+
put.run(`${MARK_KEY}${id}`, JSON.stringify({ digest, at }), id);
|
|
50
|
+
}
|
|
51
|
+
const drop = cachedStatement(db, 'DELETE FROM meta WHERE key = ?');
|
|
52
|
+
for (const id of clear)
|
|
53
|
+
drop.run(`${MARK_KEY}${id}`);
|
|
54
|
+
cachedStatement(db, SWEEP_SQL).run();
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=empty-extractions.js.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type DatabaseSyncLike } from '../../../db/index.js';
|
|
2
|
+
import type { Consolidation } from '../../../base/ports/store/consolidation.js';
|
|
3
|
+
/** The meta key prefix of a summary's hold; the summary's id follows. */
|
|
4
|
+
export declare const REBUILD_HOLD_KEY = "rebuild_hold:";
|
|
5
|
+
type RebuildFailure = Parameters<NonNullable<Consolidation['noteRebuildFailures']>>[0][number];
|
|
6
|
+
/** SQL for the failure count of the hold row `alias`; 0 when it has none, is not JSON or holds no
|
|
7
|
+
* integer count, so one bad row never stops a rebuild. The nested CASE keeps json_type off bad
|
|
8
|
+
* JSON. */
|
|
9
|
+
export declare const holdFailuresSql: (alias: string) => string;
|
|
10
|
+
/** SQL for the ISO time the hold row `alias` ends; NULL when there is none. */
|
|
11
|
+
export declare const holdRetryAfterSql: (alias: string) => string;
|
|
12
|
+
/** How many dirty live summaries are held after `at`, from the hold keys alone so the read grows
|
|
13
|
+
* with held rows, not with the queue. */
|
|
14
|
+
export declare function heldSummaryCount(db: DatabaseSyncLike, at: string): number;
|
|
15
|
+
/** Ends the summary's hold; the settle that landed it calls this in its own write scope. */
|
|
16
|
+
export declare function clearRebuildHold(db: DatabaseSyncLike, summaryId: string): void;
|
|
17
|
+
/** Stores each failure's count and hold time in one write, then drops the holds of summaries that
|
|
18
|
+
* are no longer dirty. */
|
|
19
|
+
export declare function noteRebuildFailures(db: DatabaseSyncLike, failed: ReadonlyArray<RebuildFailure>): void;
|
|
20
|
+
export {};
|
|
21
|
+
//# sourceMappingURL=rebuild-holds.d.ts.map
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// A dirty summary's rebuild hold, one `meta` row per summary: {"failures": n, "retryAfter": ISO | null}.
|
|
2
|
+
// When a hold ends is decided in src/domain/consolidate/retry-holds.ts.
|
|
3
|
+
import { DAG_LEVEL } from '../../../base/memory/entry.js';
|
|
4
|
+
import { withWriteScope } from '../../../db/index.js';
|
|
5
|
+
import { assertTenantId } from '../../../base/scope/tenant-id.js';
|
|
6
|
+
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
7
|
+
/** The meta key prefix of a summary's hold; the summary's id follows. */
|
|
8
|
+
export const REBUILD_HOLD_KEY = 'rebuild_hold:';
|
|
9
|
+
// The first key past every hold key, as ';' follows ':'.
|
|
10
|
+
const REBUILD_HOLD_END = 'rebuild_hold;';
|
|
11
|
+
// substr is 1-based, so this is the first character of the id.
|
|
12
|
+
const ID_FROM = REBUILD_HOLD_KEY.length + 1;
|
|
13
|
+
/** SQL for the failure count of the hold row `alias`; 0 when it has none, is not JSON or holds no
|
|
14
|
+
* integer count, so one bad row never stops a rebuild. The nested CASE keeps json_type off bad
|
|
15
|
+
* JSON. */
|
|
16
|
+
export const holdFailuresSql = (alias) => `CASE WHEN json_valid(${alias}.value) THEN ` +
|
|
17
|
+
`CASE WHEN json_type(${alias}.value, '$.failures') = 'integer' ` +
|
|
18
|
+
`THEN json_extract(${alias}.value, '$.failures') ELSE 0 END ELSE 0 END`;
|
|
19
|
+
/** SQL for the ISO time the hold row `alias` ends; NULL when there is none. */
|
|
20
|
+
export const holdRetryAfterSql = (alias) => `CASE WHEN json_valid(${alias}.value) THEN json_extract(${alias}.value, '$.retryAfter') END`;
|
|
21
|
+
/** How many dirty live summaries are held after `at`, from the hold keys alone so the read grows
|
|
22
|
+
* with held rows, not with the queue. */
|
|
23
|
+
export function heldSummaryCount(db, at) {
|
|
24
|
+
// SAFETY: the SQL names n, the only key of the row.
|
|
25
|
+
const row = cachedStatement(db, `SELECT COUNT(*) AS n
|
|
26
|
+
FROM meta h JOIN memories m ON m.id = substr(h.key, ${ID_FROM})
|
|
27
|
+
WHERE h.key > '${REBUILD_HOLD_KEY}' AND h.key < '${REBUILD_HOLD_END}'
|
|
28
|
+
AND m.summary_dirty = 1 AND m.kind != 'archived'
|
|
29
|
+
AND ${holdRetryAfterSql('h')} > ?`).get(at);
|
|
30
|
+
return row.n;
|
|
31
|
+
}
|
|
32
|
+
/** Ends the summary's hold; the settle that landed it calls this in its own write scope. */
|
|
33
|
+
export function clearRebuildHold(db, summaryId) {
|
|
34
|
+
cachedStatement(db, 'DELETE FROM meta WHERE key = ?').run(`${REBUILD_HOLD_KEY}${summaryId}`);
|
|
35
|
+
}
|
|
36
|
+
// Writes only while the stored count is still the one the caller saw, and only for a summary
|
|
37
|
+
// that is dirty and live: a settle that ended the hold in between wins.
|
|
38
|
+
const NOTE_FAILURE_SQL = `INSERT INTO meta(key, value)
|
|
39
|
+
SELECT ?, ?
|
|
40
|
+
WHERE EXISTS (SELECT 1 FROM memories
|
|
41
|
+
WHERE id = ? AND tenant_id = ?
|
|
42
|
+
AND dag_level IN (${DAG_LEVEL.topic}, ${DAG_LEVEL.entity})
|
|
43
|
+
AND summary_dirty = 1 AND kind != 'archived')
|
|
44
|
+
AND COALESCE((SELECT ${holdFailuresSql('h')} FROM meta h WHERE h.key = ?), 0) = ?
|
|
45
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`;
|
|
46
|
+
// A hold left for a summary that is gone, archived or clean would hold it again once it is dirty.
|
|
47
|
+
const SWEEP_SQL = `DELETE FROM meta
|
|
48
|
+
WHERE key > '${REBUILD_HOLD_KEY}' AND key < '${REBUILD_HOLD_END}'
|
|
49
|
+
AND NOT EXISTS (SELECT 1 FROM memories m
|
|
50
|
+
WHERE m.id = substr(meta.key, ${ID_FROM})
|
|
51
|
+
AND m.summary_dirty = 1 AND m.kind != 'archived')`;
|
|
52
|
+
/** Stores each failure's count and hold time in one write, then drops the holds of summaries that
|
|
53
|
+
* are no longer dirty. */
|
|
54
|
+
export function noteRebuildFailures(db, failed) {
|
|
55
|
+
if (failed.length === 0)
|
|
56
|
+
return;
|
|
57
|
+
for (const item of failed)
|
|
58
|
+
assertTenantId('noteRebuildFailures', item.tenantId);
|
|
59
|
+
withWriteScope(db, 'rebuild_hold', () => {
|
|
60
|
+
const note = cachedStatement(db, NOTE_FAILURE_SQL);
|
|
61
|
+
for (const { id, tenantId, priorFailures, failures, retryAfter } of failed) {
|
|
62
|
+
const key = `${REBUILD_HOLD_KEY}${id}`;
|
|
63
|
+
note.run(key, JSON.stringify({ failures, retryAfter }), id, tenantId, key, priorFailures);
|
|
64
|
+
}
|
|
65
|
+
cachedStatement(db, SWEEP_SQL).run();
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=rebuild-holds.js.map
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// memory/entry-filter.ts is the predicate half.
|
|
4
4
|
import { keysetPage } from '../../../base/util/keyset.js';
|
|
5
5
|
import { ALL_TENANTS } from '../../../base/util/tenant.js';
|
|
6
|
+
import { notSupersededAt } from '../../../db/sql-predicates.js';
|
|
6
7
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
7
8
|
import { entriesPage, entryPageOrder, entryRange, } from '../runtime/keyset.js';
|
|
8
9
|
import { MEMORY_SELECT_COLUMNS, rowToEntry } from '../runtime/rows.js';
|
|
@@ -48,7 +49,7 @@ const TERMS = {
|
|
|
48
49
|
params: [tag],
|
|
49
50
|
}),
|
|
50
51
|
kinds: (kinds, column) => inList(`COALESCE(${column('kind')}, 'distilled')`, kinds),
|
|
51
|
-
live: (_live, column) => ({ sql:
|
|
52
|
+
live: (_live, column) => ({ sql: notSupersededAt(column('superseded_by')), params: [] }),
|
|
52
53
|
sessionId: (sessionId, column) => ({
|
|
53
54
|
sql: `${column('source_session_id')} = ?`,
|
|
54
55
|
params: [sessionId],
|
|
@@ -3,6 +3,7 @@ import { MEMORY_SELECT_COLUMNS, rowToEntry, parseJsonArray, } from '../runtime/r
|
|
|
3
3
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
4
4
|
import { escapeLike } from '../../../base/util/escape.js';
|
|
5
5
|
import { scopeAdmitSql } from '../runtime/rule-sql.js';
|
|
6
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
6
7
|
import { chunked } from '../../../base/util/chunked.js';
|
|
7
8
|
import { byIdOrder, FRESH_RAW_ORDER, orderSql } from '../../rules/order.js';
|
|
8
9
|
import { ALL_TENANTS, DEFAULT_TENANT_ID } from '../../../base/util/tenant.js';
|
|
@@ -193,7 +194,7 @@ export function countByOrigin(db, tenantId, sourcePrefix) {
|
|
|
193
194
|
// The unit separator sorts below every character of a time or an id, so the joined string orders
|
|
194
195
|
// as (created, id).
|
|
195
196
|
return cachedStatement(db, `SELECT origin_project AS origin, COUNT(*) AS live, SUM(substr(source, 1, ?) = ?) AS prefixed, ` +
|
|
196
|
-
`MAX(created) AS newest FROM memories WHERE tenant_id = ? AND
|
|
197
|
+
`MAX(created) AS newest FROM memories WHERE tenant_id = ? AND ${notSupersededIn()} ` +
|
|
197
198
|
`GROUP BY origin_project ORDER BY MIN(created || char(31) || id)`).all(sourcePrefix.length, sourcePrefix, tenantId);
|
|
198
199
|
}
|
|
199
200
|
/** The ids among `ids` that a row of any tenant holds, one query per chunk. */
|
|
@@ -25,6 +25,7 @@ import { openStore } from '../runtime/handles.js';
|
|
|
25
25
|
import { bound } from '../runtime/bound.js';
|
|
26
26
|
import { readMemoryReach } from '../runtime/tenant-lookup.js';
|
|
27
27
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
28
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
28
29
|
/** Writes a promoted copy on the caller's open store; with `auditAs` the `promote` audit row
|
|
29
30
|
* joins the INSERT's SAVEPOINT, so both or neither. */
|
|
30
31
|
export function writePromotedEntry(db, files, entry, opts) {
|
|
@@ -278,7 +279,7 @@ export function stampOriginProjects(db, tenantId, rows) {
|
|
|
278
279
|
/** Renames a live row's source from `from` to `to`; the count is 0 when another writer moved or
|
|
279
280
|
* superseded it first. */
|
|
280
281
|
export function renameEntrySource(db, { tenantId, id, from, to }) {
|
|
281
|
-
const moved = cachedStatement(db, `UPDATE memories SET source = ? WHERE id = ? AND tenant_id = ? AND source = ? AND
|
|
282
|
+
const moved = cachedStatement(db, `UPDATE memories SET source = ? WHERE id = ? AND tenant_id = ? AND source = ? AND ${notSupersededIn()}`).run(to, id, tenantId, from);
|
|
282
283
|
return Number(moved.changes ?? 0);
|
|
283
284
|
}
|
|
284
285
|
/** Renames a row's source and, when `origin` is not null, its project; the count is 0 when the
|
|
@@ -291,13 +292,13 @@ export function renameEntrySourceAndOrigin(db, tenantId, id, options) {
|
|
|
291
292
|
}
|
|
292
293
|
/** Marks `id` superseded by `newId`; false when another writer superseded it first. */
|
|
293
294
|
export function supersedeEntry(db, { tenantId, id, newId }) {
|
|
294
|
-
const result = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND
|
|
295
|
+
const result = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND ${notSupersededIn()}`).run(newId, id, tenantId);
|
|
295
296
|
return Number(result.changes ?? 0) !== 0;
|
|
296
297
|
}
|
|
297
298
|
/** Marks `id` superseded by `newId` in the caller's transaction, with the supersede audit row and
|
|
298
299
|
* its summary parent's dirty mark; false when the row is gone or another writer got there first. */
|
|
299
300
|
export function supersedeInTx(db, { tenantId, id, newId, actor }) {
|
|
300
|
-
const won = cachedStatement(db,
|
|
301
|
+
const won = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND ${notSupersededIn()} ` +
|
|
301
302
|
'RETURNING dag_parent_id').get(newId, id, tenantId);
|
|
302
303
|
if (!won)
|
|
303
304
|
return false;
|
|
@@ -486,8 +487,8 @@ function commitSupersede(db, write) {
|
|
|
486
487
|
auditingRefusal(db, actor, () => withWriteScope(db, 'supersede', () => {
|
|
487
488
|
assertMayChange(db, write, oldId);
|
|
488
489
|
// No row means another writer got there first.
|
|
489
|
-
const won = cachedStatement(db,
|
|
490
|
-
'dag_parent_id').get(successor.id, oldId, tenantId);
|
|
490
|
+
const won = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND ${notSupersededIn()} ` +
|
|
491
|
+
'RETURNING dag_parent_id').get(successor.id, oldId, tenantId);
|
|
491
492
|
if (!won)
|
|
492
493
|
throw new ConflictError(`Memory ${oldId} already superseded by another writer`);
|
|
493
494
|
// After the CAS, so a lost race marks no parent for rebuild.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// One container's planned steps on the caller's write scope: the rows they name are re-read first,
|
|
2
2
|
// and a plan whose rows moved on since the sync read them writes nothing.
|
|
3
3
|
import { SYNC_ACTOR } from '../../../base/memory/agent-memory-tools.js';
|
|
4
|
+
import { isSuperseded } from '../../../base/memory/liveness.js';
|
|
4
5
|
import { chunked } from '../../../base/util/chunked.js';
|
|
5
6
|
import { READ_PAGE_ROWS, rowsInPages } from '../../../base/util/pages.js';
|
|
6
7
|
import { appendAuditEvents } from '../audit/audit.js';
|
|
@@ -48,8 +49,12 @@ function standingRows(db, plan) {
|
|
|
48
49
|
for (const row of readEntriesInIdOrder(db, ids, plan.tenantId))
|
|
49
50
|
rows.set(row.id, row);
|
|
50
51
|
}
|
|
51
|
-
const adoptable = plan.steps.every((step) =>
|
|
52
|
-
(
|
|
52
|
+
const adoptable = plan.steps.every((step) => {
|
|
53
|
+
if (step.kind !== 'adopt')
|
|
54
|
+
return true;
|
|
55
|
+
const row = rows.get(step.id);
|
|
56
|
+
return row !== undefined && row.source === step.from && !isSuperseded(row);
|
|
57
|
+
});
|
|
53
58
|
return adoptable ? rows : null;
|
|
54
59
|
}
|
|
55
60
|
/** A prefix under another listed one reads only rows that one already returns. */
|
|
@@ -15,6 +15,7 @@ import { setAsideRow } from '../entries/set-aside.js';
|
|
|
15
15
|
import { bound } from '../runtime/bound.js';
|
|
16
16
|
import { listEntriesWhere } from '../entries/entry-filter.js';
|
|
17
17
|
import { META_KEY } from '../../../db/meta.js';
|
|
18
|
+
import { isSuperseded } from '../../../base/memory/liveness.js';
|
|
18
19
|
const TX_LABEL = 'merge_projects';
|
|
19
20
|
/** One transaction, the store backed up first, with mirrors following the commit. */
|
|
20
21
|
function rewrite(db, files, { tenantId, backupLabel, body }) {
|
|
@@ -57,7 +58,7 @@ function fold(db, files, tenantId, { from, into }) {
|
|
|
57
58
|
// yet to read.
|
|
58
59
|
for (const row of rowsWhere(db, { tenantId, origins: [from] })) {
|
|
59
60
|
const tag = toolTag(row.source);
|
|
60
|
-
const live = isImport(row.source) && !row
|
|
61
|
+
const live = isImport(row.source) && !isSuperseded(row) && row.kind !== 'raw';
|
|
61
62
|
const copy = live && tag !== null && held.has(noteKey(row.source));
|
|
62
63
|
if (copy &&
|
|
63
64
|
setAsideRow(db, {
|
|
@@ -103,7 +104,7 @@ function setAsideCopies(db, files, tenantId, copies) {
|
|
|
103
104
|
for (const { id, origin } of copies) {
|
|
104
105
|
const row = rows.get(id);
|
|
105
106
|
const tag = row ? toolTag(row.source) : null;
|
|
106
|
-
if (!row || tag === null || row
|
|
107
|
+
if (!row || tag === null || isSuperseded(row) || (row.originProject ?? null) !== origin) {
|
|
107
108
|
continue;
|
|
108
109
|
}
|
|
109
110
|
if (setAsideRow(db, { tag, row, why: 'project-repair', rules: files }).kind === 'dormant') {
|
|
@@ -116,7 +117,7 @@ function setAsideCopies(db, files, tenantId, copies) {
|
|
|
116
117
|
* one re-tagged since stays as it is. */
|
|
117
118
|
function userGlobalMerges(db, tenantId, ids) {
|
|
118
119
|
const rows = readEntriesByIds(db, ids, tenantId);
|
|
119
|
-
return new Map([...rows].filter(([, e]) => !e
|
|
120
|
+
return new Map([...rows].filter(([, e]) => !isSuperseded(e) && e.originProject === ''));
|
|
120
121
|
}
|
|
121
122
|
/** A live row to dormant storage, restorable, as sleep's dormant move does. */
|
|
122
123
|
function retire(db, { files, row, actor, now }) {
|
|
@@ -5,6 +5,7 @@ import { DECAY_KEY_HALF_LIVES, DECAY_KEY_SLACK_DAYS, DECAY_KEY_SQL, DECAY_KEY_WH
|
|
|
5
5
|
import { withReadSnapshot } from '../../../db/index.js';
|
|
6
6
|
import { sortableStamp } from '../../../base/memory/created-stamp.js';
|
|
7
7
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
8
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
8
9
|
import { MEMORY_SELECT_COLUMNS, rowToEntry, parseJsonArray, } from '../runtime/rows.js';
|
|
9
10
|
import { pickRarestFtsQuery, listRecallEntries } from './search-rows.js';
|
|
10
11
|
import { oldestFirst } from '../../rules/order.js';
|
|
@@ -16,7 +17,12 @@ import { SCHEMA_FIT_WINDOW_ROWS } from '../../rules/windows.js';
|
|
|
16
17
|
/** The counts over `rows`; the caller adds the conflict and vector counts. */
|
|
17
18
|
function tallyStatus(rows, now, atRiskBelow, rules) {
|
|
18
19
|
const byLayer = { [Layer.Buffer]: 0, [Layer.Episodic]: 0, [Layer.Semantic]: 0, [Layer.Trace]: 0 };
|
|
19
|
-
const byConfidence = {
|
|
20
|
+
const byConfidence = {
|
|
21
|
+
verified: 0,
|
|
22
|
+
observed: 0,
|
|
23
|
+
inferred: 0,
|
|
24
|
+
stale: 0,
|
|
25
|
+
};
|
|
20
26
|
const sums = { total: 0, strength: 0, pinned: 0, atRisk: 0, agedOut: 0 };
|
|
21
27
|
for (const row of rows) {
|
|
22
28
|
const strength = calculateStrength(row, now, rules.strength);
|
|
@@ -43,8 +49,8 @@ function tallyStatus(rows, now, atRiskBelow, rules) {
|
|
|
43
49
|
avgStrength: total > 0 ? sums.strength / total : 0,
|
|
44
50
|
};
|
|
45
51
|
}
|
|
46
|
-
const AMBIENT_SCOPED =
|
|
47
|
-
const AMBIENT_PINNED =
|
|
52
|
+
const AMBIENT_SCOPED = `${notSupersededIn()} AND tenant_id = ?`;
|
|
53
|
+
const AMBIENT_PINNED = `pinned = 1 AND ${notSupersededIn()}`;
|
|
48
54
|
/** A stamp whose text does not sort by time; idx_memories_created_drift (migration v51) holds
|
|
49
55
|
* the live ones. */
|
|
50
56
|
const CREATED_DRIFTED = "(length(created) <> 24 OR created NOT LIKE '%Z')";
|
|
@@ -199,7 +205,7 @@ const decayRankSql = (model) => `pinned DESC,
|
|
|
199
205
|
/** The shared WHERE of a context read: live tenant rows, then `extra` predicates, then the scope
|
|
200
206
|
* and origin rules. */
|
|
201
207
|
export function contextRowsWhere(tenantId, filter, extra = []) {
|
|
202
|
-
const where = ['tenant_id = ?',
|
|
208
|
+
const where = ['tenant_id = ?', notSupersededIn(), ...extra];
|
|
203
209
|
const params = [tenantId];
|
|
204
210
|
if (filter.exactScope) {
|
|
205
211
|
where.push('scope = ?');
|
|
@@ -11,6 +11,7 @@ import { cachedStatement } from '../../../db/statement-cache.js';
|
|
|
11
11
|
import { escapeLike } from '../../../base/util/escape.js';
|
|
12
12
|
import { ftsAnyPhrase } from './fts-phrase.js';
|
|
13
13
|
import { placeholders } from '../../../db/sql.js';
|
|
14
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
14
15
|
import { VECTOR_CANDIDATE_LIMIT, } from '../../../base/ports/store/vectors.js';
|
|
15
16
|
import { ALL_TENANTS } from '../../../base/util/tenant.js';
|
|
16
17
|
import { tenantAnd } from '../runtime/tenant.js';
|
|
@@ -104,8 +105,8 @@ function searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjec
|
|
|
104
105
|
const aliasScope = withProject(recallScopeClause('m.', scopeFilter), 'm.', originProjects);
|
|
105
106
|
const plainScope = withProject(recallScopeClause('', scopeFilter), '', originProjects);
|
|
106
107
|
const scopeParams = aliasScope.params;
|
|
107
|
-
const currentAlias = includeSuperseded ? '' :
|
|
108
|
-
const currentNoAlias = includeSuperseded ? '' :
|
|
108
|
+
const currentAlias = includeSuperseded ? '' : ` AND ${notSupersededIn('m.')}`;
|
|
109
|
+
const currentNoAlias = includeSuperseded ? '' : ` AND ${notSupersededIn()}`;
|
|
109
110
|
return {
|
|
110
111
|
tenantPredicate,
|
|
111
112
|
tenantOnlyPredicate,
|
|
@@ -232,7 +233,7 @@ export function listRecallEntries(db, query, options) {
|
|
|
232
233
|
export async function listVectorCandidateEntries(db, queryVector, spec, index) {
|
|
233
234
|
const scope = withProject(recallScopeClause('m.', spec.scope), 'm.', spec.origin);
|
|
234
235
|
const tenant = tenantAnd(spec.tenantId, 'm.tenant_id = ?');
|
|
235
|
-
const current = spec.includeSuperseded ? '' :
|
|
236
|
+
const current = spec.includeSuperseded ? '' : ` AND ${notSupersededIn('m.')}`;
|
|
236
237
|
const params = [...tenant.params, ...scope.params];
|
|
237
238
|
const matches = await topVectorMatches(db, {
|
|
238
239
|
query: queryVector,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { withReadSnapshot } from '../../../db/busy.js';
|
|
2
2
|
import { pragmaDataVersion } from '../../../db/meta.js';
|
|
3
3
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
4
|
-
import {
|
|
4
|
+
import { CONFIDENCE_LEVELS } from '../../../base/memory/entry.js';
|
|
5
|
+
import { BAND_AT_RISK, LAYER_ORDER, listHead, memoryFacts, } from '../../../base/memory/memory-facts.js';
|
|
5
6
|
import { DAY_MS } from '../../../base/util/time.js';
|
|
6
7
|
import { FACT_COLUMNS, factInputs, FILTER_SLOTS, textFilter, tallyOf, } from './dashboard-functions.js';
|
|
7
8
|
import { IN_PROJECT, OPEN_PAIRS } from './dashboard-sql.js';
|
|
@@ -24,7 +25,7 @@ const SORT_SQL = {
|
|
|
24
25
|
retrievals: 'COALESCE(retrieval_count, 0)',
|
|
25
26
|
last: 'hippo_days(last_retrieved, :now)',
|
|
26
27
|
age: 'age',
|
|
27
|
-
confidence: rankBy(`COALESCE(confidence, 'observed')`,
|
|
28
|
+
confidence: rankBy(`COALESCE(confidence, 'observed')`, CONFIDENCE_LEVELS),
|
|
28
29
|
scope: `hippo_text_key(COALESCE(scope, ''))`,
|
|
29
30
|
};
|
|
30
31
|
const CHIP_SQL = `(:chip = 'all' OR (:chip = 'risk' AND hippo_band(${FACT_COLUMNS}, :now) = ${BAND_AT_RISK}) ` +
|
|
@@ -12,7 +12,8 @@ import { SCATTER_POINTS_MAX, } from '../../../base/ports/store/dashboard-reads.j
|
|
|
12
12
|
import { openConflictsNaming } from '../review/conflicts.js';
|
|
13
13
|
import { FACT_COLUMNS, GRID_CELLS, textFilter, PROJECT_SLOTS, tallyOf, useDashboardFunctions, } from './dashboard-functions.js';
|
|
14
14
|
import { MemoryLists } from './dashboard-lists.js';
|
|
15
|
-
import { IN_PROJECT,
|
|
15
|
+
import { IN_PROJECT, LIVE, OPEN_PAIRS } from './dashboard-sql.js';
|
|
16
|
+
import { kindIn, quarantinedIn, supersededIn, supersededKindIn, } from '../../../db/sql-predicates.js';
|
|
16
17
|
import { handlesOf, openStore } from '../runtime/handles.js';
|
|
17
18
|
import { answered, bound } from '../runtime/bound.js';
|
|
18
19
|
const EMBEDDED = 'EXISTS (SELECT 1 FROM memory_vectors v WHERE v.memory_id = memories.id)';
|
|
@@ -24,9 +25,10 @@ const noneExcluded = () => ({
|
|
|
24
25
|
quarantined: 0,
|
|
25
26
|
});
|
|
26
27
|
function excluded(db, tenantId) {
|
|
27
|
-
// SAFETY: `why` is one of the three CASE arms or NULL.
|
|
28
|
+
// SAFETY: `why` is one of the three CASE arms or NULL. The kind arm counts a `--kind superseded` row,
|
|
29
|
+
// which hippo remember saves with no successor link.
|
|
28
30
|
const counts = cachedStatement(db, `SELECT CASE WHEN ${quarantinedIn('')} THEN 'quarantined' ` +
|
|
29
|
-
`WHEN ${supersededIn(
|
|
31
|
+
`WHEN ${supersededIn()} OR ${supersededKindIn('')} THEN 'superseded' ` +
|
|
30
32
|
`WHEN ${kindIn('')} = 'archived' THEN 'archived' END AS why, COUNT(*) AS n ` +
|
|
31
33
|
`FROM memories WHERE tenant_id = ? AND NOT (${LIVE}) GROUP BY why`).all(tenantId);
|
|
32
34
|
const excluded = noneExcluded();
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
export declare const quarantinedIn: (t: string) => string;
|
|
2
|
-
export declare const supersededIn: (t: string) => string;
|
|
3
|
-
export declare const kindIn: (t: string) => string;
|
|
4
1
|
export declare const LIVE: string;
|
|
5
2
|
export declare const IN_PROJECT: string;
|
|
6
3
|
export declare const OPEN_PAIRS: string;
|
|
@@ -1,12 +1,5 @@
|
|
|
1
|
-
// The
|
|
2
|
-
|
|
3
|
-
import { QUARANTINE_SCOPE_PREFIX } from '../connectors/quarantine.js';
|
|
4
|
-
// SQL twins of isQuarantineScope and of the two halves of isLiveMemory, for columns under prefix
|
|
5
|
-
// `t`; substr compares bytes, as startsWith does, where LIKE would fold case.
|
|
6
|
-
export const quarantinedIn = (t) => `(${t}scope IS NOT NULL AND substr(${t}scope, 1, ${QUARANTINE_SCOPE_PREFIX.length}) = '${QUARANTINE_SCOPE_PREFIX}')`;
|
|
7
|
-
export const supersededIn = (t) => `(${t}superseded_by IS NOT NULL AND ${t}superseded_by <> '')`;
|
|
8
|
-
export const kindIn = (t) => `COALESCE(${t}kind, 'distilled')`;
|
|
9
|
-
const liveIn = (t) => `NOT ${supersededIn(t)} AND ${kindIn(t)} IN ('raw', 'distilled') AND NOT ${quarantinedIn(t)}`;
|
|
1
|
+
// The dashboard's live-row rules as SQL, which its reads and its sorted memory lists both filter by.
|
|
2
|
+
import { liveIn } from '../../../db/sql-predicates.js';
|
|
10
3
|
export const LIVE = liveIn('');
|
|
11
4
|
// Named parameters bound from one object: Node 22's node:sqlite skips every named slot, ?NNN too,
|
|
12
5
|
// on a positional bind.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// hippo.db's Reports group: the tenant reports, listings and usage reports the status verbs print.
|
|
2
2
|
import { withReadSnapshot } from '../../../db/index.js';
|
|
3
3
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
4
|
+
import { supersededIn } from '../../../db/sql-predicates.js';
|
|
4
5
|
import { summarizeTokenUse } from '../ledger/token-ledger.js';
|
|
5
6
|
import { summarizeFailures } from '../capture/failure-log.js';
|
|
6
7
|
import { keysetPage } from '../../../base/util/keyset.js';
|
|
@@ -29,7 +30,7 @@ function correctionLinks(db, page) {
|
|
|
29
30
|
src.created AS source_created
|
|
30
31
|
FROM memories old JOIN memories new ON new.id = old.superseded_by
|
|
31
32
|
LEFT JOIN memories src ON new.extracted_from <> '' AND src.id = new.extracted_from
|
|
32
|
-
WHERE old.
|
|
33
|
+
WHERE ${supersededIn('old.')}${range.sql}
|
|
33
34
|
ORDER BY old.created ASC, old.id ASC LIMIT ?`).all(...range.params, page.limit + 1);
|
|
34
35
|
const kept = keysetPage(rows, page.limit, (r) => ({ created: r.old_created, id: r.old_id }));
|
|
35
36
|
return {
|