hippo-memory 3.3.3 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/api/callers.d.ts +4 -1
- package/dist/api/context/context-plan.js +3 -0
- package/dist/api/context/context-pools.js +4 -1
- package/dist/api/context/user-rule.js +2 -1
- package/dist/api/dashboard/detail.js +2 -1
- package/dist/api/dashboard/live.d.ts +0 -4
- package/dist/api/dashboard/live.js +0 -8
- package/dist/api/memories/conflicts.d.ts +2 -1
- package/dist/api/memories/conflicts.js +3 -8
- package/dist/api/memories/promote.js +6 -2
- package/dist/api/objects/object-kinds.d.ts +1 -4
- package/dist/api/objects/object-kinds.js +1 -5
- package/dist/api/recall/compose.js +2 -1
- package/dist/api/recall/rank-stages.js +2 -1
- package/dist/api/recall/rows.js +2 -1
- package/dist/api/session-end.d.ts +3 -2
- package/dist/api/session-end.js +8 -4
- package/dist/api/sharing.d.ts +4 -3
- package/dist/api/sharing.js +2 -2
- package/dist/api/sleep/auto-sleep.d.ts +3 -2
- package/dist/api/sleep/auto-sleep.js +16 -3
- package/dist/api/sleep/one-sleep.d.ts +23 -0
- package/dist/api/sleep/one-sleep.js +53 -0
- package/dist/api/sleep/sleep-run.js +2 -1
- package/dist/api/sleep/sleep.d.ts +5 -0
- package/dist/api/sleep/sleep.js +2 -1
- package/dist/api/summary-parents.js +2 -1
- package/dist/base/config/config-keys.js +1 -0
- package/dist/base/config/config.d.ts +2 -0
- package/dist/base/config/config.js +1 -0
- package/dist/base/embeddings/local.d.ts +1 -1
- package/dist/base/embeddings/local.js +2 -3
- package/dist/base/memory/ambient.js +2 -1
- package/dist/base/memory/entry.d.ts +4 -1
- package/dist/base/memory/entry.js +7 -1
- package/dist/base/memory/half-life-plan.js +6 -4
- package/dist/base/memory/keep-rules.js +2 -1
- package/dist/base/memory/liveness.d.ts +7 -0
- package/dist/base/memory/liveness.js +6 -0
- package/dist/base/memory/markdown.js +15 -5
- package/dist/base/memory/memory-audit.js +2 -1
- package/dist/base/memory/memory-facts.d.ts +0 -2
- package/dist/base/memory/memory-facts.js +0 -2
- package/dist/base/memory/merged-row.js +3 -2
- package/dist/base/memory/rule-replaces.js +2 -1
- package/dist/base/memory/stale-mark.d.ts +9 -0
- package/dist/base/memory/stale-mark.js +18 -0
- package/dist/base/ports/store/capture.d.ts +2 -0
- package/dist/base/ports/store/conflicts.d.ts +5 -1
- package/dist/base/ports/store/conflicts.js +2 -0
- package/dist/base/ports/store/consolidation.d.ts +66 -2
- package/dist/base/ports/store/consolidation.js +5 -0
- package/dist/base/ports/store/context-reads.d.ts +2 -0
- package/dist/base/ports/store/objects.d.ts +15 -4
- package/dist/base/ports/store/sessions.d.ts +8 -2
- package/dist/base/ports/store/sessions.js +6 -0
- package/dist/base/ports/store/store-effects.d.ts +5 -0
- package/dist/base/ports/store.d.ts +1 -1
- package/dist/base/scope/live-memory.d.ts +7 -0
- package/dist/base/scope/live-memory.js +10 -0
- package/dist/base/scope/read-admission.d.ts +2 -0
- package/dist/base/scope/read-admission.js +9 -2
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/db/continuity-tables.d.ts +4 -1
- package/dist/db/continuity-tables.js +21 -12
- package/dist/db/migrate.js +2 -2
- package/dist/db/migrations/index.d.ts +1 -1
- package/dist/db/migrations/index.js +5 -1
- package/dist/db/migrations/v61.js +3 -3
- package/dist/db/migrations/v62.d.ts +3 -0
- package/dist/db/migrations/v62.js +43 -0
- package/dist/db/migrations/v63.d.ts +3 -0
- package/dist/db/migrations/v63.js +9 -0
- package/dist/db/required-objects.js +2 -1
- package/dist/db/sql-predicates.d.ts +22 -2
- package/dist/db/sql-predicates.js +27 -2
- package/dist/domain/agent-memories/apply.js +2 -1
- package/dist/domain/agent-memories/container-view.js +2 -1
- package/dist/domain/consolidate/conflicts.js +2 -1
- package/dist/domain/consolidate/dag-summary.d.ts +2 -0
- package/dist/domain/consolidate/dag.d.ts +3 -0
- package/dist/domain/consolidate/dag.js +53 -8
- package/dist/domain/consolidate/dedupe.js +2 -1
- package/dist/domain/consolidate/events.d.ts +2 -0
- package/dist/domain/consolidate/llm-passes.js +94 -20
- package/dist/domain/consolidate/merge.js +11 -10
- package/dist/domain/consolidate/retry-holds.d.ts +14 -0
- package/dist/domain/consolidate/retry-holds.js +26 -0
- package/dist/domain/consolidate/run.d.ts +8 -0
- package/dist/domain/consolidate/run.js +2 -0
- package/dist/domain/consolidate/tombstones.d.ts +3 -4
- package/dist/domain/consolidate/tombstones.js +3 -3
- package/dist/domain/graph/recall.js +3 -2
- package/dist/domain/learn/churn-stale.js +2 -1
- package/dist/domain/learn/extract.d.ts +10 -0
- package/dist/domain/learn/extract.js +10 -5
- package/dist/domain/learn/invalidation.d.ts +1 -1
- package/dist/domain/learn/invalidation.js +4 -11
- package/dist/domain/objects/decisions.d.ts +4 -7
- package/dist/domain/objects/decisions.js +1 -14
- package/dist/domain/objects/descriptor.d.ts +2 -0
- package/dist/domain/objects/lifecycle.js +8 -2
- package/dist/domain/search/as-of.js +3 -2
- package/dist/domain/sharing/share.d.ts +16 -4
- package/dist/domain/sharing/share.js +33 -17
- package/dist/entry/store-kit.js +2 -1
- package/dist/infra/local-embedder.js +14 -2
- package/dist/infra/spool-claims.d.ts +2 -2
- package/dist/infra/spool-claims.js +4 -3
- package/dist/infra/spool-import.js +1 -1
- package/dist/infra/spool-record.d.ts +3 -1
- package/dist/store/sqlite/capture/compaction-hooks.js +14 -6
- package/dist/store/sqlite/capture/compaction-rows.js +9 -2
- package/dist/store/sqlite/capture/spool-request-id.d.ts +3 -0
- package/dist/store/sqlite/capture/spool-request-id.js +12 -0
- package/dist/store/sqlite/connectors/quarantine.js +13 -4
- package/dist/store/sqlite/consolidation/consolidation-summaries.d.ts +4 -5
- package/dist/store/sqlite/consolidation/consolidation-summaries.js +46 -26
- package/dist/store/sqlite/consolidation/consolidation.js +7 -0
- package/dist/store/sqlite/consolidation/empty-extractions.d.ts +14 -0
- package/dist/store/sqlite/consolidation/empty-extractions.js +57 -0
- package/dist/store/sqlite/consolidation/rebuild-holds.d.ts +21 -0
- package/dist/store/sqlite/consolidation/rebuild-holds.js +68 -0
- package/dist/store/sqlite/consolidation/sleep-lock.d.ts +4 -0
- package/dist/store/sqlite/consolidation/sleep-lock.js +69 -0
- package/dist/store/sqlite/entries/entry-filter.js +3 -3
- package/dist/store/sqlite/entries/entry-reads.js +2 -1
- package/dist/store/sqlite/entries/entry-row.js +5 -1
- package/dist/store/sqlite/entries/entry-writes.js +9 -15
- package/dist/store/sqlite/entries/raw-archive.d.ts +2 -1
- package/dist/store/sqlite/entries/raw-archive.js +12 -3
- package/dist/store/sqlite/maintenance/agent-import-steps.js +7 -2
- package/dist/store/sqlite/maintenance/project-tags.js +4 -3
- package/dist/store/sqlite/recall/candidates.js +12 -11
- package/dist/store/sqlite/recall/search-rows.d.ts +3 -0
- package/dist/store/sqlite/recall/search-rows.js +26 -9
- package/dist/store/sqlite/records/object-rows.d.ts +2 -0
- package/dist/store/sqlite/records/object-rows.js +5 -1
- package/dist/store/sqlite/records/object-writes.d.ts +3 -0
- package/dist/store/sqlite/records/object-writes.js +43 -1
- package/dist/store/sqlite/reports/dashboard-lists.js +3 -2
- package/dist/store/sqlite/reports/dashboard-reads.js +5 -3
- package/dist/store/sqlite/reports/dashboard-sql.d.ts +0 -3
- package/dist/store/sqlite/reports/dashboard-sql.js +2 -9
- package/dist/store/sqlite/reports/reports.js +2 -1
- package/dist/store/sqlite/review/conflicts.js +10 -5
- package/dist/store/sqlite/runtime/rule-sql.d.ts +1 -0
- package/dist/store/sqlite/runtime/rule-sql.js +4 -1
- package/dist/store/sqlite/runtime/status-transitions.d.ts +98 -0
- package/dist/store/sqlite/runtime/status-transitions.js +105 -0
- package/dist/store/sqlite/sessions/goals.js +11 -12
- package/dist/store/sqlite/sessions/sessions.d.ts +1 -1
- package/dist/store/sqlite/sessions/sessions.js +20 -8
- package/dist/surface/capture/session-worker.d.ts +3 -2
- package/dist/surface/capture/session-worker.js +36 -10
- package/dist/surface/cli/api-context.js +2 -1
- package/dist/surface/cli/client.d.ts +6 -5
- package/dist/surface/cli/client.js +50 -19
- package/dist/surface/cli/conflicts.js +12 -1
- package/dist/surface/cli/continuity.js +4 -0
- package/dist/surface/cli/decisions.js +16 -17
- package/dist/surface/cli/hooks/session-end-worker.js +3 -2
- package/dist/surface/cli/install-steps.js +99 -13
- package/dist/surface/cli/memory-writes.js +3 -2
- package/dist/surface/cli/print.js +2 -1
- package/dist/surface/cli/sleep-render.js +7 -0
- package/dist/surface/cli/sleep.d.ts +3 -2
- package/dist/surface/cli/sleep.js +10 -4
- package/dist/surface/cli/transfer.js +6 -3
- package/dist/surface/cli/verbs/objects.js +2 -2
- package/dist/surface/dashboard/dashboard-actions.js +1 -1
- package/dist/surface/hooks-install/codex-wrapper.js +52 -11
- package/dist/surface/hooks-install/rules/hook-blocks.js +9 -2
- package/dist/surface/mcp/admin-tools.js +5 -2
- package/dist/surface/render/sleep-events.js +2 -0
- package/dist/surface/server/routes/decisions.js +2 -2
- package/dist/surface/server/sleep-child.js +5 -4
- package/dist/surface/server/sleep-offload.js +36 -25
- package/dist-ui/assets/index-Cst0ubfs.js +1 -0
- package/dist-ui/index.html +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/dist-ui/assets/index-Q9SY-1Bj.js +0 -1
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// A source's empty extraction mark, one `meta` row per source: {"digest": hash of its text, "at": ISO}.
|
|
2
|
+
// When a mark stands is decided in src/domain/consolidate/retry-holds.ts.
|
|
3
|
+
import { withWriteScope } from '../../../db/index.js';
|
|
4
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
5
|
+
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
6
|
+
const MARK_KEY = 'empty_extraction:';
|
|
7
|
+
// The first key past every mark key, as ';' follows ':'.
|
|
8
|
+
const MARK_END = 'empty_extraction;';
|
|
9
|
+
// substr is 1-based, so this is the first character of the id.
|
|
10
|
+
const ID_FROM = MARK_KEY.length + 1;
|
|
11
|
+
// A value that is not JSON with text digest and at reads as no mark; CASE keeps json_type off bad
|
|
12
|
+
// JSON, which AND would not.
|
|
13
|
+
const PAGE_SQL = `SELECT substr(key, ${ID_FROM}) AS id,
|
|
14
|
+
json_extract(value, '$.digest') AS digest, json_extract(value, '$.at') AS at
|
|
15
|
+
FROM meta
|
|
16
|
+
WHERE key > ? AND key < '${MARK_END}'
|
|
17
|
+
AND CASE WHEN json_valid(value)
|
|
18
|
+
THEN json_type(value, '$.digest') = 'text' AND json_type(value, '$.at') = 'text'
|
|
19
|
+
ELSE 0 END
|
|
20
|
+
ORDER BY key
|
|
21
|
+
LIMIT ?`;
|
|
22
|
+
/** One page of marks by source id, after the id `after`; a row that is not a mark is skipped. */
|
|
23
|
+
export function emptyExtractions(db, page) {
|
|
24
|
+
// SAFETY: the SQL names id, digest and at, and its WHERE keeps digest and at text.
|
|
25
|
+
const rows = cachedStatement(db, PAGE_SQL).all(`${MARK_KEY}${page.after ?? ''}`, page.limit + 1);
|
|
26
|
+
const marks = rows.slice(0, page.limit);
|
|
27
|
+
return { marks, next: rows.length > page.limit ? (marks.at(-1)?.id ?? null) : null };
|
|
28
|
+
}
|
|
29
|
+
// Writes only for a source row that is live, as the sleep that offers it reads it.
|
|
30
|
+
const MARK_SQL = `INSERT INTO meta(key, value)
|
|
31
|
+
SELECT ?, ?
|
|
32
|
+
WHERE EXISTS (SELECT 1 FROM memories
|
|
33
|
+
WHERE id = ? AND kind != 'archived' AND ${notSupersededIn()})
|
|
34
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`;
|
|
35
|
+
// A mark left for a source that is gone, archived or superseded would never be read again.
|
|
36
|
+
const SWEEP_SQL = `DELETE FROM meta
|
|
37
|
+
WHERE key > '${MARK_KEY}' AND key < '${MARK_END}'
|
|
38
|
+
AND NOT EXISTS (SELECT 1 FROM memories m
|
|
39
|
+
WHERE m.id = substr(meta.key, ${ID_FROM})
|
|
40
|
+
AND m.kind != 'archived' AND ${notSupersededIn('m.')})`;
|
|
41
|
+
/** Stores the marks and drops the cleared ones in one write, then drops the marks of sources that
|
|
42
|
+
* are no longer live. */
|
|
43
|
+
export function noteEmptyExtractions(db, { mark, clear }) {
|
|
44
|
+
if (mark.length === 0 && clear.length === 0)
|
|
45
|
+
return;
|
|
46
|
+
withWriteScope(db, 'empty_extraction', () => {
|
|
47
|
+
const put = cachedStatement(db, MARK_SQL);
|
|
48
|
+
for (const { id, digest, at } of mark) {
|
|
49
|
+
put.run(`${MARK_KEY}${id}`, JSON.stringify({ digest, at }), id);
|
|
50
|
+
}
|
|
51
|
+
const drop = cachedStatement(db, 'DELETE FROM meta WHERE key = ?');
|
|
52
|
+
for (const id of clear)
|
|
53
|
+
drop.run(`${MARK_KEY}${id}`);
|
|
54
|
+
cachedStatement(db, SWEEP_SQL).run();
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=empty-extractions.js.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type DatabaseSyncLike } from '../../../db/index.js';
|
|
2
|
+
import type { Consolidation } from '../../../base/ports/store/consolidation.js';
|
|
3
|
+
/** The meta key prefix of a summary's hold; the summary's id follows. */
|
|
4
|
+
export declare const REBUILD_HOLD_KEY = "rebuild_hold:";
|
|
5
|
+
type RebuildFailure = Parameters<NonNullable<Consolidation['noteRebuildFailures']>>[0][number];
|
|
6
|
+
/** SQL for the failure count of the hold row `alias`; 0 when it has none, is not JSON or holds no
|
|
7
|
+
* integer count, so one bad row never stops a rebuild. The nested CASE keeps json_type off bad
|
|
8
|
+
* JSON. */
|
|
9
|
+
export declare const holdFailuresSql: (alias: string) => string;
|
|
10
|
+
/** SQL for the ISO time the hold row `alias` ends; NULL when there is none. */
|
|
11
|
+
export declare const holdRetryAfterSql: (alias: string) => string;
|
|
12
|
+
/** How many dirty live summaries are held after `at`, from the hold keys alone so the read grows
|
|
13
|
+
* with held rows, not with the queue. */
|
|
14
|
+
export declare function heldSummaryCount(db: DatabaseSyncLike, at: string): number;
|
|
15
|
+
/** Ends the summary's hold; the settle that landed it calls this in its own write scope. */
|
|
16
|
+
export declare function clearRebuildHold(db: DatabaseSyncLike, summaryId: string): void;
|
|
17
|
+
/** Stores each failure's count and hold time in one write, then drops the holds of summaries that
|
|
18
|
+
* are no longer dirty. */
|
|
19
|
+
export declare function noteRebuildFailures(db: DatabaseSyncLike, failed: ReadonlyArray<RebuildFailure>): void;
|
|
20
|
+
export {};
|
|
21
|
+
//# sourceMappingURL=rebuild-holds.d.ts.map
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// A dirty summary's rebuild hold, one `meta` row per summary: {"failures": n, "retryAfter": ISO | null}.
|
|
2
|
+
// When a hold ends is decided in src/domain/consolidate/retry-holds.ts.
|
|
3
|
+
import { DAG_LEVEL } from '../../../base/memory/entry.js';
|
|
4
|
+
import { withWriteScope } from '../../../db/index.js';
|
|
5
|
+
import { assertTenantId } from '../../../base/scope/tenant-id.js';
|
|
6
|
+
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
7
|
+
/** The meta key prefix of a summary's hold; the summary's id follows. */
|
|
8
|
+
export const REBUILD_HOLD_KEY = 'rebuild_hold:';
|
|
9
|
+
// The first key past every hold key, as ';' follows ':'.
|
|
10
|
+
const REBUILD_HOLD_END = 'rebuild_hold;';
|
|
11
|
+
// substr is 1-based, so this is the first character of the id.
|
|
12
|
+
const ID_FROM = REBUILD_HOLD_KEY.length + 1;
|
|
13
|
+
/** SQL for the failure count of the hold row `alias`; 0 when it has none, is not JSON or holds no
|
|
14
|
+
* integer count, so one bad row never stops a rebuild. The nested CASE keeps json_type off bad
|
|
15
|
+
* JSON. */
|
|
16
|
+
export const holdFailuresSql = (alias) => `CASE WHEN json_valid(${alias}.value) THEN ` +
|
|
17
|
+
`CASE WHEN json_type(${alias}.value, '$.failures') = 'integer' ` +
|
|
18
|
+
`THEN json_extract(${alias}.value, '$.failures') ELSE 0 END ELSE 0 END`;
|
|
19
|
+
/** SQL for the ISO time the hold row `alias` ends; NULL when there is none. */
|
|
20
|
+
export const holdRetryAfterSql = (alias) => `CASE WHEN json_valid(${alias}.value) THEN json_extract(${alias}.value, '$.retryAfter') END`;
|
|
21
|
+
/** How many dirty live summaries are held after `at`, from the hold keys alone so the read grows
|
|
22
|
+
* with held rows, not with the queue. */
|
|
23
|
+
export function heldSummaryCount(db, at) {
|
|
24
|
+
// SAFETY: the SQL names n, the only key of the row.
|
|
25
|
+
const row = cachedStatement(db, `SELECT COUNT(*) AS n
|
|
26
|
+
FROM meta h JOIN memories m ON m.id = substr(h.key, ${ID_FROM})
|
|
27
|
+
WHERE h.key > '${REBUILD_HOLD_KEY}' AND h.key < '${REBUILD_HOLD_END}'
|
|
28
|
+
AND m.summary_dirty = 1 AND m.kind != 'archived'
|
|
29
|
+
AND ${holdRetryAfterSql('h')} > ?`).get(at);
|
|
30
|
+
return row.n;
|
|
31
|
+
}
|
|
32
|
+
/** Ends the summary's hold; the settle that landed it calls this in its own write scope. */
|
|
33
|
+
export function clearRebuildHold(db, summaryId) {
|
|
34
|
+
cachedStatement(db, 'DELETE FROM meta WHERE key = ?').run(`${REBUILD_HOLD_KEY}${summaryId}`);
|
|
35
|
+
}
|
|
36
|
+
// Writes only while the stored count is still the one the caller saw, and only for a summary
|
|
37
|
+
// that is dirty and live: a settle that ended the hold in between wins.
|
|
38
|
+
const NOTE_FAILURE_SQL = `INSERT INTO meta(key, value)
|
|
39
|
+
SELECT ?, ?
|
|
40
|
+
WHERE EXISTS (SELECT 1 FROM memories
|
|
41
|
+
WHERE id = ? AND tenant_id = ?
|
|
42
|
+
AND dag_level IN (${DAG_LEVEL.topic}, ${DAG_LEVEL.entity})
|
|
43
|
+
AND summary_dirty = 1 AND kind != 'archived')
|
|
44
|
+
AND COALESCE((SELECT ${holdFailuresSql('h')} FROM meta h WHERE h.key = ?), 0) = ?
|
|
45
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`;
|
|
46
|
+
// A hold left for a summary that is gone, archived or clean would hold it again once it is dirty.
|
|
47
|
+
const SWEEP_SQL = `DELETE FROM meta
|
|
48
|
+
WHERE key > '${REBUILD_HOLD_KEY}' AND key < '${REBUILD_HOLD_END}'
|
|
49
|
+
AND NOT EXISTS (SELECT 1 FROM memories m
|
|
50
|
+
WHERE m.id = substr(meta.key, ${ID_FROM})
|
|
51
|
+
AND m.summary_dirty = 1 AND m.kind != 'archived')`;
|
|
52
|
+
/** Stores each failure's count and hold time in one write, then drops the holds of summaries that
|
|
53
|
+
* are no longer dirty. */
|
|
54
|
+
export function noteRebuildFailures(db, failed) {
|
|
55
|
+
if (failed.length === 0)
|
|
56
|
+
return;
|
|
57
|
+
for (const item of failed)
|
|
58
|
+
assertTenantId('noteRebuildFailures', item.tenantId);
|
|
59
|
+
withWriteScope(db, 'rebuild_hold', () => {
|
|
60
|
+
const note = cachedStatement(db, NOTE_FAILURE_SQL);
|
|
61
|
+
for (const { id, tenantId, priorFailures, failures, retryAfter } of failed) {
|
|
62
|
+
const key = `${REBUILD_HOLD_KEY}${id}`;
|
|
63
|
+
note.run(key, JSON.stringify({ failures, retryAfter }), id, tenantId, key, priorFailures);
|
|
64
|
+
}
|
|
65
|
+
cachedStatement(db, SWEEP_SQL).run();
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=rebuild-holds.js.map
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Consolidation } from '../../../base/ports/store/consolidation.js';
|
|
2
|
+
/** The claim members of the folder's Consolidation group; the closure keeps the lock its sleep holds. */
|
|
3
|
+
export declare function sleepClaim(hippoRoot: string): Required<Pick<Consolidation, 'claimSleep' | 'releaseSleep'>>;
|
|
4
|
+
//# sourceMappingURL=sleep-lock.d.ts.map
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// A sleep's claim on the store: a file lock beside hippo.db that every process on the folder honours,
|
|
2
|
+
// freed by the operating system when its holder dies, so no age or pid check can break a live sleep.
|
|
3
|
+
import * as fs from 'node:fs';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import { takeFileLock } from '../../../host/file-lock.js';
|
|
6
|
+
import { errorFields, errorMessage, log } from '../../../base/util/log.js';
|
|
7
|
+
import { answered } from '../runtime/bound.js';
|
|
8
|
+
const SLEEP_LOCK_FILE = 'sleep.lock.db';
|
|
9
|
+
// A folder that cannot keep a lock warns once for the process, not once per sleep.
|
|
10
|
+
const warnedRoots = new Set();
|
|
11
|
+
/** When the holder took the claim: the mtime it stamped on the lock file. */
|
|
12
|
+
function claimedSince(file) {
|
|
13
|
+
// A file gone under a live holder, or one that cannot be read, has no stamp left, so the time is now.
|
|
14
|
+
try {
|
|
15
|
+
return (fs.statSync(file, { throwIfNoEntry: false })?.mtime ?? new Date()).toISOString();
|
|
16
|
+
}
|
|
17
|
+
catch (err) {
|
|
18
|
+
log.warn(`the sleep claim on ${file} could not be read: ${errorMessage(err)}`, errorFields(err));
|
|
19
|
+
return new Date().toISOString();
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
// By path, never by descriptor: a descriptor closed here would drop this process's other locks.
|
|
23
|
+
function stamp(file) {
|
|
24
|
+
try {
|
|
25
|
+
const now = new Date();
|
|
26
|
+
fs.utimesSync(file, now, now);
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
log.warn(`the sleep claim on ${file} was not stamped, so a refused sleep may name an older time: ` +
|
|
30
|
+
errorMessage(err), errorFields(err));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function warnUnusable(hippoRoot, cause) {
|
|
34
|
+
if (warnedRoots.has(hippoRoot))
|
|
35
|
+
return;
|
|
36
|
+
warnedRoots.add(hippoRoot);
|
|
37
|
+
log.warn(`the sleep claim could not be taken on ${hippoRoot}, so sleeps there run unguarded: ` +
|
|
38
|
+
errorMessage(cause), errorFields(cause));
|
|
39
|
+
}
|
|
40
|
+
/** The claim members of the folder's Consolidation group; the closure keeps the lock its sleep holds. */
|
|
41
|
+
export function sleepClaim(hippoRoot) {
|
|
42
|
+
// Joined at the claim, so a store built on a root that is no path still opens.
|
|
43
|
+
const lockFile = () => path.join(hippoRoot, SLEEP_LOCK_FILE);
|
|
44
|
+
let held = null;
|
|
45
|
+
return {
|
|
46
|
+
claimSleep: () => answered(() => {
|
|
47
|
+
const file = lockFile();
|
|
48
|
+
const take = takeFileLock(file);
|
|
49
|
+
if (take.kind === 'held')
|
|
50
|
+
return { kind: 'held', since: claimedSince(file) };
|
|
51
|
+
if (take.kind === 'unusable') {
|
|
52
|
+
warnUnusable(hippoRoot, take.cause);
|
|
53
|
+
return { kind: 'taken' };
|
|
54
|
+
}
|
|
55
|
+
held = take.lock;
|
|
56
|
+
stamp(file);
|
|
57
|
+
return { kind: 'taken' };
|
|
58
|
+
}),
|
|
59
|
+
releaseSleep: () => answered(() => {
|
|
60
|
+
try {
|
|
61
|
+
held?.release();
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
held = null;
|
|
65
|
+
}
|
|
66
|
+
}),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=sleep-lock.js.map
|
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
// memory/entry-filter.ts is the predicate half.
|
|
4
4
|
import { keysetPage } from '../../../base/util/keyset.js';
|
|
5
5
|
import { ALL_TENANTS } from '../../../base/util/tenant.js';
|
|
6
|
+
import { notSupersededAt } from '../../../db/sql-predicates.js';
|
|
6
7
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
7
8
|
import { entriesPage, entryPageOrder, entryRange, } from '../runtime/keyset.js';
|
|
8
9
|
import { MEMORY_SELECT_COLUMNS, rowToEntry } from '../runtime/rows.js';
|
|
10
|
+
import { tagsOf } from '../runtime/rule-sql.js';
|
|
9
11
|
import { tenantAnd, TENANT_IS } from '../runtime/tenant.js';
|
|
10
12
|
/** The least string above every string that starts with `prefix` in byte order, or null when
|
|
11
13
|
* none is. */
|
|
@@ -35,8 +37,6 @@ function startsWith(column, prefix) {
|
|
|
35
37
|
? { sql: `${column} >= ?`, params: [prefix] }
|
|
36
38
|
: { sql: `${column} >= ? AND ${column} < ?`, params: [prefix, upper] };
|
|
37
39
|
}
|
|
38
|
-
// A tags column that is not a JSON array reads as no tags, as parseJsonArray reads it.
|
|
39
|
-
const tagsOf = (column) => `CASE WHEN NOT json_valid(${column}) THEN '[]' WHEN json_type(${column}) = 'array' THEN ${column} ELSE '[]' END`;
|
|
40
40
|
const TERMS = {
|
|
41
41
|
ids: (ids, column) => inList(column('id'), ids),
|
|
42
42
|
origins: (origins, column) => inList(column('origin_project'), origins),
|
|
@@ -48,7 +48,7 @@ const TERMS = {
|
|
|
48
48
|
params: [tag],
|
|
49
49
|
}),
|
|
50
50
|
kinds: (kinds, column) => inList(`COALESCE(${column('kind')}, 'distilled')`, kinds),
|
|
51
|
-
live: (_live, column) => ({ sql:
|
|
51
|
+
live: (_live, column) => ({ sql: notSupersededAt(column('superseded_by')), params: [] }),
|
|
52
52
|
sessionId: (sessionId, column) => ({
|
|
53
53
|
sql: `${column('source_session_id')} = ?`,
|
|
54
54
|
params: [sessionId],
|
|
@@ -3,6 +3,7 @@ import { MEMORY_SELECT_COLUMNS, rowToEntry, parseJsonArray, } from '../runtime/r
|
|
|
3
3
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
4
4
|
import { escapeLike } from '../../../base/util/escape.js';
|
|
5
5
|
import { scopeAdmitSql } from '../runtime/rule-sql.js';
|
|
6
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
6
7
|
import { chunked } from '../../../base/util/chunked.js';
|
|
7
8
|
import { byIdOrder, FRESH_RAW_ORDER, orderSql } from '../../rules/order.js';
|
|
8
9
|
import { ALL_TENANTS, DEFAULT_TENANT_ID } from '../../../base/util/tenant.js';
|
|
@@ -193,7 +194,7 @@ export function countByOrigin(db, tenantId, sourcePrefix) {
|
|
|
193
194
|
// The unit separator sorts below every character of a time or an id, so the joined string orders
|
|
194
195
|
// as (created, id).
|
|
195
196
|
return cachedStatement(db, `SELECT origin_project AS origin, COUNT(*) AS live, SUM(substr(source, 1, ?) = ?) AS prefixed, ` +
|
|
196
|
-
`MAX(created) AS newest FROM memories WHERE tenant_id = ? AND
|
|
197
|
+
`MAX(created) AS newest FROM memories WHERE tenant_id = ? AND ${notSupersededIn()} ` +
|
|
197
198
|
`GROUP BY origin_project ORDER BY MIN(created || char(31) || id)`).all(sourcePrefix.length, sourcePrefix, tenantId);
|
|
198
199
|
}
|
|
199
200
|
/** The ids among `ids` that a row of any tenant holds, one query per chunk. */
|
|
@@ -53,6 +53,10 @@ export function updateMemoryRows(db, entries) {
|
|
|
53
53
|
`UPDATE memories SET ${set}, updated_at = datetime('now') FROM v WHERE memories.id = v.id`).run(...chunk.flatMap(memoryRowValues));
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
|
+
// Storage spells "no successor" one way: a blank link is stored as none. Whitespace stays a link.
|
|
57
|
+
function storedLink(link) {
|
|
58
|
+
return link === '' ? null : (link ?? null);
|
|
59
|
+
}
|
|
56
60
|
/** Bind values for WRITTEN_COLUMNS, in its order. */
|
|
57
61
|
function memoryRowValues(entry) {
|
|
58
62
|
// Markdown rebuilds, imports and add-ons pass stamps as written; only toISOString text sorts by
|
|
@@ -82,7 +86,7 @@ function memoryRowValues(entry) {
|
|
|
82
86
|
entry.traceOutcome ?? null,
|
|
83
87
|
entry.sourceSessionId ?? null,
|
|
84
88
|
entry.validFrom ?? created,
|
|
85
|
-
entry.supersededBy
|
|
89
|
+
storedLink(entry.supersededBy),
|
|
86
90
|
entry.extractedFrom ?? null,
|
|
87
91
|
entry.dagLevel ?? 0,
|
|
88
92
|
entry.dagParentId ?? null,
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// and the shared entry write path.
|
|
3
3
|
import { markRetrieved } from '../../../base/memory/entry.js';
|
|
4
4
|
import { entryAfterOutcome } from '../../../base/memory/keep-rules.js';
|
|
5
|
-
import { closeHippoDb, withWriteScope,
|
|
5
|
+
import { closeHippoDb, withWriteScope, } from '../../../db/index.js';
|
|
6
6
|
import { rethrowIfStoreBlocked } from '../../../base/util/store-blocked.js';
|
|
7
7
|
import { RejectedValueError, ConflictError, CrossTenantIdError, memoryNotFoundText, NotFoundError, } from '../../../base/errors/api-errors.js';
|
|
8
8
|
import { markSummaryDirtyInTx, markSummariesDirtyInTx } from './summary-dirty.js';
|
|
@@ -16,7 +16,7 @@ import { mirrorBestEffort, writeMarkdownMirror, purgeMirrorBestEffort, removeEnt
|
|
|
16
16
|
import { deferDueMirrors, markMirrorsDue } from '../folder-files/due-mirrors.js';
|
|
17
17
|
import { bindHandleClock } from '../runtime/handle-clock.js';
|
|
18
18
|
import { CLI_ACTOR } from '../../../base/scope/actor.js';
|
|
19
|
-
import { archiveRawMemory } from './raw-archive.js';
|
|
19
|
+
import { archiveRawMemory, markMirrorCleaned } from './raw-archive.js';
|
|
20
20
|
import { assertWriteRight, readerReaches, targetReader } from '../../../base/scope/recall-scope.js';
|
|
21
21
|
import { recordTraceOutcome } from '../recall/recall-trace.js';
|
|
22
22
|
import { deleteEntryCore } from './delete-and-batch.js';
|
|
@@ -25,6 +25,7 @@ import { openStore } from '../runtime/handles.js';
|
|
|
25
25
|
import { bound } from '../runtime/bound.js';
|
|
26
26
|
import { readMemoryReach } from '../runtime/tenant-lookup.js';
|
|
27
27
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
28
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
28
29
|
/** Writes a promoted copy on the caller's open store; with `auditAs` the `promote` audit row
|
|
29
30
|
* joins the INSERT's SAVEPOINT, so both or neither. */
|
|
30
31
|
export function writePromotedEntry(db, files, entry, opts) {
|
|
@@ -278,7 +279,7 @@ export function stampOriginProjects(db, tenantId, rows) {
|
|
|
278
279
|
/** Renames a live row's source from `from` to `to`; the count is 0 when another writer moved or
|
|
279
280
|
* superseded it first. */
|
|
280
281
|
export function renameEntrySource(db, { tenantId, id, from, to }) {
|
|
281
|
-
const moved = cachedStatement(db, `UPDATE memories SET source = ? WHERE id = ? AND tenant_id = ? AND source = ? AND
|
|
282
|
+
const moved = cachedStatement(db, `UPDATE memories SET source = ? WHERE id = ? AND tenant_id = ? AND source = ? AND ${notSupersededIn()}`).run(to, id, tenantId, from);
|
|
282
283
|
return Number(moved.changes ?? 0);
|
|
283
284
|
}
|
|
284
285
|
/** Renames a row's source and, when `origin` is not null, its project; the count is 0 when the
|
|
@@ -291,13 +292,13 @@ export function renameEntrySourceAndOrigin(db, tenantId, id, options) {
|
|
|
291
292
|
}
|
|
292
293
|
/** Marks `id` superseded by `newId`; false when another writer superseded it first. */
|
|
293
294
|
export function supersedeEntry(db, { tenantId, id, newId }) {
|
|
294
|
-
const result = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND
|
|
295
|
+
const result = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND ${notSupersededIn()}`).run(newId, id, tenantId);
|
|
295
296
|
return Number(result.changes ?? 0) !== 0;
|
|
296
297
|
}
|
|
297
298
|
/** Marks `id` superseded by `newId` in the caller's transaction, with the supersede audit row and
|
|
298
299
|
* its summary parent's dirty mark; false when the row is gone or another writer got there first. */
|
|
299
300
|
export function supersedeInTx(db, { tenantId, id, newId, actor }) {
|
|
300
|
-
const won = cachedStatement(db,
|
|
301
|
+
const won = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND ${notSupersededIn()} ` +
|
|
301
302
|
'RETURNING dag_parent_id').get(newId, id, tenantId);
|
|
302
303
|
if (!won)
|
|
303
304
|
return false;
|
|
@@ -486,8 +487,8 @@ function commitSupersede(db, write) {
|
|
|
486
487
|
auditingRefusal(db, actor, () => withWriteScope(db, 'supersede', () => {
|
|
487
488
|
assertMayChange(db, write, oldId);
|
|
488
489
|
// No row means another writer got there first.
|
|
489
|
-
const won = cachedStatement(db,
|
|
490
|
-
'dag_parent_id').get(successor.id, oldId, tenantId);
|
|
490
|
+
const won = cachedStatement(db, `UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND ${notSupersededIn()} ` +
|
|
491
|
+
'RETURNING dag_parent_id').get(successor.id, oldId, tenantId);
|
|
491
492
|
if (!won)
|
|
492
493
|
throw new ConflictError(`Memory ${oldId} already superseded by another writer`);
|
|
493
494
|
// After the CAS, so a lost race marks no parent for rebuild.
|
|
@@ -511,13 +512,6 @@ function cleanArchivedMirrors(db, files, id) {
|
|
|
511
512
|
`${errorMessage(mirrorErr)}`, errorFields(mirrorErr));
|
|
512
513
|
return;
|
|
513
514
|
}
|
|
514
|
-
|
|
515
|
-
cachedStatement(db, `UPDATE raw_archive SET mirror_cleaned_at = ? WHERE memory_id = ?`).run(files.clock().toISOString(), id);
|
|
516
|
-
}
|
|
517
|
-
catch (err) {
|
|
518
|
-
if (!isSqliteBusy(err))
|
|
519
|
-
throw err;
|
|
520
|
-
log.warnThenDebug('archive-mirror-stamp-busy', `archived ${id}; the store was busy, so the mirror reaper will re-check it on the next open`, errorFields(err));
|
|
521
|
-
}
|
|
515
|
+
markMirrorCleaned(db, id, files.clock().toISOString());
|
|
522
516
|
}
|
|
523
517
|
//# sourceMappingURL=entry-writes.js.map
|
|
@@ -7,7 +7,8 @@ export interface ArchiveOptions {
|
|
|
7
7
|
* audit row included. */
|
|
8
8
|
afterArchive?: (db: DatabaseSyncLike, archivedMemoryId: string) => void;
|
|
9
9
|
}
|
|
10
|
-
/** Stamps the archived copy of `memoryId` as having its markdown mirror cleaned at `at
|
|
10
|
+
/** Stamps the archived copy of `memoryId` as having its markdown mirror cleaned at `at`; a busy store
|
|
11
|
+
* leaves it unstamped for the reaper, since the change it follows has already committed. */
|
|
11
12
|
export declare function markMirrorCleaned(db: DatabaseSyncLike, memoryId: string, at: string): void;
|
|
12
13
|
/** archiveRawMemory for every row in `ids` in one write scope, in the same few statements for
|
|
13
14
|
* one row or many; throws, archiving none, if any is missing or not raw. Returns the
|
|
@@ -1,16 +1,25 @@
|
|
|
1
1
|
import { DEFAULT_TENANT_ID } from '../../../base/util/tenant.js';
|
|
2
2
|
import { memoryNotFoundText, NotFoundError } from '../../../base/errors/api-errors.js';
|
|
3
3
|
import { rawRemovalRefusal } from '../../../base/errors/raw-append-only.js';
|
|
4
|
-
import { isFtsAvailable, withWriteScope } from '../../../db/index.js';
|
|
4
|
+
import { isFtsAvailable, isSqliteBusy, withWriteScope } from '../../../db/index.js';
|
|
5
5
|
import { appendAuditEvents } from '../audit/audit.js';
|
|
6
6
|
import { reportAuditWriteFailure } from '../../../base/records/audit-report.js';
|
|
7
7
|
import { markSummaryDirtyInTx } from './summary-dirty.js';
|
|
8
8
|
import { CLI_ACTOR } from '../../../base/scope/actor.js';
|
|
9
9
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
10
10
|
import { handleStamp } from '../runtime/handle-clock.js';
|
|
11
|
-
|
|
11
|
+
import { errorFields, log } from '../../../base/util/log.js';
|
|
12
|
+
/** Stamps the archived copy of `memoryId` as having its markdown mirror cleaned at `at`; a busy store
|
|
13
|
+
* leaves it unstamped for the reaper, since the change it follows has already committed. */
|
|
12
14
|
export function markMirrorCleaned(db, memoryId, at) {
|
|
13
|
-
|
|
15
|
+
try {
|
|
16
|
+
cachedStatement(db, `UPDATE raw_archive SET mirror_cleaned_at = ? WHERE memory_id = ?`).run(at, memoryId);
|
|
17
|
+
}
|
|
18
|
+
catch (err) {
|
|
19
|
+
if (!isSqliteBusy(err))
|
|
20
|
+
throw err;
|
|
21
|
+
log.warnThenDebug('archive-mirror-stamp-busy', `removed raw ${memoryId}; the store was busy, so the mirror reaper will re-check it on the next open`, errorFields(err));
|
|
22
|
+
}
|
|
14
23
|
}
|
|
15
24
|
// Every id list binds as one JSON array, so each statement's text is the same for one row or many.
|
|
16
25
|
const IDS = 'SELECT value FROM json_each(?)';
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// One container's planned steps on the caller's write scope: the rows they name are re-read first,
|
|
2
2
|
// and a plan whose rows moved on since the sync read them writes nothing.
|
|
3
3
|
import { SYNC_ACTOR } from '../../../base/memory/agent-memory-tools.js';
|
|
4
|
+
import { isSuperseded } from '../../../base/memory/liveness.js';
|
|
4
5
|
import { chunked } from '../../../base/util/chunked.js';
|
|
5
6
|
import { READ_PAGE_ROWS, rowsInPages } from '../../../base/util/pages.js';
|
|
6
7
|
import { appendAuditEvents } from '../audit/audit.js';
|
|
@@ -48,8 +49,12 @@ function standingRows(db, plan) {
|
|
|
48
49
|
for (const row of readEntriesInIdOrder(db, ids, plan.tenantId))
|
|
49
50
|
rows.set(row.id, row);
|
|
50
51
|
}
|
|
51
|
-
const adoptable = plan.steps.every((step) =>
|
|
52
|
-
(
|
|
52
|
+
const adoptable = plan.steps.every((step) => {
|
|
53
|
+
if (step.kind !== 'adopt')
|
|
54
|
+
return true;
|
|
55
|
+
const row = rows.get(step.id);
|
|
56
|
+
return row !== undefined && row.source === step.from && !isSuperseded(row);
|
|
57
|
+
});
|
|
53
58
|
return adoptable ? rows : null;
|
|
54
59
|
}
|
|
55
60
|
/** A prefix under another listed one reads only rows that one already returns. */
|
|
@@ -15,6 +15,7 @@ import { setAsideRow } from '../entries/set-aside.js';
|
|
|
15
15
|
import { bound } from '../runtime/bound.js';
|
|
16
16
|
import { listEntriesWhere } from '../entries/entry-filter.js';
|
|
17
17
|
import { META_KEY } from '../../../db/meta.js';
|
|
18
|
+
import { isSuperseded } from '../../../base/memory/liveness.js';
|
|
18
19
|
const TX_LABEL = 'merge_projects';
|
|
19
20
|
/** One transaction, the store backed up first, with mirrors following the commit. */
|
|
20
21
|
function rewrite(db, files, { tenantId, backupLabel, body }) {
|
|
@@ -57,7 +58,7 @@ function fold(db, files, tenantId, { from, into }) {
|
|
|
57
58
|
// yet to read.
|
|
58
59
|
for (const row of rowsWhere(db, { tenantId, origins: [from] })) {
|
|
59
60
|
const tag = toolTag(row.source);
|
|
60
|
-
const live = isImport(row.source) && !row
|
|
61
|
+
const live = isImport(row.source) && !isSuperseded(row) && row.kind !== 'raw';
|
|
61
62
|
const copy = live && tag !== null && held.has(noteKey(row.source));
|
|
62
63
|
if (copy &&
|
|
63
64
|
setAsideRow(db, {
|
|
@@ -103,7 +104,7 @@ function setAsideCopies(db, files, tenantId, copies) {
|
|
|
103
104
|
for (const { id, origin } of copies) {
|
|
104
105
|
const row = rows.get(id);
|
|
105
106
|
const tag = row ? toolTag(row.source) : null;
|
|
106
|
-
if (!row || tag === null || row
|
|
107
|
+
if (!row || tag === null || isSuperseded(row) || (row.originProject ?? null) !== origin) {
|
|
107
108
|
continue;
|
|
108
109
|
}
|
|
109
110
|
if (setAsideRow(db, { tag, row, why: 'project-repair', rules: files }).kind === 'dormant') {
|
|
@@ -116,7 +117,7 @@ function setAsideCopies(db, files, tenantId, copies) {
|
|
|
116
117
|
* one re-tagged since stays as it is. */
|
|
117
118
|
function userGlobalMerges(db, tenantId, ids) {
|
|
118
119
|
const rows = readEntriesByIds(db, ids, tenantId);
|
|
119
|
-
return new Map([...rows].filter(([, e]) => !e
|
|
120
|
+
return new Map([...rows].filter(([, e]) => !isSuperseded(e) && e.originProject === ''));
|
|
120
121
|
}
|
|
121
122
|
/** A live row to dormant storage, restorable, as sleep's dormant move does. */
|
|
122
123
|
function retire(db, { files, row, actor, now }) {
|
|
@@ -5,8 +5,9 @@ import { DECAY_KEY_HALF_LIVES, DECAY_KEY_SLACK_DAYS, DECAY_KEY_SQL, DECAY_KEY_WH
|
|
|
5
5
|
import { withReadSnapshot } from '../../../db/index.js';
|
|
6
6
|
import { sortableStamp } from '../../../base/memory/created-stamp.js';
|
|
7
7
|
import { cachedStatement } from '../../../db/statement-cache.js';
|
|
8
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
8
9
|
import { MEMORY_SELECT_COLUMNS, rowToEntry, parseJsonArray, } from '../runtime/rows.js';
|
|
9
|
-
import { pickRarestFtsQuery,
|
|
10
|
+
import { pickRarestFtsQuery, listRecallEntriesWithout } from './search-rows.js';
|
|
10
11
|
import { oldestFirst } from '../../rules/order.js';
|
|
11
12
|
import { tenantAnd } from '../runtime/tenant.js';
|
|
12
13
|
import { rowsHoldingTexts } from '../sharing/copy-rows.js';
|
|
@@ -16,7 +17,12 @@ import { SCHEMA_FIT_WINDOW_ROWS } from '../../rules/windows.js';
|
|
|
16
17
|
/** The counts over `rows`; the caller adds the conflict and vector counts. */
|
|
17
18
|
function tallyStatus(rows, now, atRiskBelow, rules) {
|
|
18
19
|
const byLayer = { [Layer.Buffer]: 0, [Layer.Episodic]: 0, [Layer.Semantic]: 0, [Layer.Trace]: 0 };
|
|
19
|
-
const byConfidence = {
|
|
20
|
+
const byConfidence = {
|
|
21
|
+
verified: 0,
|
|
22
|
+
observed: 0,
|
|
23
|
+
inferred: 0,
|
|
24
|
+
stale: 0,
|
|
25
|
+
};
|
|
20
26
|
const sums = { total: 0, strength: 0, pinned: 0, atRisk: 0, agedOut: 0 };
|
|
21
27
|
for (const row of rows) {
|
|
22
28
|
const strength = calculateStrength(row, now, rules.strength);
|
|
@@ -43,8 +49,8 @@ function tallyStatus(rows, now, atRiskBelow, rules) {
|
|
|
43
49
|
avgStrength: total > 0 ? sums.strength / total : 0,
|
|
44
50
|
};
|
|
45
51
|
}
|
|
46
|
-
const AMBIENT_SCOPED =
|
|
47
|
-
const AMBIENT_PINNED =
|
|
52
|
+
const AMBIENT_SCOPED = `${notSupersededIn()} AND tenant_id = ?`;
|
|
53
|
+
const AMBIENT_PINNED = `pinned = 1 AND ${notSupersededIn()}`;
|
|
48
54
|
/** A stamp whose text does not sort by time; idx_memories_created_drift (migration v51) holds
|
|
49
55
|
* the live ones. */
|
|
50
56
|
const CREATED_DRIFTED = "(length(created) <> 24 OR created NOT LIKE '%Z')";
|
|
@@ -181,12 +187,7 @@ export function listAmbientCandidates(db, tenantId, { recentNeeded, judge, recal
|
|
|
181
187
|
function promptRecallEntries(db, tenantId, recall) {
|
|
182
188
|
const ftsQuery = pickRarestFtsQuery(db, recall.terms);
|
|
183
189
|
return ftsQuery
|
|
184
|
-
?
|
|
185
|
-
limit: recall.limit,
|
|
186
|
-
tenantId,
|
|
187
|
-
includeSuperseded: false,
|
|
188
|
-
ownScope: recall.ownScope,
|
|
189
|
-
})
|
|
190
|
+
? listRecallEntriesWithout(db, ftsQuery, { limit: recall.limit, tenantId, includeSuperseded: false, ownScope: recall.ownScope }, recall.excludeTags ?? [])
|
|
190
191
|
: [];
|
|
191
192
|
}
|
|
192
193
|
// calculateStrength's decay exponent with its reward factor under `model`: the strongest rows,
|
|
@@ -199,7 +200,7 @@ const decayRankSql = (model) => `pinned DESC,
|
|
|
199
200
|
/** The shared WHERE of a context read: live tenant rows, then `extra` predicates, then the scope
|
|
200
201
|
* and origin rules. */
|
|
201
202
|
export function contextRowsWhere(tenantId, filter, extra = []) {
|
|
202
|
-
const where = ['tenant_id = ?',
|
|
203
|
+
const where = ['tenant_id = ?', notSupersededIn(), ...extra];
|
|
203
204
|
const params = [tenantId];
|
|
204
205
|
if (filter.exactScope) {
|
|
205
206
|
where.push('scope = ?');
|
|
@@ -11,6 +11,9 @@ export declare function listSearchEntries(db: DatabaseSyncLike, query: string, l
|
|
|
11
11
|
* '%:private:%'` runs before the LIMIT so private rows cannot starve admitted ones;
|
|
12
12
|
* `passesScopeFilterForRecall` is the exact post-filter. */
|
|
13
13
|
export declare function listRecallEntries(db: DatabaseSyncLike, query: string, options: RecallSearchEntriesOptions): MemoryEntry[];
|
|
14
|
+
/** `listRecallEntries` with rows tagged `excludeTags` cut before the limit, which the recall port
|
|
15
|
+
* has no option for. */
|
|
16
|
+
export declare function listRecallEntriesWithout(db: DatabaseSyncLike, query: string, options: RecallSearchEntriesOptions, excludeTags: readonly string[]): MemoryEntry[];
|
|
14
17
|
/** The rows nearest `queryVector` that pass `spec`, nearest first; `index` proposes them when it
|
|
15
18
|
* can. */
|
|
16
19
|
export declare function listVectorCandidateEntries(db: DatabaseSyncLike, queryVector: readonly number[], spec: VectorCandidateSpec, index?: VectorIndex): Promise<MemoryEntry[]>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { isFtsAvailable } from '../../../db/index.js';
|
|
2
2
|
import { cjkIndexTerms, partitionCjkTerms, tokenize } from '../../../base/util/tokenize.js';
|
|
3
3
|
import { recallScopeFilter } from '../../../base/scope/recall-scope.js';
|
|
4
|
-
import { originInSql, scopeAdmitSql } from '../runtime/rule-sql.js';
|
|
4
|
+
import { originInSql, scopeAdmitSql, tagsOf } from '../runtime/rule-sql.js';
|
|
5
5
|
import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, spreadCjkTerms, } from '../../../base/rank/prompt-recall.js';
|
|
6
6
|
import { errorMessage, log, errorFields } from '../../../base/util/log.js';
|
|
7
7
|
import { topVectorMatches } from '../../../db/vector-match.js';
|
|
@@ -11,6 +11,7 @@ import { cachedStatement } from '../../../db/statement-cache.js';
|
|
|
11
11
|
import { escapeLike } from '../../../base/util/escape.js';
|
|
12
12
|
import { ftsAnyPhrase } from './fts-phrase.js';
|
|
13
13
|
import { placeholders } from '../../../db/sql.js';
|
|
14
|
+
import { notSupersededIn } from '../../../db/sql-predicates.js';
|
|
14
15
|
import { VECTOR_CANDIDATE_LIMIT, } from '../../../base/ports/store/vectors.js';
|
|
15
16
|
import { ALL_TENANTS } from '../../../base/util/tenant.js';
|
|
16
17
|
import { tenantAnd } from '../runtime/tenant.js';
|
|
@@ -46,10 +47,20 @@ function withProject(scope, col, origin) {
|
|
|
46
47
|
params: [...scope.params, ...originProjects],
|
|
47
48
|
};
|
|
48
49
|
}
|
|
50
|
+
// In SQL, not after the window cut, so excluded rows cannot fill the LIMIT that kept rows need.
|
|
51
|
+
function withoutTags(scope, col, excludeTags) {
|
|
52
|
+
if (excludeTags.length === 0)
|
|
53
|
+
return scope;
|
|
54
|
+
return {
|
|
55
|
+
sql: `${scope.sql} AND NOT EXISTS (SELECT 1 FROM json_each(${tagsOf(`${col}tags_json`)}) ` +
|
|
56
|
+
`WHERE value IN (${placeholders(excludeTags.length)}))`,
|
|
57
|
+
params: [...scope.params, ...excludeTags],
|
|
58
|
+
};
|
|
59
|
+
}
|
|
49
60
|
const FTS_QUERY_SYNTAX_RE = /fts5: syntax error|unterminated string/i;
|
|
50
61
|
function listSearchRows(db, { query, limit, tenantId, options = {} }) {
|
|
51
|
-
const { scopeFilter, includeSuperseded = true, originProjects } = options;
|
|
52
|
-
const p = searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjects);
|
|
62
|
+
const { scopeFilter, includeSuperseded = true, originProjects, excludeTags = [] } = options;
|
|
63
|
+
const p = searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjects, excludeTags);
|
|
53
64
|
const terms = Array.from(new Set(tokenize(query)));
|
|
54
65
|
if (terms.length === 0) {
|
|
55
66
|
// LIMIT here too, so every candidate path honours the caller's cap.
|
|
@@ -89,7 +100,7 @@ function interleaveRows(a, b, limit) {
|
|
|
89
100
|
}
|
|
90
101
|
return [...out.values()];
|
|
91
102
|
}
|
|
92
|
-
function searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjects) {
|
|
103
|
+
function searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjects, excludeTags) {
|
|
93
104
|
// One tenant reads through the composite idx_memories_tenant_created.
|
|
94
105
|
const one = tenantId !== ALL_TENANTS;
|
|
95
106
|
const tenantPredicate = tenantAnd(tenantId, 'm.tenant_id = ?').sql;
|
|
@@ -101,11 +112,11 @@ function searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjec
|
|
|
101
112
|
// Tenant-only path: for every tenant `tenantOnlyPredicate` is '', so the archived clause switches
|
|
102
113
|
// between ' AND' and ' WHERE' by caller context.
|
|
103
114
|
const archivedClauseTenantOnly = one ? ` AND kind != 'archived'` : ` WHERE kind != 'archived'`;
|
|
104
|
-
const aliasScope = withProject(recallScopeClause('m.', scopeFilter), 'm.', originProjects);
|
|
105
|
-
const plainScope = withProject(recallScopeClause('', scopeFilter), '', originProjects);
|
|
115
|
+
const aliasScope = withoutTags(withProject(recallScopeClause('m.', scopeFilter), 'm.', originProjects), 'm.', excludeTags);
|
|
116
|
+
const plainScope = withoutTags(withProject(recallScopeClause('', scopeFilter), '', originProjects), '', excludeTags);
|
|
106
117
|
const scopeParams = aliasScope.params;
|
|
107
|
-
const currentAlias = includeSuperseded ? '' :
|
|
108
|
-
const currentNoAlias = includeSuperseded ? '' :
|
|
118
|
+
const currentAlias = includeSuperseded ? '' : ` AND ${notSupersededIn('m.')}`;
|
|
119
|
+
const currentNoAlias = includeSuperseded ? '' : ` AND ${notSupersededIn()}`;
|
|
109
120
|
return {
|
|
110
121
|
tenantPredicate,
|
|
111
122
|
tenantOnlyPredicate,
|
|
@@ -214,6 +225,11 @@ export function listSearchEntries(db, query, limit = DEFAULT_SEARCH_CANDIDATE_LI
|
|
|
214
225
|
* '%:private:%'` runs before the LIMIT so private rows cannot starve admitted ones;
|
|
215
226
|
* `passesScopeFilterForRecall` is the exact post-filter. */
|
|
216
227
|
export function listRecallEntries(db, query, options) {
|
|
228
|
+
return listRecallEntriesWithout(db, query, options, []);
|
|
229
|
+
}
|
|
230
|
+
/** `listRecallEntries` with rows tagged `excludeTags` cut before the limit, which the recall port
|
|
231
|
+
* has no option for. */
|
|
232
|
+
export function listRecallEntriesWithout(db, query, options, excludeTags) {
|
|
217
233
|
const { limit = DEFAULT_SEARCH_CANDIDATE_LIMIT, tenantId, requestedScope, explicitScopeMode = 'exact', includeSuperseded = true, originProjects, ownScope, } = options;
|
|
218
234
|
const scopeFilter = recallScopeFilter(requestedScope, explicitScopeMode, ownScope);
|
|
219
235
|
return listSearchRows(db, {
|
|
@@ -224,6 +240,7 @@ export function listRecallEntries(db, query, options) {
|
|
|
224
240
|
scopeFilter,
|
|
225
241
|
includeSuperseded,
|
|
226
242
|
originProjects,
|
|
243
|
+
excludeTags,
|
|
227
244
|
},
|
|
228
245
|
}).map(rowToEntry);
|
|
229
246
|
}
|
|
@@ -232,7 +249,7 @@ export function listRecallEntries(db, query, options) {
|
|
|
232
249
|
export async function listVectorCandidateEntries(db, queryVector, spec, index) {
|
|
233
250
|
const scope = withProject(recallScopeClause('m.', spec.scope), 'm.', spec.origin);
|
|
234
251
|
const tenant = tenantAnd(spec.tenantId, 'm.tenant_id = ?');
|
|
235
|
-
const current = spec.includeSuperseded ? '' :
|
|
252
|
+
const current = spec.includeSuperseded ? '' : ` AND ${notSupersededIn('m.')}`;
|
|
236
253
|
const params = [...tenant.params, ...scope.params];
|
|
237
254
|
const matches = await topVectorMatches(db, {
|
|
238
255
|
query: queryVector,
|