@elvisw/dsh-magic-context 0.0.0-bootstrap.0 → 0.0.0-stage
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -452
- package/package.json +4 -108
- package/LICENSE +0 -23
- package/NOTICE +0 -93
- package/README.zh-CN.md +0 -368
- package/cordis.patch.yml +0 -20
- package/lib/client.js +0 -2270
- package/lib/core/compartments.js +0 -424
- package/lib/core/corpus-store.js +0 -176
- package/lib/core/directive-detect.js +0 -147
- package/lib/core/domain-reconcile.js +0 -394
- package/lib/core/domains.js +0 -127
- package/lib/core/embedding-local.js +0 -247
- package/lib/core/global-autostage.js +0 -223
- package/lib/core/global-domains.js +0 -302
- package/lib/core/journal.js +0 -248
- package/lib/core/llm-deadline.js +0 -53
- package/lib/core/machine-domain.js +0 -106
- package/lib/core/maintenance.js +0 -380
- package/lib/core/memory-bridge.js +0 -818
- package/lib/core/memory-categories.js +0 -84
- package/lib/core/memory-scoring.js +0 -65
- package/lib/core/memory-search.js +0 -414
- package/lib/core/memory-write.js +0 -1176
- package/lib/core/policy.js +0 -111
- package/lib/core/profile-confirmation.js +0 -55
- package/lib/core/profile-review.js +0 -375
- package/lib/core/profile-write.js +0 -283
- package/lib/core/reference-seeds.js +0 -63
- package/lib/core/route-health.js +0 -106
- package/lib/core/stores.js +0 -1021
- package/lib/core/tagger.js +0 -7
- package/lib/core/types.js +0 -15
- package/lib/core/vector-backfill.js +0 -140
- package/lib/core/vector-lane.js +0 -184
- package/lib/dsh/command-surface.js +0 -376
- package/lib/dsh/compaction-engine.js +0 -231
- package/lib/dsh/config-switches.js +0 -56
- package/lib/dsh/config.js +0 -13
- package/lib/dsh/context-pressure.js +0 -36
- package/lib/dsh/drop-landing.js +0 -42
- package/lib/dsh/emergency.js +0 -89
- package/lib/dsh/flush-command.js +0 -148
- package/lib/dsh/global-review.js +0 -412
- package/lib/dsh/historian.js +0 -907
- package/lib/dsh/live-journal.js +0 -81
- package/lib/dsh/maintenance-wiring.js +0 -229
- package/lib/dsh/management-pass.js +0 -564
- package/lib/dsh/memory-lane.js +0 -535
- package/lib/dsh/pre-step.js +0 -675
- package/lib/dsh/project-drift.js +0 -116
- package/lib/dsh/recomp.js +0 -192
- package/lib/dsh/reduce-guidance.js +0 -188
- package/lib/dsh/seams.js +0 -588
- package/lib/dsh/search-hint.js +0 -186
- package/lib/dsh/session-notice.js +0 -100
- package/lib/dsh/status-command.js +0 -75
- package/lib/dsh/telemetry.js +0 -349
- package/lib/dsh/tool-bridge.js +0 -1009
- package/lib/dsh/volatile.js +0 -89
- package/lib/index.js +0 -931
- package/tools/migrate-profile-package-name.mjs +0 -317
- package/vendor/onnx/LICENSE +0 -27
- package/vendor/onnx/ort-wasm-simd-threaded.mjs +0 -59
- package/vendor/onnx/ort-wasm-simd-threaded.wasm +0 -0
- package/vendor/onnx/ort.node.min.mjs +0 -8
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Magic Context's memory taxonomy, duplicated verbatim so the adapter can
|
|
3
|
-
* write rows Magic Context will accept without importing its TypeScript.
|
|
4
|
-
*
|
|
5
|
-
* Source of truth (kept in sync on purpose, with the schema-version fence in
|
|
6
|
-
* `memory-bridge.ts` as the tripwire): `packages/plugin/src/features/
|
|
7
|
-
* magic-context/memory/constants.ts` + `normalize-hash.ts`.
|
|
8
|
-
*
|
|
9
|
-
* Contract version: MC storage schema v95 (see MEMORY_SCHEMA_MAX_VERSION).
|
|
10
|
-
*/
|
|
11
|
-
/** The v2 taxonomy — the only categories new writes may use. */
|
|
12
|
-
export const V2_MEMORY_CATEGORIES = [
|
|
13
|
-
'PROJECT_RULES',
|
|
14
|
-
'ARCHITECTURE',
|
|
15
|
-
'CONSTRAINTS',
|
|
16
|
-
'CONFIG_VALUES',
|
|
17
|
-
'NAMING',
|
|
18
|
-
];
|
|
19
|
-
/**
|
|
20
|
-
* Categories the historian may promote. Legacy values stay promotable so
|
|
21
|
-
* pre-v2 rows and lingering legacy writes keep working (MC: constants.ts:17).
|
|
22
|
-
*/
|
|
23
|
-
export const PROMOTABLE_CATEGORIES = [
|
|
24
|
-
...V2_MEMORY_CATEGORIES,
|
|
25
|
-
'ARCHITECTURE_DECISIONS',
|
|
26
|
-
'CONFIG_DEFAULTS',
|
|
27
|
-
'USER_PREFERENCES',
|
|
28
|
-
'USER_DIRECTIVES',
|
|
29
|
-
'ENVIRONMENT',
|
|
30
|
-
'WORKFLOW_RULES',
|
|
31
|
-
'KNOWN_ISSUES',
|
|
32
|
-
];
|
|
33
|
-
/** Render/priority order used when injecting memories (MC: CATEGORY_PRIORITY). */
|
|
34
|
-
export const MEMORY_CATEGORY_PRIORITY = [
|
|
35
|
-
...V2_MEMORY_CATEGORIES,
|
|
36
|
-
'USER_DIRECTIVES',
|
|
37
|
-
'USER_PREFERENCES',
|
|
38
|
-
'CONFIG_DEFAULTS',
|
|
39
|
-
'ARCHITECTURE_DECISIONS',
|
|
40
|
-
'ENVIRONMENT',
|
|
41
|
-
'WORKFLOW_RULES',
|
|
42
|
-
'KNOWN_ISSUES',
|
|
43
|
-
];
|
|
44
|
-
export const MEMORY_CATEGORY_ORDER_UNKNOWN = 99;
|
|
45
|
-
export function memoryCategoryOrder(category) {
|
|
46
|
-
const index = MEMORY_CATEGORY_PRIORITY.indexOf(category);
|
|
47
|
-
return index === -1 ? MEMORY_CATEGORY_ORDER_UNKNOWN : index;
|
|
48
|
-
}
|
|
49
|
-
/** TTL in ms by category; absent = permanent (MC: CATEGORY_DEFAULT_TTL). */
|
|
50
|
-
export const CATEGORY_DEFAULT_TTL = {
|
|
51
|
-
WORKFLOW_RULES: 90 * 24 * 60 * 60 * 1000,
|
|
52
|
-
KNOWN_ISSUES: 30 * 24 * 60 * 60 * 1000,
|
|
53
|
-
};
|
|
54
|
-
/**
|
|
55
|
-
* `expires_at` for a new row: only the legacy TTL categories expire
|
|
56
|
-
* (MC: promotion.ts:36-39 → `resolveExpiresAt`).
|
|
57
|
-
*/
|
|
58
|
-
export function resolveExpiresAt(category, now) {
|
|
59
|
-
const ttl = CATEGORY_DEFAULT_TTL[category];
|
|
60
|
-
return ttl === undefined ? undefined : now + ttl;
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* TTL applied to HISTORIAN-promoted rows (user decision 2026-10-16).
|
|
64
|
-
*
|
|
65
|
-
* The promoted pool grew monotonically and nothing bounded it: measured on the
|
|
66
|
-
* live corpus, 1,941 non-domain rows, 67% of the active ones never recalled,
|
|
67
|
-
* and `expires_at` non-null on exactly ZERO of them — the contract's TTL table
|
|
68
|
-
* only covers two legacy categories, so the expiry pass had nothing to expire.
|
|
69
|
-
* Every promoted fact now carries a 90-day window; a fact that is still being
|
|
70
|
-
* re-observed refreshes it (see `promoteFacts`), so only abandoned rows age out.
|
|
71
|
-
*/
|
|
72
|
-
export const HISTORIAN_MEMORY_TTL_MS = 90 * 24 * 60 * 60 * 1000;
|
|
73
|
-
/**
|
|
74
|
-
* `expires_at` for a promoted historian row: the category's own TTL when it has
|
|
75
|
-
* one (shorter wins — KNOWN_ISSUES already expires at 30 days), else the
|
|
76
|
-
* historian window. Unlike `resolveExpiresAt` this never returns undefined:
|
|
77
|
-
* every promoted row gets a window.
|
|
78
|
-
*/
|
|
79
|
-
export function resolveHistorianExpiresAt(category, now) {
|
|
80
|
-
return now + (CATEGORY_DEFAULT_TTL[category] ?? HISTORIAN_MEMORY_TTL_MS);
|
|
81
|
-
}
|
|
82
|
-
export function isPromotableCategory(category) {
|
|
83
|
-
return typeof category === 'string' && PROMOTABLE_CATEGORIES.includes(category);
|
|
84
|
-
}
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* S(t) — the shared-memory decay score, borrowed from mon3stera's
|
|
3
|
-
* dsh-magic-context `memory.js` and reshaped to this adapter's contracts.
|
|
4
|
-
*
|
|
5
|
-
* S = clamp(0, 100, I · (1 + α·ln(1 + k)) · exp(−(ln2/τ_eff)·Δt))
|
|
6
|
-
* τ_eff = τ / (1 + β·k)
|
|
7
|
-
*
|
|
8
|
-
* I is the shared store's importance (1–100). Hits k SLOW the decay and boost
|
|
9
|
-
* the score (LRU-style: used memories do not rot); τ is the category
|
|
10
|
-
* half-life in days, null = immortal; a hit count of zero on a row we have no
|
|
11
|
-
* sidecar stats for decays nothing (cold-start rows are favored, not
|
|
12
|
-
* punished — the sidecar fills on the first recall anyway).
|
|
13
|
-
*
|
|
14
|
-
* Pure functions, no I/O and no MC import. The score gates ONLY the adapter's
|
|
15
|
-
* injection ranking; it never writes the shared store (writer-contract
|
|
16
|
-
* discipline): sub-threshold rows stop being injected here and stay fully
|
|
17
|
-
* searchable — and a hit that lifts a row back over the threshold un-retires
|
|
18
|
-
* it on the next recall.
|
|
19
|
-
*/
|
|
20
|
-
/**
|
|
21
|
-
* Half-life defaults, mon3stera's spirit translated to MC's taxonomy: durable
|
|
22
|
-
* categories are immortal; exact values and names rot; legacy categories take
|
|
23
|
-
* the MC TTL where one exists (KNOWN_ISSUES/WORKFLOW_RULES) or a conservative
|
|
24
|
-
* middle ground otherwise.
|
|
25
|
-
*
|
|
26
|
-
* `enabled` defaults TRUE since the 2026-10-13 distribution decision: the
|
|
27
|
-
* ranking takeover is the feature (stats alone are a dashboard), the corpus is
|
|
28
|
-
* plugin-owned, and shadow telemetry showed zero regressions. An explicit
|
|
29
|
-
* `false` in the profile still wins.
|
|
30
|
-
*/
|
|
31
|
-
export const DEFAULT_MEMORY_SCORING = {
|
|
32
|
-
enabled: true,
|
|
33
|
-
alpha: 0.4,
|
|
34
|
-
beta: 0.2,
|
|
35
|
-
archiveThreshold: 2,
|
|
36
|
-
halfLives: {
|
|
37
|
-
PROJECT_RULES: null,
|
|
38
|
-
ARCHITECTURE: null,
|
|
39
|
-
CONSTRAINTS: null,
|
|
40
|
-
CONFIG_VALUES: 60,
|
|
41
|
-
NAMING: 90,
|
|
42
|
-
USER_PREFERENCES: 14,
|
|
43
|
-
WORKFLOW_RULES: 30,
|
|
44
|
-
KNOWN_ISSUES: 30,
|
|
45
|
-
ENVIRONMENT: 90,
|
|
46
|
-
},
|
|
47
|
-
};
|
|
48
|
-
export function halfLifeFor(config, category) {
|
|
49
|
-
const value = config.halfLives[category];
|
|
50
|
-
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : null;
|
|
51
|
-
}
|
|
52
|
-
/** One memory's current score, on the shared store's 1–100 scale (clamped). */
|
|
53
|
-
export function scoreMemory(memory, config, now) {
|
|
54
|
-
const importance = Number.isFinite(memory.importance) ? Math.max(0, memory.importance) : 0;
|
|
55
|
-
const hits = Number.isInteger(memory.hits) && memory.hits > 0 ? memory.hits : 0;
|
|
56
|
-
const boost = 1 + config.alpha * Math.log(1 + hits);
|
|
57
|
-
const halfLifeDays = halfLifeFor(config, memory.category);
|
|
58
|
-
let decay = 1;
|
|
59
|
-
if (halfLifeDays !== null && memory.lastHitAt !== undefined) {
|
|
60
|
-
const tauEff = halfLifeDays / (1 + config.beta * hits);
|
|
61
|
-
const dtDays = Math.max(0, (now - memory.lastHitAt) / 86_400_000);
|
|
62
|
-
decay = Math.exp(-(Math.LN2 / tauEff) * dtDays);
|
|
63
|
-
}
|
|
64
|
-
return Math.min(100, Math.max(0, importance * boost * decay));
|
|
65
|
-
}
|
|
@@ -1,414 +0,0 @@
|
|
|
1
|
-
import { DatabaseSync } from 'node:sqlite';
|
|
2
|
-
import { cosineSimilarity } from "./embedding-local.js";
|
|
3
|
-
import { errText } from "./types.js";
|
|
4
|
-
/**
|
|
5
|
-
* `ctx_search`'s memory lane — a faithful port of Magic Context's hybrid
|
|
6
|
-
* retrieval (`features/magic-context/search.ts` + `memory/storage-memory-fts.ts`,
|
|
7
|
-
* v0.45.0), reading the SAME shared store read-only.
|
|
8
|
-
*
|
|
9
|
-
* Score model (MC search.ts:49-51, 320-330, 855-901):
|
|
10
|
-
* hybrid = SEMANTIC_WEIGHT(0.7) × cosine + FTS_WEIGHT(0.3) × reciprocal-rank
|
|
11
|
-
* single = score × SINGLE_SOURCE_PENALTY(0.8)
|
|
12
|
-
* cosine = clamp01; FTS order = bm25, updated_at DESC, id ASC; FTS scores
|
|
13
|
-
* are 1/(rank+1) (getFtsScores:801-803), NOT raw bm25 values.
|
|
14
|
-
* sort = score DESC, id ASC; preview = 220 chars with `…`.
|
|
15
|
-
*
|
|
16
|
-
* Lanes: FTS5 (`memories_fts MATCH`, sanitize → strict AND, 0 rows → relaxed
|
|
17
|
-
* OR — storage-memory-fts.ts:63-123) over the pool (domain visibility rules
|
|
18
|
-
* identical to recall: own rows full, foreign rows need shareable + scope +
|
|
19
|
-
* category allowlist — buildWorkspaceMemorySqlFilter:783-840), plus the
|
|
20
|
-
* semantic lane over `memory_embeddings` vectors read-only.
|
|
21
|
-
*
|
|
22
|
-
* Candidates for semantic scoring are `ftsMatches ∪ storedVectorIds` ∩ pool
|
|
23
|
-
* (MC selectSemanticCandidates:805-834). Candidates lacking a stored vector may
|
|
24
|
-
* be embedded in memory, bounded per search — the adapter never writes Magic
|
|
25
|
-
* Context's store (MC's own read-path backfill persists vectors instead).
|
|
26
|
-
*
|
|
27
|
-
* Deliberate deviations (recorded, not silent): no `visibleMemoryIds`
|
|
28
|
-
* suppression (the adapter does not track which rows the current prompt already
|
|
29
|
-
* carries, search.ts:877-880); in-memory embedding capped to bound latency.
|
|
30
|
-
*/
|
|
31
|
-
const FTS_SEMANTIC_CANDIDATE_LIMIT = 50;
|
|
32
|
-
const SEMANTIC_WEIGHT = 0.7;
|
|
33
|
-
const FTS_WEIGHT = 0.3;
|
|
34
|
-
const SINGLE_SOURCE_PENALTY = 0.8;
|
|
35
|
-
const RESULT_PREVIEW_LIMIT = 220;
|
|
36
|
-
export const DEFAULT_MEMORY_SEARCH_LIMIT = 10;
|
|
37
|
-
export const MAX_MEMORY_SEARCH_LIMIT = 25;
|
|
38
|
-
/** Upper bound on rows embedded in memory for one search (latency guard). */
|
|
39
|
-
export const MAX_IN_MEMORY_EMBEDDINGS = 32;
|
|
40
|
-
const ID_SHAPED_QUERY_MAX_TOKENS = 5;
|
|
41
|
-
const ID_SHAPED_TOKEN = /^#?\d+$/;
|
|
42
|
-
/** MC sanitizeFtsQuery (storage-memory-fts.ts:70-80) — every whitespace token
|
|
43
|
-
* becomes a quoted literal so FTS5 operators lose their meaning. */
|
|
44
|
-
export function sanitizeFtsQuery(query) {
|
|
45
|
-
const tokens = query
|
|
46
|
-
.replaceAll('\0', ' ')
|
|
47
|
-
.split(/\s+/)
|
|
48
|
-
.filter((token) => token.length > 0);
|
|
49
|
-
if (tokens.length === 0)
|
|
50
|
-
return '';
|
|
51
|
-
return tokens.map((token) => `"${token.replace(/"/g, '""')}"`).join(' ');
|
|
52
|
-
}
|
|
53
|
-
/** MC relaxedFtsQuery (storage-memory-fts.ts:63-68) — word chars only, length
|
|
54
|
-
* > 2 or containing a digit, deduped, max 16, OR-joined. */
|
|
55
|
-
export function relaxedFtsQuery(query) {
|
|
56
|
-
const tokens = [...new Set(query.match(/[\p{L}\p{N}_]+/gu) ?? [])]
|
|
57
|
-
.filter((token) => token.length > 2 || /\d/.test(token))
|
|
58
|
-
.slice(0, 16);
|
|
59
|
-
return tokens.map((token) => `"${token.replace(/"/g, '""')}"`).join(' OR ');
|
|
60
|
-
}
|
|
61
|
-
/** MC parseIdShapedQuery (search.ts:297-321) — a query made only of id tokens
|
|
62
|
-
* (`#920`, `920 921`, max 5) is a direct lookup, not a text search. */
|
|
63
|
-
export function parseIdShapedQuery(query) {
|
|
64
|
-
const trimmed = query.trim();
|
|
65
|
-
if (trimmed.length === 0)
|
|
66
|
-
return null;
|
|
67
|
-
const tokens = trimmed.split(/[\s,]+/).filter((token) => token.length > 0);
|
|
68
|
-
if (tokens.length === 0 || tokens.length > ID_SHAPED_QUERY_MAX_TOKENS)
|
|
69
|
-
return null;
|
|
70
|
-
const ids = [];
|
|
71
|
-
for (const token of tokens) {
|
|
72
|
-
if (!ID_SHAPED_TOKEN.test(token))
|
|
73
|
-
return null;
|
|
74
|
-
const parsed = Number.parseInt(token.replace(/^#/, ''), 10);
|
|
75
|
-
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
76
|
-
return null;
|
|
77
|
-
ids.push(parsed);
|
|
78
|
-
}
|
|
79
|
-
return ids;
|
|
80
|
-
}
|
|
81
|
-
/** MC previewText (search.ts:328-333). */
|
|
82
|
-
export function previewText(text) {
|
|
83
|
-
const normalized = text.replace(/\s+/g, ' ').trim();
|
|
84
|
-
if (normalized.length <= RESULT_PREVIEW_LIMIT)
|
|
85
|
-
return normalized;
|
|
86
|
-
return `${normalized.slice(0, RESULT_PREVIEW_LIMIT - 1).trimEnd()}…`;
|
|
87
|
-
}
|
|
88
|
-
/** MC normalizeCosineScore (search.ts:320-326). */
|
|
89
|
-
function normalizeCosineScore(score) {
|
|
90
|
-
if (!Number.isFinite(score))
|
|
91
|
-
return 0;
|
|
92
|
-
return Math.min(1, Math.max(0, score));
|
|
93
|
-
}
|
|
94
|
-
/** Workspace visibility predicate (MC buildWorkspaceMemorySqlFilter:783-840):
|
|
95
|
-
* own identities see everything, foreign rows must be shareable + scope
|
|
96
|
-
* project/ecosystem/universe + in the active domains' category allowlist.
|
|
97
|
-
* `table` qualifies the column refs (the vector join aliases memories as m). */
|
|
98
|
-
export function domainVisibilityPredicate(space, table = 'memories') {
|
|
99
|
-
const q = `${table}.`;
|
|
100
|
-
const own = [...new Set(space.ownIdentities)].filter((id) => id.length > 0);
|
|
101
|
-
const foreign = [...new Set(space.foreignIdentities)].filter((id) => id.length > 0 && !own.includes(id));
|
|
102
|
-
const share = [...new Set(space.shareCategories)];
|
|
103
|
-
const predicates = [];
|
|
104
|
-
const params = [];
|
|
105
|
-
if (own.length > 0) {
|
|
106
|
-
predicates.push(`${q}project_path IN (${own.map(() => '?').join(', ')})`);
|
|
107
|
-
params.push(...own);
|
|
108
|
-
}
|
|
109
|
-
if (foreign.length > 0 && share.length > 0) {
|
|
110
|
-
predicates.push(`(${q}project_path IN (${foreign.map(() => '?').join(', ')})`
|
|
111
|
-
+ ` AND ${q}category IN (${share.map(() => '?').join(', ')})`
|
|
112
|
-
+ ` AND ${q}shareable = 1`
|
|
113
|
-
+ ` AND ${q}scope IN ('project','ecosystem','universe'))`);
|
|
114
|
-
params.push(...foreign, ...share);
|
|
115
|
-
}
|
|
116
|
-
if (predicates.length === 0)
|
|
117
|
-
return { clause: ' AND 0 = 1', params: [] };
|
|
118
|
-
return { clause: ` AND (${predicates.join(' OR ')})`, params };
|
|
119
|
-
}
|
|
120
|
-
export function openMemorySearch(options) {
|
|
121
|
-
const { logger } = options;
|
|
122
|
-
let db;
|
|
123
|
-
try {
|
|
124
|
-
db = new DatabaseSync(options.dbPath, { readOnly: true });
|
|
125
|
-
}
|
|
126
|
-
catch (error) {
|
|
127
|
-
const reason = `search store unavailable: ${errText(error)}`;
|
|
128
|
-
return {
|
|
129
|
-
search: () => Promise.resolve({ ok: false, reason }),
|
|
130
|
-
dominantEmbeddingModel: () => undefined,
|
|
131
|
-
paritySample: () => undefined,
|
|
132
|
-
close: () => { },
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
const tableExists = (name) => {
|
|
136
|
-
try {
|
|
137
|
-
return db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) !== undefined;
|
|
138
|
-
}
|
|
139
|
-
catch {
|
|
140
|
-
return false;
|
|
141
|
-
}
|
|
142
|
-
};
|
|
143
|
-
const hasMemories = tableExists('memories');
|
|
144
|
-
// Both aux lanes are optional: a store without the FTS index or the
|
|
145
|
-
// embeddings table runs with the remaining lane(s) (MC treats a failed FTS
|
|
146
|
-
// query as an empty lane, search.ts:793-798).
|
|
147
|
-
const hasFts = tableExists('memories_fts');
|
|
148
|
-
const hasVectors = tableExists('memory_embeddings');
|
|
149
|
-
const search = async (request) => {
|
|
150
|
-
const run = async () => {
|
|
151
|
-
const trimmed = request.query.trim();
|
|
152
|
-
if (trimmed.length === 0) {
|
|
153
|
-
return { ok: false, reason: 'query is required' };
|
|
154
|
-
}
|
|
155
|
-
const limit = Math.max(1, Math.min(MAX_MEMORY_SEARCH_LIMIT, Math.floor(request.limit ?? DEFAULT_MEMORY_SEARCH_LIMIT)));
|
|
156
|
-
// A3: the adapter's unified lane covers the `memory` source only; a
|
|
157
|
-
// sources list that excludes it yields zero hits (MC UnifiedSearchOptions).
|
|
158
|
-
if (request.sources !== undefined && !request.sources.includes('memory')) {
|
|
159
|
-
const empty = [];
|
|
160
|
-
return {
|
|
161
|
-
ok: true,
|
|
162
|
-
hits: empty,
|
|
163
|
-
text: formatSearchResults(trimmed, empty),
|
|
164
|
-
ftsMatches: 0,
|
|
165
|
-
semanticMatches: 0,
|
|
166
|
-
suppressedVisible: 0,
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
const { clause, params } = domainVisibilityPredicate(request.space);
|
|
170
|
-
// The embeddings join aliases memories as `m` — requalify the predicate.
|
|
171
|
-
const vectorVisibility = domainVisibilityPredicate(request.space, 'm');
|
|
172
|
-
// ── Direct id lookup (MC parseIdShapedQuery short-circuit) ────────────
|
|
173
|
-
const idShape = parseIdShapedQuery(trimmed);
|
|
174
|
-
if (idShape !== null) {
|
|
175
|
-
const placeholders = idShape.map(() => '?').join(', ');
|
|
176
|
-
const rows = db.prepare(`SELECT id, category, content, created_at AS createdAt FROM memories WHERE id IN (${placeholders})${clause}`
|
|
177
|
-
+ ' AND status IN (\'active\',\'permanent\')'
|
|
178
|
-
+ ' AND (expires_at IS NULL OR expires_at > ?) ORDER BY id ASC').all(...idShape, ...params, request.now);
|
|
179
|
-
const hits = rows.map((row) => ({
|
|
180
|
-
id: row.id,
|
|
181
|
-
category: row.category ?? '',
|
|
182
|
-
score: 1,
|
|
183
|
-
matchType: 'fts',
|
|
184
|
-
preview: previewText(row.content),
|
|
185
|
-
}));
|
|
186
|
-
return {
|
|
187
|
-
ok: true,
|
|
188
|
-
hits,
|
|
189
|
-
text: formatSearchResults(trimmed, hits),
|
|
190
|
-
ftsMatches: hits.length,
|
|
191
|
-
semanticMatches: 0,
|
|
192
|
-
suppressedVisible: 0,
|
|
193
|
-
};
|
|
194
|
-
}
|
|
195
|
-
if (!hasMemories) {
|
|
196
|
-
return { ok: false, reason: 'memories table missing' };
|
|
197
|
-
}
|
|
198
|
-
// ── Lane 1: FTS5 (strict AND, then relaxed OR fallback) ───────────────
|
|
199
|
-
const ftsSql = (matchPlaceholder) => `SELECT memories.id AS id, memories.category AS category, memories.content AS content, memories.created_at AS createdAt`
|
|
200
|
-
+ ` FROM memories_fts INNER JOIN memories ON memories.id = memories_fts.rowid`
|
|
201
|
-
+ ` WHERE memories_fts MATCH ${matchPlaceholder}`
|
|
202
|
-
+ ` AND memories.status IN ('active','permanent')`
|
|
203
|
-
+ ` AND (memories.expires_at IS NULL OR memories.expires_at > ?)`
|
|
204
|
-
+ clause
|
|
205
|
-
+ ` ORDER BY bm25(memories_fts), memories.updated_at DESC, memories.id ASC`
|
|
206
|
-
+ ` LIMIT ?`;
|
|
207
|
-
const runFts = (matchValue) => db.prepare(ftsSql('?')).all(matchValue, request.now, ...params, FTS_SEMANTIC_CANDIDATE_LIMIT);
|
|
208
|
-
let ftsRows = [];
|
|
209
|
-
if (hasFts) {
|
|
210
|
-
try {
|
|
211
|
-
const sanitized = sanitizeFtsQuery(trimmed);
|
|
212
|
-
if (sanitized.length > 0)
|
|
213
|
-
ftsRows = runFts(sanitized);
|
|
214
|
-
if (ftsRows.length === 0) {
|
|
215
|
-
const relaxed = relaxedFtsQuery(trimmed);
|
|
216
|
-
if (relaxed.length > 0)
|
|
217
|
-
ftsRows = runFts(relaxed);
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
catch (error) {
|
|
221
|
-
// MC getFtsMatches:793-798 — a failed FTS query is an empty lane.
|
|
222
|
-
logger?.warn?.(`magic-context FTS lane failed: ${errText(error)}`);
|
|
223
|
-
ftsRows = [];
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
// ── Lane 2: semantic over stored vectors (+ bounded in-memory fill) ───
|
|
227
|
-
const metaById = new Map(ftsRows.map((row) => [row.id, row]));
|
|
228
|
-
const stored = new Map();
|
|
229
|
-
const storedIds = [];
|
|
230
|
-
if (hasVectors && request.queryEmbedding !== null && request.queryModelId !== null) {
|
|
231
|
-
const rows = db.prepare(`SELECT m.id AS id, m.category AS category, m.content AS content, m.created_at AS createdAt, e.embedding AS embedding`
|
|
232
|
-
+ ` FROM memory_embeddings e INNER JOIN memories m ON m.id = e.memory_id`
|
|
233
|
-
+ ` WHERE e.model_id = ? AND m.status IN ('active','permanent')`
|
|
234
|
-
+ ` AND (m.expires_at IS NULL OR m.expires_at > ?)${vectorVisibility.clause}`).all(request.queryModelId, request.now, ...vectorVisibility.params);
|
|
235
|
-
const dims = request.queryEmbedding.length;
|
|
236
|
-
for (const row of rows) {
|
|
237
|
-
if (row.embedding.length / 4 !== dims)
|
|
238
|
-
continue;
|
|
239
|
-
stored.set(row.id, new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 4));
|
|
240
|
-
storedIds.push(row.id);
|
|
241
|
-
if (!metaById.has(row.id)) {
|
|
242
|
-
metaById.set(row.id, { id: row.id, category: row.category, content: row.content, createdAt: row.createdAt });
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
const semanticScores = new Map();
|
|
247
|
-
if (request.queryEmbedding !== null) {
|
|
248
|
-
const candidateIds = new Set([
|
|
249
|
-
...ftsRows.map((row) => row.id),
|
|
250
|
-
...storedIds,
|
|
251
|
-
]);
|
|
252
|
-
// Bounded in-memory fill for candidates without a stored vector.
|
|
253
|
-
const missing = [...candidateIds]
|
|
254
|
-
.filter((id) => !stored.has(id) && metaById.has(id))
|
|
255
|
-
.slice(0, MAX_IN_MEMORY_EMBEDDINGS);
|
|
256
|
-
if (missing.length > 0 && request.embedTexts !== undefined) {
|
|
257
|
-
try {
|
|
258
|
-
const vectors = await request.embedTexts(missing.map((id) => metaById.get(id).content));
|
|
259
|
-
if (vectors !== null) {
|
|
260
|
-
for (let i = 0; i < missing.length; i++) {
|
|
261
|
-
const vector = vectors[i];
|
|
262
|
-
if (vector !== undefined)
|
|
263
|
-
stored.set(missing[i], vector);
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
catch {
|
|
268
|
-
// fail-open: the stored vectors already cover the rest
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
for (const id of candidateIds) {
|
|
272
|
-
const vector = stored.get(id);
|
|
273
|
-
if (vector === undefined)
|
|
274
|
-
continue;
|
|
275
|
-
semanticScores.set(id, normalizeCosineScore(cosineSimilarity(request.queryEmbedding, vector)));
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
// ── Merge (MC mergeMemoryResults:836-904) ─────────────────────────────
|
|
279
|
-
const ftsScores = new Map(ftsRows.map((row, rank) => [row.id, 1 / (rank + 1)]));
|
|
280
|
-
const hits = [];
|
|
281
|
-
let suppressedVisible = 0;
|
|
282
|
-
const candidateIds = new Set([...semanticScores.keys(), ...ftsScores.keys()]);
|
|
283
|
-
for (const id of candidateIds) {
|
|
284
|
-
const meta = metaById.get(id);
|
|
285
|
-
if (meta === undefined)
|
|
286
|
-
continue;
|
|
287
|
-
// A3: created_at window (inclusive both ends).
|
|
288
|
-
if (request.from !== undefined && meta.createdAt < request.from)
|
|
289
|
-
continue;
|
|
290
|
-
if (request.to !== undefined && meta.createdAt > request.to)
|
|
291
|
-
continue;
|
|
292
|
-
// A2: rows the injected block already renders are suppressed from
|
|
293
|
-
// text results (MC suppressedVisibleMemoryIds; worst case without it
|
|
294
|
-
// is a redundant hit, never a correctness issue).
|
|
295
|
-
if (request.visibleMemoryIds?.has(id) === true) {
|
|
296
|
-
suppressedVisible += 1;
|
|
297
|
-
continue;
|
|
298
|
-
}
|
|
299
|
-
const semanticScore = semanticScores.get(id);
|
|
300
|
-
const ftsScore = ftsScores.get(id);
|
|
301
|
-
let score = 0;
|
|
302
|
-
let matchType = 'fts';
|
|
303
|
-
if (semanticScore !== undefined && ftsScore !== undefined) {
|
|
304
|
-
score = SEMANTIC_WEIGHT * semanticScore + FTS_WEIGHT * ftsScore;
|
|
305
|
-
matchType = 'hybrid';
|
|
306
|
-
}
|
|
307
|
-
else if (semanticScore !== undefined) {
|
|
308
|
-
score = semanticScore * SINGLE_SOURCE_PENALTY;
|
|
309
|
-
matchType = 'semantic';
|
|
310
|
-
}
|
|
311
|
-
else if (ftsScore !== undefined) {
|
|
312
|
-
score = ftsScore * SINGLE_SOURCE_PENALTY;
|
|
313
|
-
matchType = 'fts';
|
|
314
|
-
}
|
|
315
|
-
if (score <= 0)
|
|
316
|
-
continue;
|
|
317
|
-
hits.push({
|
|
318
|
-
id,
|
|
319
|
-
category: meta.category ?? '',
|
|
320
|
-
score,
|
|
321
|
-
matchType,
|
|
322
|
-
preview: previewText(meta.content),
|
|
323
|
-
});
|
|
324
|
-
}
|
|
325
|
-
hits.sort((left, right) => {
|
|
326
|
-
if (right.score !== left.score)
|
|
327
|
-
return right.score - left.score;
|
|
328
|
-
return left.id - right.id;
|
|
329
|
-
});
|
|
330
|
-
const page = hits.slice(0, limit);
|
|
331
|
-
return {
|
|
332
|
-
ok: true,
|
|
333
|
-
hits: page,
|
|
334
|
-
text: formatSearchResults(trimmed, page),
|
|
335
|
-
ftsMatches: ftsRows.length,
|
|
336
|
-
semanticMatches: semanticScores.size,
|
|
337
|
-
suppressedVisible,
|
|
338
|
-
};
|
|
339
|
-
};
|
|
340
|
-
// B3 (MC single_store_reads): one deferred read transaction so the FTS
|
|
341
|
-
// lane, the vector reads and the pool probes observe ONE snapshot.
|
|
342
|
-
try {
|
|
343
|
-
if (hasMemories || hasFts || hasVectors)
|
|
344
|
-
db.exec('BEGIN');
|
|
345
|
-
try {
|
|
346
|
-
return await run();
|
|
347
|
-
}
|
|
348
|
-
finally {
|
|
349
|
-
try {
|
|
350
|
-
db.exec('COMMIT');
|
|
351
|
-
}
|
|
352
|
-
catch {
|
|
353
|
-
/* aborted read transaction — readonly, nothing to roll back */
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
catch (error) {
|
|
358
|
-
return {
|
|
359
|
-
ok: false,
|
|
360
|
-
reason: `memory search failed: ${errText(error)}`,
|
|
361
|
-
};
|
|
362
|
-
}
|
|
363
|
-
};
|
|
364
|
-
return {
|
|
365
|
-
search,
|
|
366
|
-
dominantEmbeddingModel: () => {
|
|
367
|
-
if (!hasVectors)
|
|
368
|
-
return undefined;
|
|
369
|
-
try {
|
|
370
|
-
const row = db.prepare(`SELECT model_id, MIN(length(embedding)) / 4 AS dims, COUNT(*) AS n`
|
|
371
|
-
+ ` FROM memory_embeddings GROUP BY model_id ORDER BY n DESC, model_id ASC LIMIT 1`).get();
|
|
372
|
-
if (row === undefined || row.model_id.length === 0)
|
|
373
|
-
return undefined;
|
|
374
|
-
return { modelId: row.model_id, dims: row.dims ?? 0 };
|
|
375
|
-
}
|
|
376
|
-
catch {
|
|
377
|
-
return undefined;
|
|
378
|
-
}
|
|
379
|
-
},
|
|
380
|
-
paritySample: (modelId) => {
|
|
381
|
-
if (!hasVectors)
|
|
382
|
-
return undefined;
|
|
383
|
-
try {
|
|
384
|
-
const row = db.prepare(`SELECT m.content AS content, e.embedding AS embedding`
|
|
385
|
-
+ ` FROM memory_embeddings e INNER JOIN memories m ON m.id = e.memory_id`
|
|
386
|
-
+ ` WHERE e.model_id = ? ORDER BY e.rowid DESC LIMIT 1`).get(modelId);
|
|
387
|
-
if (row === undefined)
|
|
388
|
-
return undefined;
|
|
389
|
-
return {
|
|
390
|
-
content: row.content,
|
|
391
|
-
embedding: new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 4),
|
|
392
|
-
};
|
|
393
|
-
}
|
|
394
|
-
catch {
|
|
395
|
-
return undefined;
|
|
396
|
-
}
|
|
397
|
-
},
|
|
398
|
-
close: () => {
|
|
399
|
-
try {
|
|
400
|
-
db.close();
|
|
401
|
-
}
|
|
402
|
-
catch { /* already closed */ }
|
|
403
|
-
},
|
|
404
|
-
};
|
|
405
|
-
}
|
|
406
|
-
/** MC formatSearchResults (search.ts:2390-2413) restricted to the memory lane;
|
|
407
|
-
* per-hit line format is MC formatUnifiedSearchResult:2304-2311. */
|
|
408
|
-
export function formatSearchResults(query, hits) {
|
|
409
|
-
if (hits.length === 0) {
|
|
410
|
-
return `No results found for "${query}" in project memories.`;
|
|
411
|
-
}
|
|
412
|
-
const body = hits.map((hit, index) => `[${index + 1}] [memory] score=${hit.score.toFixed(2)} id=${hit.id} category=${hit.category} match=${hit.matchType}\n${hit.preview}`);
|
|
413
|
-
return `Found ${hits.length} result${hits.length === 1 ? '' : 's'} for "${query}":\n\n${body.join('\n\n')}`;
|
|
414
|
-
}
|