@jungjaehoon/mama-core 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -10
- package/db/migrations/032-add-memory-provenance-columns.sql +21 -0
- package/db/migrations/033-create-model-runs-and-tool-traces.sql +53 -0
- package/db/migrations/034-add-connector-event-scope-columns.sql +14 -0
- package/db/migrations/035-create-twin-edges.sql +63 -0
- package/db/migrations/036-create-agent-situation-packets.sql +61 -0
- package/db/migrations/037-create-context-packets.sql +32 -0
- package/dist/agent-graph/alias-write.d.ts +3 -0
- package/dist/agent-graph/alias-write.js +231 -0
- package/dist/agent-graph/entity-resolve.d.ts +3 -0
- package/dist/agent-graph/entity-resolve.js +275 -0
- package/dist/agent-graph/errors.d.ts +4 -0
- package/dist/agent-graph/errors.js +11 -0
- package/dist/agent-graph/graph-query.d.ts +5 -0
- package/dist/agent-graph/graph-query.js +359 -0
- package/dist/agent-graph/index.d.ts +6 -0
- package/dist/agent-graph/index.js +22 -0
- package/dist/agent-graph/types.d.ts +162 -0
- package/dist/agent-graph/types.js +3 -0
- package/dist/agent-situation/builder.d.ts +8 -0
- package/dist/agent-situation/builder.js +239 -0
- package/dist/agent-situation/cache-key.d.ts +4 -0
- package/dist/agent-situation/cache-key.js +64 -0
- package/dist/agent-situation/index.d.ts +7 -0
- package/dist/agent-situation/index.js +28 -0
- package/dist/agent-situation/packet-store.d.ts +27 -0
- package/dist/agent-situation/packet-store.js +228 -0
- package/dist/agent-situation/ranking-policy.d.ts +18 -0
- package/dist/agent-situation/ranking-policy.js +75 -0
- package/dist/agent-situation/source-readers.d.ts +52 -0
- package/dist/agent-situation/source-readers.js +300 -0
- package/dist/agent-situation/types.d.ts +195 -0
- package/dist/agent-situation/types.js +3 -0
- package/dist/cases/search-rollup.d.ts +4 -0
- package/dist/cases/search-rollup.js +84 -0
- package/dist/cases/wiki-page-index.js +6 -2
- package/dist/connectors/event-index.d.ts +1 -0
- package/dist/connectors/event-index.js +19 -4
- package/dist/connectors/raw-query.d.ts +18 -0
- package/dist/connectors/raw-query.js +305 -0
- package/dist/connectors/types.d.ts +40 -0
- package/dist/context-compile/boundary-defaults.d.ts +12 -0
- package/dist/context-compile/boundary-defaults.js +105 -0
- package/dist/context-compile/compiler-policy.d.ts +39 -0
- package/dist/context-compile/compiler-policy.js +147 -0
- package/dist/context-compile/compiler.d.ts +18 -0
- package/dist/context-compile/compiler.js +401 -0
- package/dist/context-compile/index.d.ts +9 -0
- package/dist/context-compile/index.js +25 -0
- package/dist/context-compile/packet-store.d.ts +9 -0
- package/dist/context-compile/packet-store.js +255 -0
- package/dist/context-compile/ref.d.ts +7 -0
- package/dist/context-compile/ref.js +102 -0
- package/dist/context-compile/source-readers.d.ts +69 -0
- package/dist/context-compile/source-readers.js +707 -0
- package/dist/context-compile/types.d.ts +138 -0
- package/dist/context-compile/types.js +11 -0
- package/dist/context-compile/visibility.d.ts +21 -0
- package/dist/context-compile/visibility.js +200 -0
- package/dist/db-adapter/node-sqlite-adapter.d.ts +9 -0
- package/dist/db-adapter/node-sqlite-adapter.js +207 -0
- package/dist/db-manager.d.ts +8 -1
- package/dist/db-manager.js +53 -5
- package/dist/edges/ref-validation.d.ts +8 -0
- package/dist/edges/ref-validation.js +301 -0
- package/dist/edges/store.d.ts +10 -0
- package/dist/edges/store.js +343 -0
- package/dist/edges/types.d.ts +88 -0
- package/dist/edges/types.js +18 -0
- package/dist/embeddings.d.ts +3 -0
- package/dist/embeddings.js +49 -9
- package/dist/entities/store.d.ts +4 -1
- package/dist/entities/store.js +206 -78
- package/dist/entities/types.d.ts +5 -1
- package/dist/entities/types.js +1 -1
- package/dist/index.d.ts +18 -3
- package/dist/index.js +54 -4
- package/dist/mama-api.d.ts +32 -35
- package/dist/mama-api.js +173 -22
- package/dist/memory/api.d.ts +17 -56
- package/dist/memory/api.js +494 -52
- package/dist/memory/event-store.d.ts +3 -0
- package/dist/memory/event-store.js +21 -2
- package/dist/memory/provenance-audit.d.ts +22 -0
- package/dist/memory/provenance-audit.js +112 -0
- package/dist/memory/provenance-query.d.ts +11 -0
- package/dist/memory/provenance-query.js +193 -0
- package/dist/memory/provenance.d.ts +29 -0
- package/dist/memory/provenance.js +137 -0
- package/dist/memory/scope-backfill.d.ts +17 -0
- package/dist/memory/scope-backfill.js +77 -0
- package/dist/memory/types.d.ts +87 -14
- package/dist/model-runs/store.d.ts +13 -0
- package/dist/model-runs/store.js +457 -0
- package/dist/model-runs/tool-trace-store.d.ts +4 -0
- package/dist/model-runs/tool-trace-store.js +114 -0
- package/dist/model-runs/types.d.ts +69 -0
- package/dist/model-runs/types.js +5 -0
- package/dist/runtime/trusted-provenance.d.ts +3 -0
- package/dist/runtime/trusted-provenance.js +6 -0
- package/dist/search/search-quality.d.ts +36 -0
- package/dist/search/search-quality.js +29 -0
- package/package.json +11 -3
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.searchRaw = searchRaw;
|
|
4
|
+
exports.searchAllRaw = searchAllRaw;
|
|
5
|
+
exports.getRawById = getRawById;
|
|
6
|
+
exports.getRawWindow = getRawWindow;
|
|
7
|
+
const types_js_1 = require("../memory/types.js");
|
|
8
|
+
const event_index_js_1 = require("./event-index.js");
|
|
9
|
+
const DEFAULT_RAW_LIMIT = 25;
|
|
10
|
+
const MAX_RAW_LIMIT = 100;
|
|
11
|
+
const DEFAULT_WINDOW_SIZE = 5;
|
|
12
|
+
const MAX_WINDOW_SIZE = 50;
|
|
13
|
+
const VALID_SCOPE_KINDS = new Set(types_js_1.MEMORY_SCOPE_KINDS);
|
|
14
|
+
function placeholders(values) {
|
|
15
|
+
if (values.length === 0) {
|
|
16
|
+
throw new Error('Cannot build SQL placeholders for an empty list.');
|
|
17
|
+
}
|
|
18
|
+
return values.map(() => '?').join(', ');
|
|
19
|
+
}
|
|
20
|
+
function normalizeLimit(value) {
|
|
21
|
+
if (value === undefined) {
|
|
22
|
+
return DEFAULT_RAW_LIMIT;
|
|
23
|
+
}
|
|
24
|
+
return Math.min(MAX_RAW_LIMIT, Math.max(0, Math.floor(value)));
|
|
25
|
+
}
|
|
26
|
+
function normalizeWindowSize(value) {
|
|
27
|
+
if (value === undefined) {
|
|
28
|
+
return DEFAULT_WINDOW_SIZE;
|
|
29
|
+
}
|
|
30
|
+
return Math.min(MAX_WINDOW_SIZE, Math.max(0, Math.floor(value)));
|
|
31
|
+
}
|
|
32
|
+
function escapeFtsQuery(query) {
|
|
33
|
+
return `"${query.replace(/"/g, '""')}"`;
|
|
34
|
+
}
|
|
35
|
+
function encodeCursor(row) {
|
|
36
|
+
return Buffer.from(JSON.stringify({
|
|
37
|
+
rank: row.rank,
|
|
38
|
+
timestampMs: row.source_timestamp_ms,
|
|
39
|
+
rawId: row.event_index_id,
|
|
40
|
+
}), 'utf8').toString('base64url');
|
|
41
|
+
}
|
|
42
|
+
function decodeCursor(cursor) {
|
|
43
|
+
if (!cursor) {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
try {
|
|
47
|
+
const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
|
|
48
|
+
if (typeof parsed.rank !== 'number' ||
|
|
49
|
+
!Number.isFinite(parsed.rank) ||
|
|
50
|
+
typeof parsed.timestampMs !== 'number' ||
|
|
51
|
+
!Number.isFinite(parsed.timestampMs) ||
|
|
52
|
+
typeof parsed.rawId !== 'string' ||
|
|
53
|
+
parsed.rawId.length === 0) {
|
|
54
|
+
throw new Error('invalid shape');
|
|
55
|
+
}
|
|
56
|
+
return {
|
|
57
|
+
rank: parsed.rank,
|
|
58
|
+
timestampMs: parsed.timestampMs,
|
|
59
|
+
rawId: parsed.rawId,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
64
|
+
throw new Error(`Invalid raw search cursor: ${message}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function normalizeConnectors(connectors) {
|
|
68
|
+
return [...new Set((connectors ?? []).map((connector) => connector.trim()).filter(Boolean))];
|
|
69
|
+
}
|
|
70
|
+
function normalizeScopes(scopes) {
|
|
71
|
+
return (scopes ?? [])
|
|
72
|
+
.map((scope) => {
|
|
73
|
+
const kind = scope.kind.trim();
|
|
74
|
+
if (!VALID_SCOPE_KINDS.has(kind)) {
|
|
75
|
+
throw new Error(`Invalid raw search scope kind: ${kind}`);
|
|
76
|
+
}
|
|
77
|
+
return { kind: kind, id: scope.id.trim() };
|
|
78
|
+
})
|
|
79
|
+
.filter((scope) => scope.id.length > 0);
|
|
80
|
+
}
|
|
81
|
+
function appendFilters(clauses, params, input, alias = 'e') {
|
|
82
|
+
const connectors = normalizeConnectors(input.connectors);
|
|
83
|
+
if (connectors.length > 0) {
|
|
84
|
+
clauses.push(`${alias}.source_connector IN (${placeholders(connectors)})`);
|
|
85
|
+
params.push(...connectors);
|
|
86
|
+
}
|
|
87
|
+
const scopes = normalizeScopes(input.scopes);
|
|
88
|
+
if (scopes.length > 0) {
|
|
89
|
+
clauses.push(`(${scopes.map(() => `(${alias}.memory_scope_kind = ? AND ${alias}.memory_scope_id = ?)`).join(' OR ')})`);
|
|
90
|
+
for (const scope of scopes) {
|
|
91
|
+
params.push(scope.kind, scope.id);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (input.fromMs !== undefined) {
|
|
95
|
+
clauses.push(`${alias}.source_timestamp_ms >= ?`);
|
|
96
|
+
params.push(input.fromMs);
|
|
97
|
+
}
|
|
98
|
+
if (input.toMs !== undefined) {
|
|
99
|
+
clauses.push(`${alias}.source_timestamp_ms <= ?`);
|
|
100
|
+
params.push(input.toMs);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
function appendCursorFilter(clauses, params, cursor) {
|
|
104
|
+
if (!cursor) {
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
clauses.push(`(rank > ? OR (rank = ? AND source_timestamp_ms < ?) OR ` +
|
|
108
|
+
`(rank = ? AND source_timestamp_ms = ? AND event_index_id > ?))`);
|
|
109
|
+
params.push(cursor.rank, cursor.rank, cursor.timestampMs, cursor.rank, cursor.timestampMs, cursor.rawId);
|
|
110
|
+
}
|
|
111
|
+
function parseMetadata(metadataJson) {
|
|
112
|
+
if (!metadataJson) {
|
|
113
|
+
return {};
|
|
114
|
+
}
|
|
115
|
+
try {
|
|
116
|
+
const parsed = JSON.parse(metadataJson);
|
|
117
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
118
|
+
? parsed
|
|
119
|
+
: {};
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
return {};
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function contentPreview(content) {
|
|
126
|
+
const compact = content.replace(/\s+/g, ' ').trim();
|
|
127
|
+
return compact.length > 240 ? `${compact.slice(0, 237)}...` : compact;
|
|
128
|
+
}
|
|
129
|
+
function scoreFromRank(rank) {
|
|
130
|
+
return Number.isFinite(rank) ? 1 / (1 + Math.exp(rank)) : 0;
|
|
131
|
+
}
|
|
132
|
+
function timestampToIso(timestampMs) {
|
|
133
|
+
if (timestampMs === null || !Number.isFinite(timestampMs)) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
return new Date(timestampMs).toISOString();
|
|
137
|
+
}
|
|
138
|
+
function toRawHit(row) {
|
|
139
|
+
const record = (0, event_index_js_1.mapConnectorEventIndexRecord)(row);
|
|
140
|
+
return {
|
|
141
|
+
raw_id: record.event_index_id,
|
|
142
|
+
connector: record.source_connector,
|
|
143
|
+
source_id: record.source_id,
|
|
144
|
+
channel_id: record.channel,
|
|
145
|
+
author_label: record.author,
|
|
146
|
+
created_at: timestampToIso(record.event_datetime ?? record.source_timestamp_ms),
|
|
147
|
+
content_preview: contentPreview(record.content),
|
|
148
|
+
score: scoreFromRank(Number(row.rank)),
|
|
149
|
+
source_ref: record.source_locator ?? record.artifact_locator,
|
|
150
|
+
metadata: parseMetadata(record.metadata_json),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function runSearch(adapter, input) {
|
|
154
|
+
const query = input.query.trim();
|
|
155
|
+
if (query.length === 0) {
|
|
156
|
+
return { hits: [], next_cursor: null };
|
|
157
|
+
}
|
|
158
|
+
const limit = normalizeLimit(input.limit);
|
|
159
|
+
if (limit === 0) {
|
|
160
|
+
return { hits: [], next_cursor: null };
|
|
161
|
+
}
|
|
162
|
+
const params = [escapeFtsQuery(query)];
|
|
163
|
+
const clauses = [];
|
|
164
|
+
appendFilters(clauses, params, input);
|
|
165
|
+
const cursor = decodeCursor(input.cursor);
|
|
166
|
+
const outerClauses = [];
|
|
167
|
+
const outerParams = [];
|
|
168
|
+
appendCursorFilter(outerClauses, outerParams, cursor);
|
|
169
|
+
const whereSql = clauses.length > 0 ? `AND ${clauses.join(' AND ')}` : '';
|
|
170
|
+
const cursorSql = outerClauses.length > 0 ? `WHERE ${outerClauses.join(' AND ')}` : '';
|
|
171
|
+
const rows = adapter
|
|
172
|
+
.prepare(`
|
|
173
|
+
WITH ranked AS (
|
|
174
|
+
SELECT e.*, bm25(connector_event_index_fts) AS rank
|
|
175
|
+
FROM connector_event_index_fts
|
|
176
|
+
JOIN connector_event_index e
|
|
177
|
+
ON e.event_index_id = connector_event_index_fts.event_index_id
|
|
178
|
+
WHERE connector_event_index_fts MATCH ?
|
|
179
|
+
${whereSql}
|
|
180
|
+
)
|
|
181
|
+
SELECT *
|
|
182
|
+
FROM ranked
|
|
183
|
+
${cursorSql}
|
|
184
|
+
ORDER BY rank ASC, source_timestamp_ms DESC, event_index_id ASC
|
|
185
|
+
LIMIT ?
|
|
186
|
+
`)
|
|
187
|
+
.all(...params, ...outerParams, limit + 1);
|
|
188
|
+
const pageRows = rows.slice(0, limit);
|
|
189
|
+
const nextRow = rows.length > limit ? pageRows[pageRows.length - 1] : undefined;
|
|
190
|
+
return {
|
|
191
|
+
hits: pageRows.map(toRawHit),
|
|
192
|
+
next_cursor: nextRow
|
|
193
|
+
? encodeCursor({
|
|
194
|
+
rank: Number(nextRow.rank),
|
|
195
|
+
source_timestamp_ms: Number(nextRow.source_timestamp_ms),
|
|
196
|
+
event_index_id: String(nextRow.event_index_id),
|
|
197
|
+
})
|
|
198
|
+
: null,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
function searchRaw(adapter, input) {
|
|
202
|
+
const connectors = normalizeConnectors(input.connectors);
|
|
203
|
+
if (connectors.length !== 1) {
|
|
204
|
+
throw new Error('searchRaw requires exactly one connector filter.');
|
|
205
|
+
}
|
|
206
|
+
return runSearch(adapter, { ...input, connectors });
|
|
207
|
+
}
|
|
208
|
+
function searchAllRaw(adapter, input) {
|
|
209
|
+
return runSearch(adapter, input);
|
|
210
|
+
}
|
|
211
|
+
function getRawById(adapter, rawId, visibility) {
|
|
212
|
+
const params = [rawId];
|
|
213
|
+
const clauses = ['e.event_index_id = ?'];
|
|
214
|
+
appendFilters(clauses, params, { query: '*', ...visibility });
|
|
215
|
+
const row = adapter
|
|
216
|
+
.prepare(`
|
|
217
|
+
SELECT e.*, 0 AS rank
|
|
218
|
+
FROM connector_event_index e
|
|
219
|
+
WHERE ${clauses.join(' AND ')}
|
|
220
|
+
LIMIT 1
|
|
221
|
+
`)
|
|
222
|
+
.get(...params);
|
|
223
|
+
return row ? toRawHit(row) : null;
|
|
224
|
+
}
|
|
225
|
+
function getRawWindow(adapter, rawId, input) {
|
|
226
|
+
const targetParams = [rawId];
|
|
227
|
+
const targetClauses = ['e.event_index_id = ?'];
|
|
228
|
+
appendFilters(targetClauses, targetParams, {
|
|
229
|
+
query: '*',
|
|
230
|
+
connectors: input.connectors,
|
|
231
|
+
scopes: input.scopes,
|
|
232
|
+
});
|
|
233
|
+
const targetRow = adapter
|
|
234
|
+
.prepare(`
|
|
235
|
+
SELECT e.*, 0 AS rank
|
|
236
|
+
FROM connector_event_index e
|
|
237
|
+
WHERE ${targetClauses.join(' AND ')}
|
|
238
|
+
LIMIT 1
|
|
239
|
+
`)
|
|
240
|
+
.get(...targetParams);
|
|
241
|
+
if (!targetRow) {
|
|
242
|
+
return null;
|
|
243
|
+
}
|
|
244
|
+
const target = (0, event_index_js_1.mapConnectorEventIndexRecord)(targetRow);
|
|
245
|
+
const before = normalizeWindowSize(input.before);
|
|
246
|
+
const after = normalizeWindowSize(input.after);
|
|
247
|
+
const beforeRows = beforeWindowRows(adapter, target, input, before);
|
|
248
|
+
const afterRows = afterWindowRows(adapter, target, input, after);
|
|
249
|
+
const targetHit = toRawHit(targetRow);
|
|
250
|
+
return {
|
|
251
|
+
target: targetHit,
|
|
252
|
+
items: [...beforeRows.reverse().map(toRawHit), targetHit, ...afterRows.map(toRawHit)],
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
function beforeWindowRows(adapter, target, input, limit) {
|
|
256
|
+
if (limit === 0) {
|
|
257
|
+
return [];
|
|
258
|
+
}
|
|
259
|
+
const params = [target.source_connector, target.channel, target.source_timestamp_ms];
|
|
260
|
+
const clauses = [
|
|
261
|
+
'e.source_connector = ?',
|
|
262
|
+
target.channel === null ? 'e.channel IS ?' : 'e.channel = ?',
|
|
263
|
+
'e.source_timestamp_ms < ?',
|
|
264
|
+
];
|
|
265
|
+
appendFilters(clauses, params, {
|
|
266
|
+
query: '*',
|
|
267
|
+
connectors: input.connectors,
|
|
268
|
+
scopes: input.scopes,
|
|
269
|
+
});
|
|
270
|
+
return adapter
|
|
271
|
+
.prepare(`
|
|
272
|
+
SELECT e.*, 0 AS rank
|
|
273
|
+
FROM connector_event_index e
|
|
274
|
+
WHERE ${clauses.join(' AND ')}
|
|
275
|
+
ORDER BY e.source_timestamp_ms DESC, e.event_index_id DESC
|
|
276
|
+
LIMIT ?
|
|
277
|
+
`)
|
|
278
|
+
.all(...params, limit);
|
|
279
|
+
}
|
|
280
|
+
function afterWindowRows(adapter, target, input, limit) {
|
|
281
|
+
if (limit === 0) {
|
|
282
|
+
return [];
|
|
283
|
+
}
|
|
284
|
+
const params = [target.source_connector, target.channel, target.source_timestamp_ms];
|
|
285
|
+
const clauses = [
|
|
286
|
+
'e.source_connector = ?',
|
|
287
|
+
target.channel === null ? 'e.channel IS ?' : 'e.channel = ?',
|
|
288
|
+
'e.source_timestamp_ms > ?',
|
|
289
|
+
];
|
|
290
|
+
appendFilters(clauses, params, {
|
|
291
|
+
query: '*',
|
|
292
|
+
connectors: input.connectors,
|
|
293
|
+
scopes: input.scopes,
|
|
294
|
+
});
|
|
295
|
+
return adapter
|
|
296
|
+
.prepare(`
|
|
297
|
+
SELECT e.*, 0 AS rank
|
|
298
|
+
FROM connector_event_index e
|
|
299
|
+
WHERE ${clauses.join(' AND ')}
|
|
300
|
+
ORDER BY e.source_timestamp_ms ASC, e.event_index_id ASC
|
|
301
|
+
LIMIT ?
|
|
302
|
+
`)
|
|
303
|
+
.all(...params, limit);
|
|
304
|
+
}
|
|
305
|
+
//# sourceMappingURL=raw-query.js.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Buffer } from 'node:buffer';
|
|
2
|
+
import type { MemoryScopeRef } from '../memory/types.js';
|
|
2
3
|
export interface ConnectorEventIndexRecord {
|
|
3
4
|
event_index_id: string;
|
|
4
5
|
source_connector: string;
|
|
@@ -12,6 +13,11 @@ export interface ConnectorEventIndexRecord {
|
|
|
12
13
|
event_datetime: number | null;
|
|
13
14
|
event_date: string | null;
|
|
14
15
|
source_timestamp_ms: number;
|
|
16
|
+
source_cursor: string | null;
|
|
17
|
+
tenant_id: string | null;
|
|
18
|
+
project_id: string | null;
|
|
19
|
+
memory_scope_kind: string | null;
|
|
20
|
+
memory_scope_id: string | null;
|
|
15
21
|
metadata_json: string | null;
|
|
16
22
|
artifact_locator: string | null;
|
|
17
23
|
artifact_title: string | null;
|
|
@@ -32,6 +38,11 @@ export interface UpsertConnectorEventIndexInput {
|
|
|
32
38
|
event_datetime?: number | null;
|
|
33
39
|
event_date?: string | null;
|
|
34
40
|
source_timestamp_ms?: number | null;
|
|
41
|
+
source_cursor?: string | null;
|
|
42
|
+
tenant_id?: string | null;
|
|
43
|
+
project_id?: string | null;
|
|
44
|
+
memory_scope_kind?: string | null;
|
|
45
|
+
memory_scope_id?: string | null;
|
|
35
46
|
metadata_json?: string | null;
|
|
36
47
|
metadata?: unknown;
|
|
37
48
|
artifact_locator?: string | null;
|
|
@@ -65,5 +76,34 @@ export interface ConnectorEventSearchHit extends ConnectorEventIndexRecord {
|
|
|
65
76
|
rank: number;
|
|
66
77
|
score: number;
|
|
67
78
|
}
|
|
79
|
+
export interface RawSearchScopeFilter {
|
|
80
|
+
kind: MemoryScopeRef['kind'];
|
|
81
|
+
id: string;
|
|
82
|
+
}
|
|
83
|
+
export interface RawSearchInput {
|
|
84
|
+
query: string;
|
|
85
|
+
connectors?: string[];
|
|
86
|
+
scopes?: RawSearchScopeFilter[];
|
|
87
|
+
fromMs?: number;
|
|
88
|
+
toMs?: number;
|
|
89
|
+
cursor?: string;
|
|
90
|
+
limit?: number;
|
|
91
|
+
}
|
|
92
|
+
export interface RawSearchHit {
|
|
93
|
+
raw_id: string;
|
|
94
|
+
connector: string;
|
|
95
|
+
source_id: string;
|
|
96
|
+
channel_id: string | null;
|
|
97
|
+
author_label: string | null;
|
|
98
|
+
created_at: string | null;
|
|
99
|
+
content_preview: string;
|
|
100
|
+
score: number;
|
|
101
|
+
source_ref: string | null;
|
|
102
|
+
metadata: Record<string, unknown>;
|
|
103
|
+
}
|
|
104
|
+
export interface RawSearchResult {
|
|
105
|
+
hits: RawSearchHit[];
|
|
106
|
+
next_cursor: string | null;
|
|
107
|
+
}
|
|
68
108
|
export type ConnectorEventStalenessStatus = 'healthy' | 'stale-but-warming' | 'warn' | 'unhealthy' | 'never_swept';
|
|
69
109
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ContextBoundary, ContextProjectRef, ContextRange } from './types.js';
|
|
2
|
+
import type { MemoryScopeRef } from '../memory/types.js';
|
|
3
|
+
export type ContextBoundaryReadableInput = {
|
|
4
|
+
scopes?: MemoryScopeRef[];
|
|
5
|
+
connectors?: string[];
|
|
6
|
+
project_refs?: ContextProjectRef[];
|
|
7
|
+
tenant_id?: string | null;
|
|
8
|
+
range?: ContextRange;
|
|
9
|
+
as_of?: string | number | null;
|
|
10
|
+
};
|
|
11
|
+
export declare function applyContextBoundaryReadDefaults<T extends ContextBoundaryReadableInput>(input: T, boundary: ContextBoundary | undefined): T;
|
|
12
|
+
//# sourceMappingURL=boundary-defaults.d.ts.map
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.applyContextBoundaryReadDefaults = applyContextBoundaryReadDefaults;
|
|
4
|
+
function parseBoundaryTimeMs(value, field) {
|
|
5
|
+
if (value === null || value === undefined) {
|
|
6
|
+
return null;
|
|
7
|
+
}
|
|
8
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
9
|
+
return Math.floor(value);
|
|
10
|
+
}
|
|
11
|
+
if (typeof value === 'string') {
|
|
12
|
+
const trimmed = value.trim();
|
|
13
|
+
if (trimmed.length === 0) {
|
|
14
|
+
throw new Error(`Invalid context boundary ${field}: ${String(value)}`);
|
|
15
|
+
}
|
|
16
|
+
const numeric = Number(trimmed);
|
|
17
|
+
if (Number.isFinite(numeric)) {
|
|
18
|
+
return Math.floor(numeric);
|
|
19
|
+
}
|
|
20
|
+
const parsed = Date.parse(trimmed);
|
|
21
|
+
if (Number.isFinite(parsed)) {
|
|
22
|
+
return parsed;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
throw new Error(`Invalid context boundary ${field}: ${String(value)}`);
|
|
26
|
+
}
|
|
27
|
+
function rangeBoundaryMs(value, field) {
|
|
28
|
+
if (value === undefined) {
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
32
|
+
return Math.floor(value);
|
|
33
|
+
}
|
|
34
|
+
throw new Error(`Invalid context boundary ${field}: ${String(value)}`);
|
|
35
|
+
}
|
|
36
|
+
function intersectRange(range, boundaryRange) {
|
|
37
|
+
if (!boundaryRange) {
|
|
38
|
+
return range;
|
|
39
|
+
}
|
|
40
|
+
const requestedStartMs = rangeBoundaryMs(range?.start_ms, 'range.start_ms');
|
|
41
|
+
const requestedEndMs = rangeBoundaryMs(range?.end_ms, 'range.end_ms');
|
|
42
|
+
const boundaryStartMs = rangeBoundaryMs(boundaryRange.start_ms, 'boundary.range.start_ms');
|
|
43
|
+
const boundaryEndMs = rangeBoundaryMs(boundaryRange.end_ms, 'boundary.range.end_ms');
|
|
44
|
+
const startMs = requestedStartMs === null
|
|
45
|
+
? boundaryStartMs
|
|
46
|
+
: boundaryStartMs === null
|
|
47
|
+
? requestedStartMs
|
|
48
|
+
: Math.max(requestedStartMs, boundaryStartMs);
|
|
49
|
+
const endMs = requestedEndMs === null
|
|
50
|
+
? boundaryEndMs
|
|
51
|
+
: boundaryEndMs === null
|
|
52
|
+
? requestedEndMs
|
|
53
|
+
: Math.min(requestedEndMs, boundaryEndMs);
|
|
54
|
+
if (startMs === null && endMs === null) {
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
if (startMs !== null && endMs !== null && startMs > endMs) {
|
|
58
|
+
throw new RangeError(`Context boundary range is empty after intersection: start_ms ${startMs} > end_ms ${endMs}`);
|
|
59
|
+
}
|
|
60
|
+
return {
|
|
61
|
+
...(startMs !== null ? { start_ms: startMs } : {}),
|
|
62
|
+
...(endMs !== null ? { end_ms: endMs } : {}),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
function clampAsOf(requested, boundaryAsOf) {
|
|
66
|
+
const requestedMs = parseBoundaryTimeMs(requested, 'as_of');
|
|
67
|
+
const boundaryMs = parseBoundaryTimeMs(boundaryAsOf, 'boundary.as_of');
|
|
68
|
+
if (requestedMs === null) {
|
|
69
|
+
return boundaryAsOf ?? null;
|
|
70
|
+
}
|
|
71
|
+
if (boundaryMs === null) {
|
|
72
|
+
return requested ?? null;
|
|
73
|
+
}
|
|
74
|
+
return requestedMs <= boundaryMs ? (requested ?? null) : (boundaryAsOf ?? null);
|
|
75
|
+
}
|
|
76
|
+
function applyContextBoundaryReadDefaults(input, boundary) {
|
|
77
|
+
if (!boundary) {
|
|
78
|
+
return input;
|
|
79
|
+
}
|
|
80
|
+
const requestedTenantId = input.tenant_id;
|
|
81
|
+
const boundaryTenantId = boundary.tenant_id;
|
|
82
|
+
if (requestedTenantId !== undefined &&
|
|
83
|
+
requestedTenantId !== null &&
|
|
84
|
+
boundaryTenantId !== undefined &&
|
|
85
|
+
boundaryTenantId !== null &&
|
|
86
|
+
requestedTenantId !== boundaryTenantId) {
|
|
87
|
+
throw new Error('Requested tenant is outside the context boundary');
|
|
88
|
+
}
|
|
89
|
+
const scopedInput = {
|
|
90
|
+
...input,
|
|
91
|
+
scopes: input.scopes === undefined ? boundary.scopes : input.scopes,
|
|
92
|
+
connectors: input.connectors === undefined ? boundary.connectors : input.connectors,
|
|
93
|
+
project_refs: input.project_refs === undefined ? boundary.project_refs : input.project_refs,
|
|
94
|
+
range: intersectRange(input.range, boundary.range),
|
|
95
|
+
as_of: clampAsOf(input.as_of, boundary.as_of),
|
|
96
|
+
};
|
|
97
|
+
if (boundaryTenantId !== undefined || requestedTenantId !== undefined) {
|
|
98
|
+
return {
|
|
99
|
+
...scopedInput,
|
|
100
|
+
tenant_id: boundaryTenantId !== undefined ? boundaryTenantId : requestedTenantId,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
return scopedInput;
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=boundary-defaults.js.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { ContextEvidence, ContextRef } from './types.js';
|
|
2
|
+
import type { ContextCandidate, HiddenCandidateAggregate } from './source-readers.js';
|
|
3
|
+
export interface ContextCompilerPolicyInput {
|
|
4
|
+
task: string;
|
|
5
|
+
candidates: ContextCandidate[];
|
|
6
|
+
hidden: HiddenCandidateAggregate;
|
|
7
|
+
limit?: number;
|
|
8
|
+
strictness?: 'recall' | 'balanced' | 'strict' | 'low' | 'medium' | 'high';
|
|
9
|
+
max_tokens?: number;
|
|
10
|
+
}
|
|
11
|
+
export interface ContextCompilerPolicyResult {
|
|
12
|
+
selected_evidence: ContextEvidence[];
|
|
13
|
+
source_refs: ContextRef[];
|
|
14
|
+
evidence_clusters: unknown[];
|
|
15
|
+
related_decisions: Array<{
|
|
16
|
+
memory_id: string;
|
|
17
|
+
title: string;
|
|
18
|
+
}>;
|
|
19
|
+
rejected_refs: ContextRef[];
|
|
20
|
+
rejected_summary: string[];
|
|
21
|
+
missing_context: string[];
|
|
22
|
+
caveats: string[];
|
|
23
|
+
retrieval_diagnostics: {
|
|
24
|
+
candidate_count: number;
|
|
25
|
+
selected_count: number;
|
|
26
|
+
rejected_count: number;
|
|
27
|
+
hidden: HiddenCandidateAggregate;
|
|
28
|
+
deduplicated_count: number;
|
|
29
|
+
strict_vector_only_rejected_count: number;
|
|
30
|
+
token_budget_rejected_count: number;
|
|
31
|
+
limit_rejected_count: number;
|
|
32
|
+
truncated_by_tokens: boolean;
|
|
33
|
+
estimated_tokens: number;
|
|
34
|
+
};
|
|
35
|
+
estimated_tokens: number;
|
|
36
|
+
}
|
|
37
|
+
export declare function estimateEvidenceTokens(evidence: Pick<ContextEvidence, 'title' | 'excerpt'>): number;
|
|
38
|
+
export declare function applyContextCompilerPolicy(input: ContextCompilerPolicyInput): ContextCompilerPolicyResult;
|
|
39
|
+
//# sourceMappingURL=compiler-policy.d.ts.map
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.estimateEvidenceTokens = estimateEvidenceTokens;
|
|
4
|
+
exports.applyContextCompilerPolicy = applyContextCompilerPolicy;
|
|
5
|
+
const ref_js_1 = require("./ref.js");
|
|
6
|
+
function normalizeLimit(limit) {
|
|
7
|
+
return Math.max(0, Math.min(100, Math.floor(limit ?? 10)));
|
|
8
|
+
}
|
|
9
|
+
function estimateEvidenceTokens(evidence) {
|
|
10
|
+
const text = `${evidence.title ?? ''}\n${evidence.excerpt ?? ''}`;
|
|
11
|
+
return Math.max(1, Math.ceil(text.length / 4));
|
|
12
|
+
}
|
|
13
|
+
function sortedCandidates(candidates) {
|
|
14
|
+
return [...candidates].sort((left, right) => {
|
|
15
|
+
const scoreDiff = right.score - left.score;
|
|
16
|
+
if (scoreDiff !== 0) {
|
|
17
|
+
return scoreDiff;
|
|
18
|
+
}
|
|
19
|
+
return (right.timestamp_ms ?? 0) - (left.timestamp_ms ?? 0);
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
function shouldRejectVectorOnly(candidate, strictness) {
|
|
23
|
+
return (strictness === 'strict' &&
|
|
24
|
+
candidate.support.is_vector_only === true &&
|
|
25
|
+
candidate.support.confirmation_signals.length === 0);
|
|
26
|
+
}
|
|
27
|
+
function evidenceFromCandidate(candidate) {
|
|
28
|
+
return {
|
|
29
|
+
ref: candidate.ref,
|
|
30
|
+
title: candidate.title,
|
|
31
|
+
excerpt: candidate.excerpt,
|
|
32
|
+
score: candidate.score,
|
|
33
|
+
reasons: [
|
|
34
|
+
...(candidate.support.confirmation_signals ?? []),
|
|
35
|
+
...(candidate.support.graph_expanded ? ['graph_expanded'] : []),
|
|
36
|
+
],
|
|
37
|
+
...(candidate.retrieval_diagnostics
|
|
38
|
+
? { retrieval_diagnostics: candidate.retrieval_diagnostics }
|
|
39
|
+
: {}),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function addCountSummary(parts, label, count) {
|
|
43
|
+
if (count > 0) {
|
|
44
|
+
parts.push(`${label}: ${count}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function normalizeStrictness(strictness) {
|
|
48
|
+
switch (strictness) {
|
|
49
|
+
case 'low':
|
|
50
|
+
case 'recall':
|
|
51
|
+
return 'recall';
|
|
52
|
+
case 'high':
|
|
53
|
+
case 'strict':
|
|
54
|
+
return 'strict';
|
|
55
|
+
case 'medium':
|
|
56
|
+
case 'balanced':
|
|
57
|
+
case undefined:
|
|
58
|
+
return 'balanced';
|
|
59
|
+
}
|
|
60
|
+
return 'balanced';
|
|
61
|
+
}
|
|
62
|
+
function applyContextCompilerPolicy(input) {
|
|
63
|
+
const limit = normalizeLimit(input.limit);
|
|
64
|
+
const strictness = normalizeStrictness(input.strictness);
|
|
65
|
+
const maxTokens = typeof input.max_tokens === 'number' && Number.isFinite(input.max_tokens)
|
|
66
|
+
? Math.max(0, Math.floor(input.max_tokens))
|
|
67
|
+
: null;
|
|
68
|
+
const seen = new Set();
|
|
69
|
+
const selected = [];
|
|
70
|
+
const sourceRefs = [];
|
|
71
|
+
const rejectedRefs = [];
|
|
72
|
+
const counters = {
|
|
73
|
+
deduplicated: 0,
|
|
74
|
+
strictVectorOnly: 0,
|
|
75
|
+
tokenBudget: 0,
|
|
76
|
+
limit: 0,
|
|
77
|
+
};
|
|
78
|
+
let estimatedTokens = 0;
|
|
79
|
+
for (const candidate of sortedCandidates(input.candidates)) {
|
|
80
|
+
if (!candidate.visible) {
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const key = (0, ref_js_1.serializeContextRefForProvenance)(candidate.ref);
|
|
84
|
+
if (seen.has(key)) {
|
|
85
|
+
counters.deduplicated += 1;
|
|
86
|
+
rejectedRefs.push(candidate.ref);
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (shouldRejectVectorOnly(candidate, strictness)) {
|
|
90
|
+
counters.strictVectorOnly += 1;
|
|
91
|
+
rejectedRefs.push(candidate.ref);
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
if (selected.length >= limit) {
|
|
95
|
+
counters.limit += 1;
|
|
96
|
+
rejectedRefs.push(candidate.ref);
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
const evidence = evidenceFromCandidate(candidate);
|
|
100
|
+
const nextTokens = estimateEvidenceTokens(evidence);
|
|
101
|
+
if (maxTokens !== null && estimatedTokens + nextTokens > maxTokens) {
|
|
102
|
+
counters.tokenBudget += 1;
|
|
103
|
+
rejectedRefs.push(candidate.ref);
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
selected.push(evidence);
|
|
107
|
+
sourceRefs.push(candidate.ref);
|
|
108
|
+
seen.add(key);
|
|
109
|
+
estimatedTokens += nextTokens;
|
|
110
|
+
}
|
|
111
|
+
const rejectedSummary = [];
|
|
112
|
+
addCountSummary(rejectedSummary, 'deduplicated duplicate candidates', counters.deduplicated);
|
|
113
|
+
addCountSummary(rejectedSummary, 'strictness rejected vector-only candidates', counters.strictVectorOnly);
|
|
114
|
+
addCountSummary(rejectedSummary, 'token budget rejected candidates', counters.tokenBudget);
|
|
115
|
+
addCountSummary(rejectedSummary, 'limit rejected candidates', counters.limit);
|
|
116
|
+
const caveats = [];
|
|
117
|
+
if (input.hidden.total > 0) {
|
|
118
|
+
caveats.push(`hidden candidates omitted: ${input.hidden.total}`);
|
|
119
|
+
}
|
|
120
|
+
const relatedDecisions = selected.flatMap((evidence) => evidence.ref.kind === 'memory'
|
|
121
|
+
? [{ memory_id: evidence.ref.id, title: evidence.title ?? evidence.ref.id }]
|
|
122
|
+
: []);
|
|
123
|
+
return {
|
|
124
|
+
selected_evidence: selected,
|
|
125
|
+
source_refs: sourceRefs,
|
|
126
|
+
evidence_clusters: [],
|
|
127
|
+
related_decisions: relatedDecisions,
|
|
128
|
+
rejected_refs: rejectedRefs,
|
|
129
|
+
rejected_summary: rejectedSummary,
|
|
130
|
+
missing_context: selected.length === 0 ? [`No visible evidence selected for task: ${input.task}`] : [],
|
|
131
|
+
caveats,
|
|
132
|
+
retrieval_diagnostics: {
|
|
133
|
+
candidate_count: input.candidates.length,
|
|
134
|
+
selected_count: selected.length,
|
|
135
|
+
rejected_count: rejectedRefs.length,
|
|
136
|
+
hidden: input.hidden,
|
|
137
|
+
deduplicated_count: counters.deduplicated,
|
|
138
|
+
strict_vector_only_rejected_count: counters.strictVectorOnly,
|
|
139
|
+
token_budget_rejected_count: counters.tokenBudget,
|
|
140
|
+
limit_rejected_count: counters.limit,
|
|
141
|
+
truncated_by_tokens: counters.tokenBudget > 0,
|
|
142
|
+
estimated_tokens: estimatedTokens,
|
|
143
|
+
},
|
|
144
|
+
estimated_tokens: estimatedTokens,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=compiler-policy.js.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { DatabaseAdapter } from '../db-manager.js';
|
|
2
|
+
import type { ContextBoundary, ContextCompileInput, ContextPacket, ContextRef } from './types.js';
|
|
3
|
+
import { type ContextSourceReadInput, type ContextSourceReadResult } from './source-readers.js';
|
|
4
|
+
type ContextCompilerAdapter = Pick<DatabaseAdapter, 'prepare'>;
|
|
5
|
+
export interface ContextCompilerDeps {
|
|
6
|
+
adapter?: ContextCompilerAdapter;
|
|
7
|
+
boundary?: ContextBoundary;
|
|
8
|
+
signal?: AbortSignal;
|
|
9
|
+
deadlineMs?: number;
|
|
10
|
+
now?: () => number;
|
|
11
|
+
packetId?: () => string;
|
|
12
|
+
readMemoryCandidates?: (input: ContextSourceReadInput) => Promise<ContextSourceReadResult> | ContextSourceReadResult;
|
|
13
|
+
readRawCandidates?: (input: ContextSourceReadInput) => Promise<ContextSourceReadResult> | ContextSourceReadResult;
|
|
14
|
+
readGraphCandidates?: (input: ContextSourceReadInput, visibleRefs: readonly ContextRef[]) => Promise<ContextSourceReadResult> | ContextSourceReadResult;
|
|
15
|
+
}
|
|
16
|
+
export declare function compileContext(input: ContextCompileInput, deps?: ContextCompilerDeps): Promise<ContextPacket>;
|
|
17
|
+
export {};
|
|
18
|
+
//# sourceMappingURL=compiler.d.ts.map
|