@jungjaehoon/mama-core 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/026-create-canonical-entity-tables.sql +99 -0
- package/db/migrations/027-create-entity-review-support.sql +44 -0
- package/db/migrations/028-create-entity-audit-tables.sql +45 -0
- package/db/migrations/029-extend-entity-observations-observation-type.sql +60 -0
- package/db/migrations/030-case-first-memory-substrate.sql +901 -0
- package/db/migrations/031-wiki-page-index-provenance-columns.sql +2 -0
- package/dist/canonicalize.d.ts +38 -0
- package/dist/canonicalize.js +106 -0
- package/dist/cases/case-links.d.ts +116 -0
- package/dist/cases/case-links.js +794 -0
- package/dist/cases/composition-overrides.d.ts +90 -0
- package/dist/cases/composition-overrides.js +724 -0
- package/dist/cases/corrections.d.ts +156 -0
- package/dist/cases/corrections.js +1045 -0
- package/dist/cases/freshness.d.ts +57 -0
- package/dist/cases/freshness.js +283 -0
- package/dist/cases/live-state.d.ts +48 -0
- package/dist/cases/live-state.js +256 -0
- package/dist/cases/membership-explain.d.ts +56 -0
- package/dist/cases/membership-explain.js +140 -0
- package/dist/cases/membership-matcher.d.ts +23 -0
- package/dist/cases/membership-matcher.js +193 -0
- package/dist/cases/merge-split.d.ts +61 -0
- package/dist/cases/merge-split.js +368 -0
- package/dist/cases/role-inference.d.ts +10 -0
- package/dist/cases/role-inference.js +85 -0
- package/dist/cases/search-rollup.d.ts +23 -0
- package/dist/cases/search-rollup.js +225 -0
- package/dist/cases/sqlite-transaction.d.ts +10 -0
- package/dist/cases/sqlite-transaction.js +62 -0
- package/dist/cases/store.d.ts +74 -0
- package/dist/cases/store.js +816 -0
- package/dist/cases/target-ref.d.ts +25 -0
- package/dist/cases/target-ref.js +22 -0
- package/dist/cases/timeline-range.d.ts +6 -0
- package/dist/cases/timeline-range.js +322 -0
- package/dist/cases/tombstone-sweeper.d.ts +13 -0
- package/dist/cases/tombstone-sweeper.js +128 -0
- package/dist/cases/types.d.ts +244 -0
- package/dist/cases/types.js +47 -0
- package/dist/cases/wiki-page-index.d.ts +50 -0
- package/dist/cases/wiki-page-index.js +350 -0
- package/dist/connectors/event-index.d.ts +44 -0
- package/dist/connectors/event-index.js +366 -0
- package/dist/connectors/types.d.ts +69 -0
- package/dist/connectors/types.js +3 -0
- package/dist/db-manager.d.ts +4 -0
- package/dist/db-manager.js +10 -3
- package/dist/decision-formatter.d.ts +2 -0
- package/dist/decision-formatter.js +11 -8
- package/dist/embedding-server/mobile/daemon.d.ts +1 -0
- package/dist/embedding-server/mobile/daemon.js +11 -1
- package/dist/entities/audit-metrics.d.ts +73 -0
- package/dist/entities/audit-metrics.js +128 -0
- package/dist/entities/candidate-generator.d.ts +9 -0
- package/dist/entities/candidate-generator.js +436 -0
- package/dist/entities/entity-impact.d.ts +41 -0
- package/dist/entities/entity-impact.js +140 -0
- package/dist/entities/entity-linked-decision-counts.d.ts +2 -0
- package/dist/entities/entity-linked-decision-counts.js +29 -0
- package/dist/entities/entity-list.d.ts +23 -0
- package/dist/entities/entity-list.js +99 -0
- package/dist/entities/entity-orphan-list.d.ts +26 -0
- package/dist/entities/entity-orphan-list.js +91 -0
- package/dist/entities/entity-search.d.ts +21 -0
- package/dist/entities/entity-search.js +150 -0
- package/dist/entities/errors.d.ts +65 -0
- package/dist/entities/errors.js +124 -0
- package/dist/entities/exact-merge-backfill.d.ts +11 -0
- package/dist/entities/exact-merge-backfill.js +161 -0
- package/dist/entities/lineage-backfill.d.ts +12 -0
- package/dist/entities/lineage-backfill.js +101 -0
- package/dist/entities/lineage-store.d.ts +51 -0
- package/dist/entities/lineage-store.js +308 -0
- package/dist/entities/normalization.d.ts +14 -0
- package/dist/entities/normalization.js +63 -0
- package/dist/entities/policy-store.d.ts +17 -0
- package/dist/entities/policy-store.js +263 -0
- package/dist/entities/policy-types.d.ts +72 -0
- package/dist/entities/policy-types.js +59 -0
- package/dist/entities/projection.d.ts +8 -0
- package/dist/entities/projection.js +82 -0
- package/dist/entities/provenance-query.d.ts +33 -0
- package/dist/entities/provenance-query.js +167 -0
- package/dist/entities/read-identity.d.ts +31 -0
- package/dist/entities/read-identity.js +264 -0
- package/dist/entities/recall-bridge.d.ts +5 -0
- package/dist/entities/recall-bridge.js +171 -0
- package/dist/entities/resolution-engine.d.ts +15 -0
- package/dist/entities/resolution-engine.js +69 -0
- package/dist/entities/rollback-preview.d.ts +39 -0
- package/dist/entities/rollback-preview.js +235 -0
- package/dist/entities/source-locator.d.ts +3 -0
- package/dist/entities/source-locator.js +32 -0
- package/dist/entities/store.d.ts +75 -0
- package/dist/entities/store.js +465 -0
- package/dist/entities/types.d.ts +162 -0
- package/dist/entities/types.js +30 -0
- package/dist/index.d.ts +47 -2
- package/dist/index.js +66 -25
- package/dist/mama-api.d.ts +22 -1
- package/dist/mama-api.js +272 -64
- package/dist/memory/api.d.ts +28 -1
- package/dist/memory/api.js +293 -31
- package/dist/memory/types.d.ts +11 -2
- package/dist/search/feedback-store.d.ts +59 -0
- package/dist/search/feedback-store.js +211 -0
- package/dist/search/question-type.d.ts +5 -0
- package/dist/search/question-type.js +41 -0
- package/dist/search/ranker-features.d.ts +23 -0
- package/dist/search/ranker-features.js +104 -0
- package/dist/search/ranker-rescore.d.ts +31 -0
- package/dist/search/ranker-rescore.js +226 -0
- package/dist/search/ranker-trainer.d.ts +94 -0
- package/dist/search/ranker-trainer.js +860 -0
- package/dist/test-utils.d.ts +14 -0
- package/dist/test-utils.js +52 -0
- package/package.json +21 -3
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { DatabaseAdapter } from '../db-manager.js';
|
|
2
|
+
import type { ConnectorEventIndexCursorRecord, ConnectorEventIndexRecord, ConnectorEventSearchHit, ConnectorEventStalenessStatus, UpsertConnectorEventIndexCursorInput, UpsertConnectorEventIndexInput } from './types.js';
|
|
3
|
+
type ConnectorEventIndexAdapter = Pick<DatabaseAdapter, 'prepare' | 'transaction'>;
|
|
4
|
+
interface ListConnectorEventsByDatetimeRangeInput {
|
|
5
|
+
fromMs: number;
|
|
6
|
+
toMs: number;
|
|
7
|
+
limit?: number;
|
|
8
|
+
connectors?: string[];
|
|
9
|
+
order?: 'asc' | 'desc';
|
|
10
|
+
}
|
|
11
|
+
interface SearchConnectorEventsOptions {
|
|
12
|
+
limit?: number;
|
|
13
|
+
connectors?: string[];
|
|
14
|
+
}
|
|
15
|
+
interface DeleteExpiredConnectorEventsInput {
|
|
16
|
+
nowMs: number;
|
|
17
|
+
retentionMs: number;
|
|
18
|
+
connectorName?: string;
|
|
19
|
+
}
|
|
20
|
+
interface StalenessOptions {
|
|
21
|
+
nowMs: number;
|
|
22
|
+
staleAfterMs?: number;
|
|
23
|
+
stalenessWarnAfterMs?: number;
|
|
24
|
+
unhealthyAfterMs?: number;
|
|
25
|
+
}
|
|
26
|
+
export declare function connectorEventIndexId(sourceConnector: string, sourceId: string): string;
|
|
27
|
+
export declare function connectorEventContentHash(input: {
|
|
28
|
+
source_connector: string;
|
|
29
|
+
source_id: string;
|
|
30
|
+
content: string;
|
|
31
|
+
event_datetime?: number | null;
|
|
32
|
+
}): Buffer;
|
|
33
|
+
export declare function upsertConnectorEventIndex(adapter: ConnectorEventIndexAdapter, input: UpsertConnectorEventIndexInput): ConnectorEventIndexRecord;
|
|
34
|
+
export declare function getConnectorEventIndexRecord(adapter: ConnectorEventIndexAdapter, sourceConnector: string, sourceId: string): ConnectorEventIndexRecord | null;
|
|
35
|
+
export declare function listConnectorEventsByDatetimeRange(adapter: ConnectorEventIndexAdapter, input: ListConnectorEventsByDatetimeRangeInput): ConnectorEventIndexRecord[];
|
|
36
|
+
export declare function searchConnectorEventsByFTS(adapter: ConnectorEventIndexAdapter, query: string, options?: SearchConnectorEventsOptions): ConnectorEventSearchHit[];
|
|
37
|
+
export declare function readConnectorCursor(adapter: ConnectorEventIndexAdapter, connectorName: string): ConnectorEventIndexCursorRecord | null;
|
|
38
|
+
export declare function upsertConnectorCursor(adapter: ConnectorEventIndexAdapter, input: UpsertConnectorEventIndexCursorInput): ConnectorEventIndexCursorRecord;
|
|
39
|
+
export declare function deleteExpiredConnectorEvents(adapter: ConnectorEventIndexAdapter, input: DeleteExpiredConnectorEventsInput): {
|
|
40
|
+
rows_deleted: number;
|
|
41
|
+
};
|
|
42
|
+
export declare function computeStalenessStatus(cursor: ConnectorEventIndexCursorRecord | null, options: StalenessOptions): ConnectorEventStalenessStatus;
|
|
43
|
+
export type { ConnectorEventIndexCursorRecord, ConnectorEventIndexRecord, ConnectorEventSearchHit, ConnectorEventStalenessStatus, UpsertConnectorEventIndexCursorInput, UpsertConnectorEventIndexInput, } from './types.js';
|
|
44
|
+
//# sourceMappingURL=event-index.d.ts.map
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.connectorEventIndexId = connectorEventIndexId;
|
|
4
|
+
exports.connectorEventContentHash = connectorEventContentHash;
|
|
5
|
+
exports.upsertConnectorEventIndex = upsertConnectorEventIndex;
|
|
6
|
+
exports.getConnectorEventIndexRecord = getConnectorEventIndexRecord;
|
|
7
|
+
exports.listConnectorEventsByDatetimeRange = listConnectorEventsByDatetimeRange;
|
|
8
|
+
exports.searchConnectorEventsByFTS = searchConnectorEventsByFTS;
|
|
9
|
+
exports.readConnectorCursor = readConnectorCursor;
|
|
10
|
+
exports.upsertConnectorCursor = upsertConnectorCursor;
|
|
11
|
+
exports.deleteExpiredConnectorEvents = deleteExpiredConnectorEvents;
|
|
12
|
+
exports.computeStalenessStatus = computeStalenessStatus;
|
|
13
|
+
const node_crypto_1 = require("node:crypto");
|
|
14
|
+
const canonicalize_js_1 = require("../canonicalize.js");
|
|
15
|
+
const DEFAULT_RANGE_LIMIT = 100;
|
|
16
|
+
const DEFAULT_SEARCH_LIMIT = 25;
|
|
17
|
+
const MAX_RANGE_LIMIT = 500;
|
|
18
|
+
const FIVE_MINUTES_MS = 5 * 60 * 1000;
|
|
19
|
+
const FIFTEEN_MINUTES_MS = 15 * 60 * 1000;
|
|
20
|
+
const SIXTY_MINUTES_MS = 60 * 60 * 1000;
|
|
21
|
+
function placeholders(values) {
|
|
22
|
+
if (values.length === 0) {
|
|
23
|
+
throw new Error('Cannot build SQL placeholders for an empty list.');
|
|
24
|
+
}
|
|
25
|
+
return values.map(() => '?').join(', ');
|
|
26
|
+
}
|
|
27
|
+
function positiveLimit(value, fallback, max = MAX_RANGE_LIMIT) {
|
|
28
|
+
if (value === undefined) {
|
|
29
|
+
return fallback;
|
|
30
|
+
}
|
|
31
|
+
return Math.min(max, Math.max(0, Math.floor(value)));
|
|
32
|
+
}
|
|
33
|
+
function nowIso() {
|
|
34
|
+
return new Date().toISOString();
|
|
35
|
+
}
|
|
36
|
+
function eventDateFromMs(ms) {
|
|
37
|
+
if (ms === null || !Number.isFinite(ms)) {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
return new Date(ms).toISOString().slice(0, 10);
|
|
41
|
+
}
|
|
42
|
+
function normalizeTimestampMs(input) {
|
|
43
|
+
const timestamp = input.source_timestamp_ms ?? input.event_datetime;
|
|
44
|
+
if (typeof timestamp !== 'number' || !Number.isFinite(timestamp)) {
|
|
45
|
+
throw new Error('connector_event_index.source_timestamp_ms must be a finite number.');
|
|
46
|
+
}
|
|
47
|
+
return Math.floor(timestamp);
|
|
48
|
+
}
|
|
49
|
+
function normalizeEventDatetime(input) {
|
|
50
|
+
if (input.event_datetime === null || input.event_datetime === undefined) {
|
|
51
|
+
return normalizeTimestampMs(input);
|
|
52
|
+
}
|
|
53
|
+
if (!Number.isFinite(input.event_datetime)) {
|
|
54
|
+
throw new Error('connector_event_index.event_datetime must be a finite number when provided.');
|
|
55
|
+
}
|
|
56
|
+
return Math.floor(input.event_datetime);
|
|
57
|
+
}
|
|
58
|
+
function normalizeMetadataJson(input) {
|
|
59
|
+
if (input.metadata_json !== undefined) {
|
|
60
|
+
return input.metadata_json;
|
|
61
|
+
}
|
|
62
|
+
if (input.metadata === undefined || input.metadata === null) {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
return (0, canonicalize_js_1.canonicalizeJSON)(input.metadata);
|
|
66
|
+
}
|
|
67
|
+
function toBuffer(value) {
|
|
68
|
+
if (Buffer.isBuffer(value)) {
|
|
69
|
+
return value;
|
|
70
|
+
}
|
|
71
|
+
if (value instanceof Uint8Array) {
|
|
72
|
+
return Buffer.from(value);
|
|
73
|
+
}
|
|
74
|
+
throw new Error('connector_event_index.content_hash must be a 32-byte Buffer.');
|
|
75
|
+
}
|
|
76
|
+
function mapConnectorEventIndexRow(row) {
|
|
77
|
+
return {
|
|
78
|
+
event_index_id: String(row.event_index_id),
|
|
79
|
+
source_connector: String(row.source_connector),
|
|
80
|
+
source_type: String(row.source_type),
|
|
81
|
+
source_id: String(row.source_id),
|
|
82
|
+
source_locator: row.source_locator === null ? null : String(row.source_locator),
|
|
83
|
+
channel: row.channel === null ? null : String(row.channel),
|
|
84
|
+
author: row.author === null ? null : String(row.author),
|
|
85
|
+
title: row.title === null ? null : String(row.title),
|
|
86
|
+
content: String(row.content),
|
|
87
|
+
event_datetime: typeof row.event_datetime === 'number' && Number.isFinite(row.event_datetime)
|
|
88
|
+
? row.event_datetime
|
|
89
|
+
: null,
|
|
90
|
+
event_date: row.event_date === null ? null : String(row.event_date),
|
|
91
|
+
source_timestamp_ms: Number(row.source_timestamp_ms),
|
|
92
|
+
metadata_json: row.metadata_json === null ? null : String(row.metadata_json),
|
|
93
|
+
artifact_locator: row.artifact_locator === null ? null : String(row.artifact_locator),
|
|
94
|
+
artifact_title: row.artifact_title === null ? null : String(row.artifact_title),
|
|
95
|
+
content_hash: toBuffer(row.content_hash),
|
|
96
|
+
indexed_at: String(row.indexed_at),
|
|
97
|
+
updated_at: String(row.updated_at),
|
|
98
|
+
expires_at: row.expires_at === null ? null : String(row.expires_at),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
function mapConnectorCursorRow(row) {
|
|
102
|
+
return {
|
|
103
|
+
connector_name: String(row.connector_name),
|
|
104
|
+
last_seen_timestamp_ms: Number(row.last_seen_timestamp_ms),
|
|
105
|
+
last_seen_source_id: String(row.last_seen_source_id),
|
|
106
|
+
last_sweep_at: row.last_sweep_at === null ? null : String(row.last_sweep_at),
|
|
107
|
+
last_success_at: row.last_success_at === null ? null : String(row.last_success_at),
|
|
108
|
+
last_error: row.last_error === null ? null : String(row.last_error),
|
|
109
|
+
last_error_at: row.last_error_at === null ? null : String(row.last_error_at),
|
|
110
|
+
indexed_count: Number(row.indexed_count),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
function connectorFilterSql(connectors, alias = '') {
|
|
114
|
+
if (!connectors || connectors.length === 0) {
|
|
115
|
+
return '';
|
|
116
|
+
}
|
|
117
|
+
const prefix = alias.length > 0 ? `${alias}.` : '';
|
|
118
|
+
return ` AND ${prefix}source_connector IN (${placeholders(connectors)})`;
|
|
119
|
+
}
|
|
120
|
+
function hasOwn(input, key) {
|
|
121
|
+
return Object.prototype.hasOwnProperty.call(input, key);
|
|
122
|
+
}
|
|
123
|
+
function connectorEventIndexId(sourceConnector, sourceId) {
|
|
124
|
+
const digest = (0, node_crypto_1.createHash)('sha1').update(`${sourceConnector}\0${sourceId}`, 'utf8').digest('hex');
|
|
125
|
+
return `evt_${digest.slice(0, 16)}`;
|
|
126
|
+
}
|
|
127
|
+
function connectorEventContentHash(input) {
|
|
128
|
+
return (0, node_crypto_1.createHash)('sha256')
|
|
129
|
+
.update((0, canonicalize_js_1.canonicalizeJSON)({
|
|
130
|
+
source_connector: input.source_connector,
|
|
131
|
+
source_id: input.source_id,
|
|
132
|
+
content: input.content,
|
|
133
|
+
event_datetime: input.event_datetime ?? null,
|
|
134
|
+
}), 'utf8')
|
|
135
|
+
.digest();
|
|
136
|
+
}
|
|
137
|
+
function upsertConnectorEventIndex(adapter, input) {
|
|
138
|
+
const sourceTimestampMs = normalizeTimestampMs(input);
|
|
139
|
+
const eventDatetime = normalizeEventDatetime(input);
|
|
140
|
+
const eventDate = input.event_date ?? eventDateFromMs(eventDatetime);
|
|
141
|
+
const timestamp = input.updated_at ?? nowIso();
|
|
142
|
+
const eventIndexId = connectorEventIndexId(input.source_connector, input.source_id);
|
|
143
|
+
const contentHash = input.content_hash
|
|
144
|
+
? Buffer.from(input.content_hash)
|
|
145
|
+
: connectorEventContentHash({
|
|
146
|
+
source_connector: input.source_connector,
|
|
147
|
+
source_id: input.source_id,
|
|
148
|
+
content: input.content,
|
|
149
|
+
event_datetime: eventDatetime,
|
|
150
|
+
});
|
|
151
|
+
if (contentHash.byteLength !== 32) {
|
|
152
|
+
throw new Error('connector_event_index.content_hash must be exactly 32 bytes.');
|
|
153
|
+
}
|
|
154
|
+
const metadataJson = normalizeMetadataJson(input);
|
|
155
|
+
return adapter.transaction(() => {
|
|
156
|
+
adapter
|
|
157
|
+
.prepare(`
|
|
158
|
+
INSERT INTO connector_event_index (
|
|
159
|
+
event_index_id, source_connector, source_type, source_id, source_locator,
|
|
160
|
+
channel, author, title, content, event_datetime, event_date, source_timestamp_ms,
|
|
161
|
+
metadata_json, artifact_locator, artifact_title, content_hash, indexed_at,
|
|
162
|
+
updated_at, expires_at
|
|
163
|
+
)
|
|
164
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
165
|
+
ON CONFLICT(source_connector, source_id) DO UPDATE SET
|
|
166
|
+
source_type = excluded.source_type,
|
|
167
|
+
source_locator = excluded.source_locator,
|
|
168
|
+
channel = excluded.channel,
|
|
169
|
+
author = excluded.author,
|
|
170
|
+
title = excluded.title,
|
|
171
|
+
content = excluded.content,
|
|
172
|
+
event_datetime = excluded.event_datetime,
|
|
173
|
+
event_date = excluded.event_date,
|
|
174
|
+
source_timestamp_ms = excluded.source_timestamp_ms,
|
|
175
|
+
metadata_json = excluded.metadata_json,
|
|
176
|
+
artifact_locator = excluded.artifact_locator,
|
|
177
|
+
artifact_title = excluded.artifact_title,
|
|
178
|
+
content_hash = excluded.content_hash,
|
|
179
|
+
updated_at = excluded.updated_at,
|
|
180
|
+
expires_at = excluded.expires_at
|
|
181
|
+
`)
|
|
182
|
+
.run(eventIndexId, input.source_connector, input.source_type, input.source_id, input.source_locator ?? null, input.channel ?? null, input.author ?? null, input.title ?? null, input.content, eventDatetime, eventDate, sourceTimestampMs, metadataJson, input.artifact_locator ?? null, input.artifact_title ?? null, contentHash, input.indexed_at ?? timestamp, timestamp, input.expires_at ?? null);
|
|
183
|
+
const saved = getConnectorEventIndexRecord(adapter, input.source_connector, input.source_id);
|
|
184
|
+
if (!saved) {
|
|
185
|
+
throw new Error(`Failed to read connector_event_index row after upsert: ${input.source_connector}/${input.source_id}`);
|
|
186
|
+
}
|
|
187
|
+
return saved;
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
function getConnectorEventIndexRecord(adapter, sourceConnector, sourceId) {
|
|
191
|
+
const row = adapter
|
|
192
|
+
.prepare(`
|
|
193
|
+
SELECT *
|
|
194
|
+
FROM connector_event_index
|
|
195
|
+
WHERE source_connector = ?
|
|
196
|
+
AND source_id = ?
|
|
197
|
+
LIMIT 1
|
|
198
|
+
`)
|
|
199
|
+
.get(sourceConnector, sourceId);
|
|
200
|
+
return row ? mapConnectorEventIndexRow(row) : null;
|
|
201
|
+
}
|
|
202
|
+
function listConnectorEventsByDatetimeRange(adapter, input) {
|
|
203
|
+
const limit = positiveLimit(input.limit, DEFAULT_RANGE_LIMIT);
|
|
204
|
+
if (limit === 0) {
|
|
205
|
+
return [];
|
|
206
|
+
}
|
|
207
|
+
const order = input.order === 'desc' ? 'DESC' : 'ASC';
|
|
208
|
+
const connectors = input.connectors?.filter(Boolean) ?? [];
|
|
209
|
+
const params = [input.fromMs, input.toMs, ...connectors, limit];
|
|
210
|
+
const rows = adapter
|
|
211
|
+
.prepare(`
|
|
212
|
+
SELECT *
|
|
213
|
+
FROM connector_event_index
|
|
214
|
+
WHERE event_datetime >= ?
|
|
215
|
+
AND event_datetime <= ?
|
|
216
|
+
${connectorFilterSql(connectors)}
|
|
217
|
+
ORDER BY event_datetime ${order}, event_index_id ASC
|
|
218
|
+
LIMIT ?
|
|
219
|
+
`)
|
|
220
|
+
.all(...params);
|
|
221
|
+
return rows.map(mapConnectorEventIndexRow);
|
|
222
|
+
}
|
|
223
|
+
function searchConnectorEventsByFTS(adapter, query, options = {}) {
|
|
224
|
+
const normalizedQuery = query.trim();
|
|
225
|
+
if (normalizedQuery.length === 0) {
|
|
226
|
+
return [];
|
|
227
|
+
}
|
|
228
|
+
const limit = positiveLimit(options.limit, DEFAULT_SEARCH_LIMIT, 100);
|
|
229
|
+
if (limit === 0) {
|
|
230
|
+
return [];
|
|
231
|
+
}
|
|
232
|
+
const connectors = options.connectors?.filter(Boolean) ?? [];
|
|
233
|
+
const params = [normalizedQuery, ...connectors, limit];
|
|
234
|
+
const rows = adapter
|
|
235
|
+
.prepare(`
|
|
236
|
+
SELECT e.*, bm25(connector_event_index_fts) AS rank
|
|
237
|
+
FROM connector_event_index_fts
|
|
238
|
+
JOIN connector_event_index e
|
|
239
|
+
ON e.event_index_id = connector_event_index_fts.event_index_id
|
|
240
|
+
WHERE connector_event_index_fts MATCH ?
|
|
241
|
+
${connectorFilterSql(connectors, 'e')}
|
|
242
|
+
ORDER BY rank ASC, e.event_datetime DESC, e.event_index_id ASC
|
|
243
|
+
LIMIT ?
|
|
244
|
+
`)
|
|
245
|
+
.all(...params);
|
|
246
|
+
return rows.map((row) => {
|
|
247
|
+
const rank = Number(row.rank);
|
|
248
|
+
return {
|
|
249
|
+
...mapConnectorEventIndexRow(row),
|
|
250
|
+
rank,
|
|
251
|
+
score: Number.isFinite(rank) ? 1 / (1 + Math.exp(rank)) : 0,
|
|
252
|
+
};
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
function readConnectorCursor(adapter, connectorName) {
|
|
256
|
+
const row = adapter
|
|
257
|
+
.prepare(`
|
|
258
|
+
SELECT *
|
|
259
|
+
FROM connector_event_index_cursors
|
|
260
|
+
WHERE connector_name = ?
|
|
261
|
+
LIMIT 1
|
|
262
|
+
`)
|
|
263
|
+
.get(connectorName);
|
|
264
|
+
return row ? mapConnectorCursorRow(row) : null;
|
|
265
|
+
}
|
|
266
|
+
function upsertConnectorCursor(adapter, input) {
|
|
267
|
+
const current = readConnectorCursor(adapter, input.connector_name);
|
|
268
|
+
const merged = {
|
|
269
|
+
connector_name: input.connector_name,
|
|
270
|
+
last_seen_timestamp_ms: input.last_seen_timestamp_ms ?? current?.last_seen_timestamp_ms ?? 0,
|
|
271
|
+
last_seen_source_id: input.last_seen_source_id ?? current?.last_seen_source_id ?? '',
|
|
272
|
+
last_sweep_at: hasOwn(input, 'last_sweep_at')
|
|
273
|
+
? (input.last_sweep_at ?? null)
|
|
274
|
+
: (current?.last_sweep_at ?? null),
|
|
275
|
+
last_success_at: hasOwn(input, 'last_success_at')
|
|
276
|
+
? (input.last_success_at ?? null)
|
|
277
|
+
: (current?.last_success_at ?? null),
|
|
278
|
+
last_error: hasOwn(input, 'last_error')
|
|
279
|
+
? (input.last_error ?? null)
|
|
280
|
+
: (current?.last_error ?? null),
|
|
281
|
+
last_error_at: hasOwn(input, 'last_error_at')
|
|
282
|
+
? (input.last_error_at ?? null)
|
|
283
|
+
: (current?.last_error_at ?? null),
|
|
284
|
+
indexed_count: input.indexed_count ?? current?.indexed_count ?? 0,
|
|
285
|
+
};
|
|
286
|
+
adapter
|
|
287
|
+
.prepare(`
|
|
288
|
+
INSERT INTO connector_event_index_cursors (
|
|
289
|
+
connector_name, last_seen_timestamp_ms, last_seen_source_id, last_sweep_at,
|
|
290
|
+
last_success_at, last_error, last_error_at, indexed_count
|
|
291
|
+
)
|
|
292
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
293
|
+
ON CONFLICT(connector_name) DO UPDATE SET
|
|
294
|
+
last_seen_timestamp_ms = excluded.last_seen_timestamp_ms,
|
|
295
|
+
last_seen_source_id = excluded.last_seen_source_id,
|
|
296
|
+
last_sweep_at = excluded.last_sweep_at,
|
|
297
|
+
last_success_at = excluded.last_success_at,
|
|
298
|
+
last_error = excluded.last_error,
|
|
299
|
+
last_error_at = excluded.last_error_at,
|
|
300
|
+
indexed_count = excluded.indexed_count
|
|
301
|
+
`)
|
|
302
|
+
.run(merged.connector_name, merged.last_seen_timestamp_ms, merged.last_seen_source_id, merged.last_sweep_at, merged.last_success_at, merged.last_error, merged.last_error_at, merged.indexed_count);
|
|
303
|
+
const saved = readConnectorCursor(adapter, input.connector_name);
|
|
304
|
+
if (!saved) {
|
|
305
|
+
throw new Error(`Failed to read connector cursor after upsert: ${input.connector_name}`);
|
|
306
|
+
}
|
|
307
|
+
return saved;
|
|
308
|
+
}
|
|
309
|
+
function deleteExpiredConnectorEvents(adapter, input) {
|
|
310
|
+
if (!Number.isFinite(input.nowMs)) {
|
|
311
|
+
throw new Error('deleteExpiredConnectorEvents.nowMs must be a finite number.');
|
|
312
|
+
}
|
|
313
|
+
if (!Number.isFinite(input.retentionMs) || input.retentionMs < 0) {
|
|
314
|
+
throw new Error('deleteExpiredConnectorEvents.retentionMs must be a non-negative finite number.');
|
|
315
|
+
}
|
|
316
|
+
const cutoffMs = input.nowMs - input.retentionMs;
|
|
317
|
+
const result = input.connectorName
|
|
318
|
+
? adapter
|
|
319
|
+
.prepare(`
|
|
320
|
+
DELETE FROM connector_event_index
|
|
321
|
+
WHERE source_connector = ?
|
|
322
|
+
AND event_datetime IS NOT NULL
|
|
323
|
+
AND event_datetime < ?
|
|
324
|
+
AND artifact_locator IS NULL
|
|
325
|
+
`)
|
|
326
|
+
.run(input.connectorName, cutoffMs)
|
|
327
|
+
: adapter
|
|
328
|
+
.prepare(`
|
|
329
|
+
DELETE FROM connector_event_index
|
|
330
|
+
WHERE event_datetime IS NOT NULL
|
|
331
|
+
AND event_datetime < ?
|
|
332
|
+
AND artifact_locator IS NULL
|
|
333
|
+
`)
|
|
334
|
+
.run(cutoffMs);
|
|
335
|
+
return { rows_deleted: result.changes };
|
|
336
|
+
}
|
|
337
|
+
function computeStalenessStatus(cursor, options) {
|
|
338
|
+
if (!cursor) {
|
|
339
|
+
return 'never_swept';
|
|
340
|
+
}
|
|
341
|
+
if (cursor.last_error !== null) {
|
|
342
|
+
return 'unhealthy';
|
|
343
|
+
}
|
|
344
|
+
if (cursor.last_success_at === null) {
|
|
345
|
+
return 'never_swept';
|
|
346
|
+
}
|
|
347
|
+
const lastSuccessMs = Date.parse(cursor.last_success_at);
|
|
348
|
+
if (!Number.isFinite(lastSuccessMs)) {
|
|
349
|
+
return 'unhealthy';
|
|
350
|
+
}
|
|
351
|
+
const elapsedMs = options.nowMs - lastSuccessMs;
|
|
352
|
+
const staleAfterMs = options.staleAfterMs ?? FIVE_MINUTES_MS;
|
|
353
|
+
const warnAfterMs = options.stalenessWarnAfterMs ?? FIFTEEN_MINUTES_MS;
|
|
354
|
+
const unhealthyAfterMs = options.unhealthyAfterMs ?? SIXTY_MINUTES_MS;
|
|
355
|
+
if (elapsedMs > unhealthyAfterMs) {
|
|
356
|
+
return 'unhealthy';
|
|
357
|
+
}
|
|
358
|
+
if (elapsedMs > warnAfterMs) {
|
|
359
|
+
return 'warn';
|
|
360
|
+
}
|
|
361
|
+
if (elapsedMs > staleAfterMs) {
|
|
362
|
+
return 'stale-but-warming';
|
|
363
|
+
}
|
|
364
|
+
return 'healthy';
|
|
365
|
+
}
|
|
366
|
+
//# sourceMappingURL=event-index.js.map
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import type { Buffer } from 'node:buffer';
|
|
2
|
+
export interface ConnectorEventIndexRecord {
|
|
3
|
+
event_index_id: string;
|
|
4
|
+
source_connector: string;
|
|
5
|
+
source_type: string;
|
|
6
|
+
source_id: string;
|
|
7
|
+
source_locator: string | null;
|
|
8
|
+
channel: string | null;
|
|
9
|
+
author: string | null;
|
|
10
|
+
title: string | null;
|
|
11
|
+
content: string;
|
|
12
|
+
event_datetime: number | null;
|
|
13
|
+
event_date: string | null;
|
|
14
|
+
source_timestamp_ms: number;
|
|
15
|
+
metadata_json: string | null;
|
|
16
|
+
artifact_locator: string | null;
|
|
17
|
+
artifact_title: string | null;
|
|
18
|
+
content_hash: Buffer;
|
|
19
|
+
indexed_at: string;
|
|
20
|
+
updated_at: string;
|
|
21
|
+
expires_at: string | null;
|
|
22
|
+
}
|
|
23
|
+
export interface UpsertConnectorEventIndexInput {
|
|
24
|
+
source_connector: string;
|
|
25
|
+
source_type: string;
|
|
26
|
+
source_id: string;
|
|
27
|
+
source_locator?: string | null;
|
|
28
|
+
channel?: string | null;
|
|
29
|
+
author?: string | null;
|
|
30
|
+
title?: string | null;
|
|
31
|
+
content: string;
|
|
32
|
+
event_datetime?: number | null;
|
|
33
|
+
event_date?: string | null;
|
|
34
|
+
source_timestamp_ms?: number | null;
|
|
35
|
+
metadata_json?: string | null;
|
|
36
|
+
metadata?: unknown;
|
|
37
|
+
artifact_locator?: string | null;
|
|
38
|
+
artifact_title?: string | null;
|
|
39
|
+
content_hash?: Buffer | Uint8Array | null;
|
|
40
|
+
indexed_at?: string;
|
|
41
|
+
updated_at?: string;
|
|
42
|
+
expires_at?: string | null;
|
|
43
|
+
}
|
|
44
|
+
export interface ConnectorEventIndexCursorRecord {
|
|
45
|
+
connector_name: string;
|
|
46
|
+
last_seen_timestamp_ms: number;
|
|
47
|
+
last_seen_source_id: string;
|
|
48
|
+
last_sweep_at: string | null;
|
|
49
|
+
last_success_at: string | null;
|
|
50
|
+
last_error: string | null;
|
|
51
|
+
last_error_at: string | null;
|
|
52
|
+
indexed_count: number;
|
|
53
|
+
}
|
|
54
|
+
export interface UpsertConnectorEventIndexCursorInput {
|
|
55
|
+
connector_name: string;
|
|
56
|
+
last_seen_timestamp_ms?: number;
|
|
57
|
+
last_seen_source_id?: string;
|
|
58
|
+
last_sweep_at?: string | null;
|
|
59
|
+
last_success_at?: string | null;
|
|
60
|
+
last_error?: string | null;
|
|
61
|
+
last_error_at?: string | null;
|
|
62
|
+
indexed_count?: number;
|
|
63
|
+
}
|
|
64
|
+
export interface ConnectorEventSearchHit extends ConnectorEventIndexRecord {
|
|
65
|
+
rank: number;
|
|
66
|
+
score: number;
|
|
67
|
+
}
|
|
68
|
+
export type ConnectorEventStalenessStatus = 'healthy' | 'stale-but-warming' | 'warn' | 'unhealthy' | 'never_swept';
|
|
69
|
+
//# sourceMappingURL=types.d.ts.map
|
package/dist/db-manager.d.ts
CHANGED
|
@@ -57,6 +57,8 @@ export interface DecisionRecord {
|
|
|
57
57
|
edges?: DecisionEdgeRow[];
|
|
58
58
|
/** ISO 8601 date when the event actually occurred. Null if not set. */
|
|
59
59
|
event_date?: string | null;
|
|
60
|
+
/** Source event timestamp in milliseconds when known. Null if not set. */
|
|
61
|
+
event_datetime?: number | null;
|
|
60
62
|
}
|
|
61
63
|
export interface OutcomeData {
|
|
62
64
|
outcome?: string | null;
|
|
@@ -202,6 +204,8 @@ export interface DecisionInput {
|
|
|
202
204
|
risks?: string | null;
|
|
203
205
|
/** ISO 8601 date string for when the event actually occurred (e.g. "2023-01-15") */
|
|
204
206
|
event_date?: string | null;
|
|
207
|
+
/** Source event timestamp in milliseconds when known. */
|
|
208
|
+
event_datetime?: number | null;
|
|
205
209
|
}
|
|
206
210
|
/**
|
|
207
211
|
* Insert decision with embedding
|
package/dist/db-manager.js
CHANGED
|
@@ -267,6 +267,13 @@ async function insertDecisionWithEmbedding(decision) {
|
|
|
267
267
|
throw new Error(`Invalid event_date: must be ISO 8601 YYYY-MM-DD (got: ${d})`);
|
|
268
268
|
}
|
|
269
269
|
}
|
|
270
|
+
if (decision.event_datetime !== null && decision.event_datetime !== undefined) {
|
|
271
|
+
if (typeof decision.event_datetime !== 'number' ||
|
|
272
|
+
!Number.isFinite(decision.event_datetime) ||
|
|
273
|
+
decision.event_datetime <= 0) {
|
|
274
|
+
throw new Error(`Invalid event_datetime: must be a positive millisecond timestamp (got: ${decision.event_datetime})`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
270
277
|
try {
|
|
271
278
|
// Generate embedding BEFORE transaction (required for SQLite's sync transaction)
|
|
272
279
|
// Note: Redact topic for privacy - only log length
|
|
@@ -299,14 +306,14 @@ async function insertDecisionWithEmbedding(decision) {
|
|
|
299
306
|
confidence, created_at, updated_at,
|
|
300
307
|
needs_validation, validation_attempts, last_validated_at, usage_count,
|
|
301
308
|
trust_context, usage_success, usage_failure, time_saved,
|
|
302
|
-
evidence, alternatives, risks, event_date
|
|
303
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
309
|
+
evidence, alternatives, risks, event_date, event_datetime
|
|
310
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
304
311
|
`);
|
|
305
312
|
const insertResult = stmt.run(decision.id, decision.topic, decision.decision, decision.reasoning || null, decision.outcome || null, decision.failure_reason || null, decision.limitation || null, decision.user_involvement || null, decision.session_id || null, decision.supersedes || null, decision.superseded_by || null, decision.refined_from ? JSON.stringify(decision.refined_from) : null, decision.confidence !== undefined ? decision.confidence : 0.5,
|
|
306
313
|
// IMPORTANT: All timestamps are stored in milliseconds (Date.now()).
|
|
307
314
|
// The schema DEFAULT uses unixepoch() (seconds) but is never used
|
|
308
315
|
// since all inserts go through this function which always provides ms.
|
|
309
|
-
decision.created_at || Date.now(), decision.updated_at || Date.now(), decision.needs_validation !== undefined ? decision.needs_validation : 0, decision.validation_attempts || 0, decision.last_validated_at || null, decision.usage_count || 0, decision.trust_context || null, decision.usage_success || 0, decision.usage_failure || 0, decision.time_saved || 0, decision.evidence || null, decision.alternatives || null, decision.risks || null, decision.event_date || null);
|
|
316
|
+
decision.created_at || Date.now(), decision.updated_at || Date.now(), decision.needs_validation !== undefined ? decision.needs_validation : 0, decision.validation_attempts || 0, decision.last_validated_at || null, decision.usage_count || 0, decision.trust_context || null, decision.usage_success || 0, decision.usage_failure || 0, decision.time_saved || 0, decision.evidence || null, decision.alternatives || null, decision.risks || null, decision.event_date || null, decision.event_datetime ?? null);
|
|
310
317
|
const rowid = Number(insertResult.lastInsertRowid);
|
|
311
318
|
// Insert embedding in same transaction to ensure rowid matching
|
|
312
319
|
(0, debug_logger_js_1.info)(`[db-manager] Vector search enabled: ${adapter.vectorSearchEnabled}`);
|
|
@@ -38,6 +38,8 @@ export interface DecisionForFormat {
|
|
|
38
38
|
risks?: string;
|
|
39
39
|
/** ISO 8601 date when the event actually occurred. Null if not set. */
|
|
40
40
|
event_date?: string | null;
|
|
41
|
+
/** Source event timestamp in milliseconds when known. Null if not set. */
|
|
42
|
+
event_datetime?: number | null;
|
|
41
43
|
}
|
|
42
44
|
/**
|
|
43
45
|
* Trust context object
|
|
@@ -140,7 +140,7 @@ Top ${full.length} Most Relevant Decisions:
|
|
|
140
140
|
`.trim();
|
|
141
141
|
for (let i = 0; i < full.length; i++) {
|
|
142
142
|
const d = full[i];
|
|
143
|
-
const duration = calculateDuration(d
|
|
143
|
+
const duration = calculateDuration(getDecisionTimestamp(d));
|
|
144
144
|
const outcomeEmoji = getOutcomeEmoji(d.outcome ?? null);
|
|
145
145
|
const relevancePercent = Math.round((d.relevanceScore || 0) * 100);
|
|
146
146
|
context += `\n\n${i + 1}. ${d.decision} (${duration}, relevance: ${relevancePercent}%) ${outcomeEmoji}`;
|
|
@@ -166,7 +166,7 @@ Top ${full.length} Most Relevant Decisions:
|
|
|
166
166
|
* Format small decision history (3 or fewer)
|
|
167
167
|
*/
|
|
168
168
|
function formatSmallHistory(current, history) {
|
|
169
|
-
const duration = calculateDuration(current
|
|
169
|
+
const duration = calculateDuration(getDecisionTimestamp(current));
|
|
170
170
|
let context = `
|
|
171
171
|
🧠 DECISION HISTORY: ${current.topic}
|
|
172
172
|
|
|
@@ -284,6 +284,9 @@ function calculateDuration(timestamp) {
|
|
|
284
284
|
}
|
|
285
285
|
return `${diffDays} day${diffDays !== 1 ? 's' : ''} ago`;
|
|
286
286
|
}
|
|
287
|
+
function getDecisionTimestamp(decision) {
|
|
288
|
+
return decision.event_datetime ?? decision.created_at;
|
|
289
|
+
}
|
|
287
290
|
/**
|
|
288
291
|
* Calculate duration between two timestamps in days
|
|
289
292
|
*/
|
|
@@ -560,7 +563,7 @@ function formatTeaserList(decisions, topN = 3) {
|
|
|
560
563
|
output += `\n "${preview}"`;
|
|
561
564
|
// Recency metadata (NEW - Gaussian Decay)
|
|
562
565
|
if (d.recency_age_days !== undefined && d.created_at) {
|
|
563
|
-
const timeAgo = calculateDuration(d
|
|
566
|
+
const timeAgo = calculateDuration(getDecisionTimestamp(d));
|
|
564
567
|
const recencyScore = d.recency_score ? Math.round(d.recency_score * 100) : null;
|
|
565
568
|
const finalScore = d.final_score ? Math.round(d.final_score * 100) : null;
|
|
566
569
|
output += `\n ⏰ ${timeAgo}`;
|
|
@@ -582,7 +585,7 @@ function formatTeaser(decision) {
|
|
|
582
585
|
if (!decision) {
|
|
583
586
|
return null;
|
|
584
587
|
}
|
|
585
|
-
const timeAgo = calculateDuration(decision
|
|
588
|
+
const timeAgo = calculateDuration(getDecisionTimestamp(decision));
|
|
586
589
|
// Extract preview (first 60 chars)
|
|
587
590
|
const preview = decision.decision.length > 60 ? decision.decision.substring(0, 60) + '...' : decision.decision;
|
|
588
591
|
// Extract files from trust_context or show generic
|
|
@@ -633,7 +636,7 @@ function formatRecall(decisions, semanticEdges = null) {
|
|
|
633
636
|
* Format single decision with full detail
|
|
634
637
|
*/
|
|
635
638
|
function formatSingleDecision(decision) {
|
|
636
|
-
const timeAgo = calculateDuration(decision
|
|
639
|
+
const timeAgo = calculateDuration(getDecisionTimestamp(decision));
|
|
637
640
|
const confidencePercent = Math.round((decision.confidence || 0) * 100);
|
|
638
641
|
const outcomeEmoji = getOutcomeEmoji(decision.outcome ?? null);
|
|
639
642
|
const outcomeText = decision.outcome || 'Not yet tracked';
|
|
@@ -725,7 +728,7 @@ function formatDecisionHistory(decisions, semanticEdges = null) {
|
|
|
725
728
|
📋 Decision History: ${topic}
|
|
726
729
|
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
727
730
|
|
|
728
|
-
Latest Decision (${calculateDuration(latest
|
|
731
|
+
Latest Decision (${calculateDuration(getDecisionTimestamp(latest))}):
|
|
729
732
|
${latest.decision}
|
|
730
733
|
`.trim();
|
|
731
734
|
// Show brief reasoning if available
|
|
@@ -777,7 +780,7 @@ ${latest.decision}
|
|
|
777
780
|
output += `\nPrevious Decisions (${older.length}):\n`;
|
|
778
781
|
for (let i = 0; i < Math.min(older.length, 5); i++) {
|
|
779
782
|
const d = older[i];
|
|
780
|
-
const timeAgo = calculateDuration(d
|
|
783
|
+
const timeAgo = calculateDuration(getDecisionTimestamp(d));
|
|
781
784
|
const emoji = getOutcomeEmoji(d.outcome ?? null);
|
|
782
785
|
output += `\n${i + 2}. ${d.decision} (${timeAgo}) ${emoji}`;
|
|
783
786
|
if (d.outcome === 'FAILED' && d.failure_reason) {
|
|
@@ -876,7 +879,7 @@ function formatList(decisions, options = {}) {
|
|
|
876
879
|
output += '━'.repeat(60) + '\n';
|
|
877
880
|
for (let i = 0; i < items.length; i++) {
|
|
878
881
|
const d = items[i];
|
|
879
|
-
const timeAgo = calculateDuration(d
|
|
882
|
+
const timeAgo = calculateDuration(getDecisionTimestamp(d));
|
|
880
883
|
const type = d.user_involvement === 'approved' ? '👤 User' : '🤖 Assistant';
|
|
881
884
|
const status = d.outcome ? getOutcomeEmoji(d.outcome) + ' ' + d.outcome : '⏳ Pending';
|
|
882
885
|
const confidence = Math.round((d.confidence || 0) * 100);
|