@jungjaehoon/mama-core 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/026-create-canonical-entity-tables.sql +99 -0
- package/db/migrations/027-create-entity-review-support.sql +44 -0
- package/db/migrations/028-create-entity-audit-tables.sql +45 -0
- package/db/migrations/029-extend-entity-observations-observation-type.sql +60 -0
- package/db/migrations/030-case-first-memory-substrate.sql +901 -0
- package/db/migrations/031-wiki-page-index-provenance-columns.sql +2 -0
- package/dist/canonicalize.d.ts +38 -0
- package/dist/canonicalize.js +106 -0
- package/dist/cases/case-links.d.ts +116 -0
- package/dist/cases/case-links.js +794 -0
- package/dist/cases/composition-overrides.d.ts +90 -0
- package/dist/cases/composition-overrides.js +724 -0
- package/dist/cases/corrections.d.ts +156 -0
- package/dist/cases/corrections.js +1045 -0
- package/dist/cases/freshness.d.ts +57 -0
- package/dist/cases/freshness.js +283 -0
- package/dist/cases/live-state.d.ts +48 -0
- package/dist/cases/live-state.js +256 -0
- package/dist/cases/membership-explain.d.ts +56 -0
- package/dist/cases/membership-explain.js +140 -0
- package/dist/cases/membership-matcher.d.ts +23 -0
- package/dist/cases/membership-matcher.js +193 -0
- package/dist/cases/merge-split.d.ts +61 -0
- package/dist/cases/merge-split.js +368 -0
- package/dist/cases/role-inference.d.ts +10 -0
- package/dist/cases/role-inference.js +85 -0
- package/dist/cases/search-rollup.d.ts +23 -0
- package/dist/cases/search-rollup.js +225 -0
- package/dist/cases/sqlite-transaction.d.ts +10 -0
- package/dist/cases/sqlite-transaction.js +62 -0
- package/dist/cases/store.d.ts +74 -0
- package/dist/cases/store.js +816 -0
- package/dist/cases/target-ref.d.ts +25 -0
- package/dist/cases/target-ref.js +22 -0
- package/dist/cases/timeline-range.d.ts +6 -0
- package/dist/cases/timeline-range.js +322 -0
- package/dist/cases/tombstone-sweeper.d.ts +13 -0
- package/dist/cases/tombstone-sweeper.js +128 -0
- package/dist/cases/types.d.ts +244 -0
- package/dist/cases/types.js +47 -0
- package/dist/cases/wiki-page-index.d.ts +50 -0
- package/dist/cases/wiki-page-index.js +350 -0
- package/dist/config-loader.d.ts +1 -0
- package/dist/config-loader.js +43 -0
- package/dist/connectors/event-index.d.ts +44 -0
- package/dist/connectors/event-index.js +366 -0
- package/dist/connectors/types.d.ts +69 -0
- package/dist/connectors/types.js +3 -0
- package/dist/db-manager.d.ts +4 -0
- package/dist/db-manager.js +10 -3
- package/dist/decision-formatter.d.ts +2 -0
- package/dist/decision-formatter.js +11 -8
- package/dist/embedding-server/mobile/daemon.d.ts +1 -0
- package/dist/embedding-server/mobile/daemon.js +11 -1
- package/dist/entities/audit-metrics.d.ts +73 -0
- package/dist/entities/audit-metrics.js +128 -0
- package/dist/entities/candidate-generator.d.ts +9 -0
- package/dist/entities/candidate-generator.js +436 -0
- package/dist/entities/entity-impact.d.ts +41 -0
- package/dist/entities/entity-impact.js +140 -0
- package/dist/entities/entity-linked-decision-counts.d.ts +2 -0
- package/dist/entities/entity-linked-decision-counts.js +29 -0
- package/dist/entities/entity-list.d.ts +23 -0
- package/dist/entities/entity-list.js +99 -0
- package/dist/entities/entity-orphan-list.d.ts +26 -0
- package/dist/entities/entity-orphan-list.js +91 -0
- package/dist/entities/entity-search.d.ts +21 -0
- package/dist/entities/entity-search.js +150 -0
- package/dist/entities/errors.d.ts +65 -0
- package/dist/entities/errors.js +124 -0
- package/dist/entities/exact-merge-backfill.d.ts +11 -0
- package/dist/entities/exact-merge-backfill.js +161 -0
- package/dist/entities/lineage-backfill.d.ts +12 -0
- package/dist/entities/lineage-backfill.js +101 -0
- package/dist/entities/lineage-store.d.ts +51 -0
- package/dist/entities/lineage-store.js +308 -0
- package/dist/entities/normalization.d.ts +14 -0
- package/dist/entities/normalization.js +63 -0
- package/dist/entities/policy-store.d.ts +17 -0
- package/dist/entities/policy-store.js +263 -0
- package/dist/entities/policy-types.d.ts +72 -0
- package/dist/entities/policy-types.js +59 -0
- package/dist/entities/projection.d.ts +8 -0
- package/dist/entities/projection.js +82 -0
- package/dist/entities/provenance-query.d.ts +33 -0
- package/dist/entities/provenance-query.js +167 -0
- package/dist/entities/read-identity.d.ts +31 -0
- package/dist/entities/read-identity.js +264 -0
- package/dist/entities/recall-bridge.d.ts +5 -0
- package/dist/entities/recall-bridge.js +171 -0
- package/dist/entities/resolution-engine.d.ts +15 -0
- package/dist/entities/resolution-engine.js +69 -0
- package/dist/entities/rollback-preview.d.ts +39 -0
- package/dist/entities/rollback-preview.js +235 -0
- package/dist/entities/source-locator.d.ts +3 -0
- package/dist/entities/source-locator.js +32 -0
- package/dist/entities/store.d.ts +75 -0
- package/dist/entities/store.js +465 -0
- package/dist/entities/types.d.ts +162 -0
- package/dist/entities/types.js +30 -0
- package/dist/index.d.ts +47 -2
- package/dist/index.js +66 -25
- package/dist/mama-api.d.ts +22 -1
- package/dist/mama-api.js +272 -64
- package/dist/memory/api.d.ts +28 -1
- package/dist/memory/api.js +293 -31
- package/dist/memory/types.d.ts +11 -2
- package/dist/search/feedback-store.d.ts +59 -0
- package/dist/search/feedback-store.js +211 -0
- package/dist/search/question-type.d.ts +5 -0
- package/dist/search/question-type.js +41 -0
- package/dist/search/ranker-features.d.ts +23 -0
- package/dist/search/ranker-features.js +104 -0
- package/dist/search/ranker-rescore.d.ts +31 -0
- package/dist/search/ranker-rescore.js +226 -0
- package/dist/search/ranker-trainer.d.ts +94 -0
- package/dist/search/ranker-trainer.js +860 -0
- package/dist/test-utils.d.ts +14 -0
- package/dist/test-utils.js +52 -0
- package/package.json +21 -3
package/dist/config-loader.js
CHANGED
|
@@ -29,13 +29,48 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
29
29
|
const path_1 = __importDefault(require("path"));
|
|
30
30
|
const os_1 = __importDefault(require("os"));
|
|
31
31
|
const debug_logger_js_1 = require("./debug-logger.js");
|
|
32
|
+
// Current config version — bump this when adding migrations
|
|
33
|
+
const CURRENT_CONFIG_VERSION = 2;
|
|
32
34
|
// Default configuration
|
|
33
35
|
exports.DEFAULT_CONFIG = {
|
|
36
|
+
configVersion: CURRENT_CONFIG_VERSION,
|
|
34
37
|
modelName: 'Xenova/multilingual-e5-large',
|
|
35
38
|
embeddingDim: 1024,
|
|
36
39
|
quantized: true,
|
|
37
40
|
cacheDir: path_1.default.join(os_1.default.homedir(), '.cache', 'huggingface', 'transformers'),
|
|
38
41
|
};
|
|
42
|
+
/**
|
|
43
|
+
* Config migrations — each entry upgrades from (version - 1) to version.
|
|
44
|
+
* Migrations run sequentially and mutate the config object in place.
|
|
45
|
+
*/
|
|
46
|
+
const CONFIG_MIGRATIONS = {
|
|
47
|
+
// v1 → v2: e5-small (384-dim) → e5-large (1024-dim)
|
|
48
|
+
2: (cfg) => {
|
|
49
|
+
if (cfg.modelName === 'Xenova/multilingual-e5-small' && cfg.embeddingDim === 384) {
|
|
50
|
+
(0, debug_logger_js_1.info)('[config] Migration v2: upgrading embedding model e5-small → e5-large');
|
|
51
|
+
cfg.modelName = 'Xenova/multilingual-e5-large';
|
|
52
|
+
cfg.embeddingDim = 1024;
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Apply pending migrations to a config object.
|
|
58
|
+
* Returns true if any migrations were applied.
|
|
59
|
+
*/
|
|
60
|
+
function applyMigrations(config) {
|
|
61
|
+
const startVersion = config.configVersion || 1;
|
|
62
|
+
if (startVersion >= CURRENT_CONFIG_VERSION)
|
|
63
|
+
return false;
|
|
64
|
+
for (let v = startVersion + 1; v <= CURRENT_CONFIG_VERSION; v++) {
|
|
65
|
+
const migrate = CONFIG_MIGRATIONS[v];
|
|
66
|
+
if (migrate) {
|
|
67
|
+
migrate(config);
|
|
68
|
+
(0, debug_logger_js_1.info)(`[config] Applied migration v${v}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
config.configVersion = CURRENT_CONFIG_VERSION;
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
39
74
|
// Config file path
|
|
40
75
|
const CONFIG_DIR = path_1.default.join(os_1.default.homedir(), '.mama');
|
|
41
76
|
const CONFIG_PATH = path_1.default.join(CONFIG_DIR, 'config.json');
|
|
@@ -80,11 +115,19 @@ function loadConfig(reload = false) {
|
|
|
80
115
|
// Read and parse config file
|
|
81
116
|
const configData = fs_1.default.readFileSync(CONFIG_PATH, 'utf8');
|
|
82
117
|
const userConfig = JSON.parse(configData);
|
|
118
|
+
// Determine file's config version before merging with defaults
|
|
119
|
+
const fileVersion = userConfig.configVersion || 1;
|
|
83
120
|
// Merge with defaults (user config overrides)
|
|
84
121
|
const config = {
|
|
85
122
|
...exports.DEFAULT_CONFIG,
|
|
86
123
|
...userConfig,
|
|
124
|
+
configVersion: fileVersion, // preserve file version for migration check
|
|
87
125
|
};
|
|
126
|
+
// Apply pending migrations based on file's version
|
|
127
|
+
if (applyMigrations(config)) {
|
|
128
|
+
fs_1.default.writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2), 'utf8');
|
|
129
|
+
(0, debug_logger_js_1.info)(`[config] Migrated config saved to ${CONFIG_PATH}`);
|
|
130
|
+
}
|
|
88
131
|
// Validate configuration
|
|
89
132
|
if (!config.modelName || typeof config.modelName !== 'string') {
|
|
90
133
|
(0, debug_logger_js_1.warn)('[config] Invalid modelName, using default:', exports.DEFAULT_CONFIG.modelName);
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { DatabaseAdapter } from '../db-manager.js';
|
|
2
|
+
import type { ConnectorEventIndexCursorRecord, ConnectorEventIndexRecord, ConnectorEventSearchHit, ConnectorEventStalenessStatus, UpsertConnectorEventIndexCursorInput, UpsertConnectorEventIndexInput } from './types.js';
|
|
3
|
+
type ConnectorEventIndexAdapter = Pick<DatabaseAdapter, 'prepare' | 'transaction'>;
|
|
4
|
+
interface ListConnectorEventsByDatetimeRangeInput {
|
|
5
|
+
fromMs: number;
|
|
6
|
+
toMs: number;
|
|
7
|
+
limit?: number;
|
|
8
|
+
connectors?: string[];
|
|
9
|
+
order?: 'asc' | 'desc';
|
|
10
|
+
}
|
|
11
|
+
interface SearchConnectorEventsOptions {
|
|
12
|
+
limit?: number;
|
|
13
|
+
connectors?: string[];
|
|
14
|
+
}
|
|
15
|
+
interface DeleteExpiredConnectorEventsInput {
|
|
16
|
+
nowMs: number;
|
|
17
|
+
retentionMs: number;
|
|
18
|
+
connectorName?: string;
|
|
19
|
+
}
|
|
20
|
+
interface StalenessOptions {
|
|
21
|
+
nowMs: number;
|
|
22
|
+
staleAfterMs?: number;
|
|
23
|
+
stalenessWarnAfterMs?: number;
|
|
24
|
+
unhealthyAfterMs?: number;
|
|
25
|
+
}
|
|
26
|
+
export declare function connectorEventIndexId(sourceConnector: string, sourceId: string): string;
|
|
27
|
+
export declare function connectorEventContentHash(input: {
|
|
28
|
+
source_connector: string;
|
|
29
|
+
source_id: string;
|
|
30
|
+
content: string;
|
|
31
|
+
event_datetime?: number | null;
|
|
32
|
+
}): Buffer;
|
|
33
|
+
export declare function upsertConnectorEventIndex(adapter: ConnectorEventIndexAdapter, input: UpsertConnectorEventIndexInput): ConnectorEventIndexRecord;
|
|
34
|
+
export declare function getConnectorEventIndexRecord(adapter: ConnectorEventIndexAdapter, sourceConnector: string, sourceId: string): ConnectorEventIndexRecord | null;
|
|
35
|
+
export declare function listConnectorEventsByDatetimeRange(adapter: ConnectorEventIndexAdapter, input: ListConnectorEventsByDatetimeRangeInput): ConnectorEventIndexRecord[];
|
|
36
|
+
export declare function searchConnectorEventsByFTS(adapter: ConnectorEventIndexAdapter, query: string, options?: SearchConnectorEventsOptions): ConnectorEventSearchHit[];
|
|
37
|
+
export declare function readConnectorCursor(adapter: ConnectorEventIndexAdapter, connectorName: string): ConnectorEventIndexCursorRecord | null;
|
|
38
|
+
export declare function upsertConnectorCursor(adapter: ConnectorEventIndexAdapter, input: UpsertConnectorEventIndexCursorInput): ConnectorEventIndexCursorRecord;
|
|
39
|
+
export declare function deleteExpiredConnectorEvents(adapter: ConnectorEventIndexAdapter, input: DeleteExpiredConnectorEventsInput): {
|
|
40
|
+
rows_deleted: number;
|
|
41
|
+
};
|
|
42
|
+
export declare function computeStalenessStatus(cursor: ConnectorEventIndexCursorRecord | null, options: StalenessOptions): ConnectorEventStalenessStatus;
|
|
43
|
+
export type { ConnectorEventIndexCursorRecord, ConnectorEventIndexRecord, ConnectorEventSearchHit, ConnectorEventStalenessStatus, UpsertConnectorEventIndexCursorInput, UpsertConnectorEventIndexInput, } from './types.js';
|
|
44
|
+
//# sourceMappingURL=event-index.d.ts.map
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.connectorEventIndexId = connectorEventIndexId;
|
|
4
|
+
exports.connectorEventContentHash = connectorEventContentHash;
|
|
5
|
+
exports.upsertConnectorEventIndex = upsertConnectorEventIndex;
|
|
6
|
+
exports.getConnectorEventIndexRecord = getConnectorEventIndexRecord;
|
|
7
|
+
exports.listConnectorEventsByDatetimeRange = listConnectorEventsByDatetimeRange;
|
|
8
|
+
exports.searchConnectorEventsByFTS = searchConnectorEventsByFTS;
|
|
9
|
+
exports.readConnectorCursor = readConnectorCursor;
|
|
10
|
+
exports.upsertConnectorCursor = upsertConnectorCursor;
|
|
11
|
+
exports.deleteExpiredConnectorEvents = deleteExpiredConnectorEvents;
|
|
12
|
+
exports.computeStalenessStatus = computeStalenessStatus;
|
|
13
|
+
const node_crypto_1 = require("node:crypto");
|
|
14
|
+
const canonicalize_js_1 = require("../canonicalize.js");
|
|
15
|
+
const DEFAULT_RANGE_LIMIT = 100;
|
|
16
|
+
const DEFAULT_SEARCH_LIMIT = 25;
|
|
17
|
+
const MAX_RANGE_LIMIT = 500;
|
|
18
|
+
const FIVE_MINUTES_MS = 5 * 60 * 1000;
|
|
19
|
+
const FIFTEEN_MINUTES_MS = 15 * 60 * 1000;
|
|
20
|
+
const SIXTY_MINUTES_MS = 60 * 60 * 1000;
|
|
21
|
+
function placeholders(values) {
|
|
22
|
+
if (values.length === 0) {
|
|
23
|
+
throw new Error('Cannot build SQL placeholders for an empty list.');
|
|
24
|
+
}
|
|
25
|
+
return values.map(() => '?').join(', ');
|
|
26
|
+
}
|
|
27
|
+
function positiveLimit(value, fallback, max = MAX_RANGE_LIMIT) {
|
|
28
|
+
if (value === undefined) {
|
|
29
|
+
return fallback;
|
|
30
|
+
}
|
|
31
|
+
return Math.min(max, Math.max(0, Math.floor(value)));
|
|
32
|
+
}
|
|
33
|
+
function nowIso() {
|
|
34
|
+
return new Date().toISOString();
|
|
35
|
+
}
|
|
36
|
+
function eventDateFromMs(ms) {
|
|
37
|
+
if (ms === null || !Number.isFinite(ms)) {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
return new Date(ms).toISOString().slice(0, 10);
|
|
41
|
+
}
|
|
42
|
+
function normalizeTimestampMs(input) {
|
|
43
|
+
const timestamp = input.source_timestamp_ms ?? input.event_datetime;
|
|
44
|
+
if (typeof timestamp !== 'number' || !Number.isFinite(timestamp)) {
|
|
45
|
+
throw new Error('connector_event_index.source_timestamp_ms must be a finite number.');
|
|
46
|
+
}
|
|
47
|
+
return Math.floor(timestamp);
|
|
48
|
+
}
|
|
49
|
+
function normalizeEventDatetime(input) {
|
|
50
|
+
if (input.event_datetime === null || input.event_datetime === undefined) {
|
|
51
|
+
return normalizeTimestampMs(input);
|
|
52
|
+
}
|
|
53
|
+
if (!Number.isFinite(input.event_datetime)) {
|
|
54
|
+
throw new Error('connector_event_index.event_datetime must be a finite number when provided.');
|
|
55
|
+
}
|
|
56
|
+
return Math.floor(input.event_datetime);
|
|
57
|
+
}
|
|
58
|
+
function normalizeMetadataJson(input) {
|
|
59
|
+
if (input.metadata_json !== undefined) {
|
|
60
|
+
return input.metadata_json;
|
|
61
|
+
}
|
|
62
|
+
if (input.metadata === undefined || input.metadata === null) {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
return (0, canonicalize_js_1.canonicalizeJSON)(input.metadata);
|
|
66
|
+
}
|
|
67
|
+
function toBuffer(value) {
|
|
68
|
+
if (Buffer.isBuffer(value)) {
|
|
69
|
+
return value;
|
|
70
|
+
}
|
|
71
|
+
if (value instanceof Uint8Array) {
|
|
72
|
+
return Buffer.from(value);
|
|
73
|
+
}
|
|
74
|
+
throw new Error('connector_event_index.content_hash must be a 32-byte Buffer.');
|
|
75
|
+
}
|
|
76
|
+
function mapConnectorEventIndexRow(row) {
|
|
77
|
+
return {
|
|
78
|
+
event_index_id: String(row.event_index_id),
|
|
79
|
+
source_connector: String(row.source_connector),
|
|
80
|
+
source_type: String(row.source_type),
|
|
81
|
+
source_id: String(row.source_id),
|
|
82
|
+
source_locator: row.source_locator === null ? null : String(row.source_locator),
|
|
83
|
+
channel: row.channel === null ? null : String(row.channel),
|
|
84
|
+
author: row.author === null ? null : String(row.author),
|
|
85
|
+
title: row.title === null ? null : String(row.title),
|
|
86
|
+
content: String(row.content),
|
|
87
|
+
event_datetime: typeof row.event_datetime === 'number' && Number.isFinite(row.event_datetime)
|
|
88
|
+
? row.event_datetime
|
|
89
|
+
: null,
|
|
90
|
+
event_date: row.event_date === null ? null : String(row.event_date),
|
|
91
|
+
source_timestamp_ms: Number(row.source_timestamp_ms),
|
|
92
|
+
metadata_json: row.metadata_json === null ? null : String(row.metadata_json),
|
|
93
|
+
artifact_locator: row.artifact_locator === null ? null : String(row.artifact_locator),
|
|
94
|
+
artifact_title: row.artifact_title === null ? null : String(row.artifact_title),
|
|
95
|
+
content_hash: toBuffer(row.content_hash),
|
|
96
|
+
indexed_at: String(row.indexed_at),
|
|
97
|
+
updated_at: String(row.updated_at),
|
|
98
|
+
expires_at: row.expires_at === null ? null : String(row.expires_at),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
function mapConnectorCursorRow(row) {
|
|
102
|
+
return {
|
|
103
|
+
connector_name: String(row.connector_name),
|
|
104
|
+
last_seen_timestamp_ms: Number(row.last_seen_timestamp_ms),
|
|
105
|
+
last_seen_source_id: String(row.last_seen_source_id),
|
|
106
|
+
last_sweep_at: row.last_sweep_at === null ? null : String(row.last_sweep_at),
|
|
107
|
+
last_success_at: row.last_success_at === null ? null : String(row.last_success_at),
|
|
108
|
+
last_error: row.last_error === null ? null : String(row.last_error),
|
|
109
|
+
last_error_at: row.last_error_at === null ? null : String(row.last_error_at),
|
|
110
|
+
indexed_count: Number(row.indexed_count),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
function connectorFilterSql(connectors, alias = '') {
|
|
114
|
+
if (!connectors || connectors.length === 0) {
|
|
115
|
+
return '';
|
|
116
|
+
}
|
|
117
|
+
const prefix = alias.length > 0 ? `${alias}.` : '';
|
|
118
|
+
return ` AND ${prefix}source_connector IN (${placeholders(connectors)})`;
|
|
119
|
+
}
|
|
120
|
+
function hasOwn(input, key) {
|
|
121
|
+
return Object.prototype.hasOwnProperty.call(input, key);
|
|
122
|
+
}
|
|
123
|
+
function connectorEventIndexId(sourceConnector, sourceId) {
|
|
124
|
+
const digest = (0, node_crypto_1.createHash)('sha1').update(`${sourceConnector}\0${sourceId}`, 'utf8').digest('hex');
|
|
125
|
+
return `evt_${digest.slice(0, 16)}`;
|
|
126
|
+
}
|
|
127
|
+
function connectorEventContentHash(input) {
|
|
128
|
+
return (0, node_crypto_1.createHash)('sha256')
|
|
129
|
+
.update((0, canonicalize_js_1.canonicalizeJSON)({
|
|
130
|
+
source_connector: input.source_connector,
|
|
131
|
+
source_id: input.source_id,
|
|
132
|
+
content: input.content,
|
|
133
|
+
event_datetime: input.event_datetime ?? null,
|
|
134
|
+
}), 'utf8')
|
|
135
|
+
.digest();
|
|
136
|
+
}
|
|
137
|
+
function upsertConnectorEventIndex(adapter, input) {
|
|
138
|
+
const sourceTimestampMs = normalizeTimestampMs(input);
|
|
139
|
+
const eventDatetime = normalizeEventDatetime(input);
|
|
140
|
+
const eventDate = input.event_date ?? eventDateFromMs(eventDatetime);
|
|
141
|
+
const timestamp = input.updated_at ?? nowIso();
|
|
142
|
+
const eventIndexId = connectorEventIndexId(input.source_connector, input.source_id);
|
|
143
|
+
const contentHash = input.content_hash
|
|
144
|
+
? Buffer.from(input.content_hash)
|
|
145
|
+
: connectorEventContentHash({
|
|
146
|
+
source_connector: input.source_connector,
|
|
147
|
+
source_id: input.source_id,
|
|
148
|
+
content: input.content,
|
|
149
|
+
event_datetime: eventDatetime,
|
|
150
|
+
});
|
|
151
|
+
if (contentHash.byteLength !== 32) {
|
|
152
|
+
throw new Error('connector_event_index.content_hash must be exactly 32 bytes.');
|
|
153
|
+
}
|
|
154
|
+
const metadataJson = normalizeMetadataJson(input);
|
|
155
|
+
return adapter.transaction(() => {
|
|
156
|
+
adapter
|
|
157
|
+
.prepare(`
|
|
158
|
+
INSERT INTO connector_event_index (
|
|
159
|
+
event_index_id, source_connector, source_type, source_id, source_locator,
|
|
160
|
+
channel, author, title, content, event_datetime, event_date, source_timestamp_ms,
|
|
161
|
+
metadata_json, artifact_locator, artifact_title, content_hash, indexed_at,
|
|
162
|
+
updated_at, expires_at
|
|
163
|
+
)
|
|
164
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
165
|
+
ON CONFLICT(source_connector, source_id) DO UPDATE SET
|
|
166
|
+
source_type = excluded.source_type,
|
|
167
|
+
source_locator = excluded.source_locator,
|
|
168
|
+
channel = excluded.channel,
|
|
169
|
+
author = excluded.author,
|
|
170
|
+
title = excluded.title,
|
|
171
|
+
content = excluded.content,
|
|
172
|
+
event_datetime = excluded.event_datetime,
|
|
173
|
+
event_date = excluded.event_date,
|
|
174
|
+
source_timestamp_ms = excluded.source_timestamp_ms,
|
|
175
|
+
metadata_json = excluded.metadata_json,
|
|
176
|
+
artifact_locator = excluded.artifact_locator,
|
|
177
|
+
artifact_title = excluded.artifact_title,
|
|
178
|
+
content_hash = excluded.content_hash,
|
|
179
|
+
updated_at = excluded.updated_at,
|
|
180
|
+
expires_at = excluded.expires_at
|
|
181
|
+
`)
|
|
182
|
+
.run(eventIndexId, input.source_connector, input.source_type, input.source_id, input.source_locator ?? null, input.channel ?? null, input.author ?? null, input.title ?? null, input.content, eventDatetime, eventDate, sourceTimestampMs, metadataJson, input.artifact_locator ?? null, input.artifact_title ?? null, contentHash, input.indexed_at ?? timestamp, timestamp, input.expires_at ?? null);
|
|
183
|
+
const saved = getConnectorEventIndexRecord(adapter, input.source_connector, input.source_id);
|
|
184
|
+
if (!saved) {
|
|
185
|
+
throw new Error(`Failed to read connector_event_index row after upsert: ${input.source_connector}/${input.source_id}`);
|
|
186
|
+
}
|
|
187
|
+
return saved;
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
function getConnectorEventIndexRecord(adapter, sourceConnector, sourceId) {
|
|
191
|
+
const row = adapter
|
|
192
|
+
.prepare(`
|
|
193
|
+
SELECT *
|
|
194
|
+
FROM connector_event_index
|
|
195
|
+
WHERE source_connector = ?
|
|
196
|
+
AND source_id = ?
|
|
197
|
+
LIMIT 1
|
|
198
|
+
`)
|
|
199
|
+
.get(sourceConnector, sourceId);
|
|
200
|
+
return row ? mapConnectorEventIndexRow(row) : null;
|
|
201
|
+
}
|
|
202
|
+
function listConnectorEventsByDatetimeRange(adapter, input) {
|
|
203
|
+
const limit = positiveLimit(input.limit, DEFAULT_RANGE_LIMIT);
|
|
204
|
+
if (limit === 0) {
|
|
205
|
+
return [];
|
|
206
|
+
}
|
|
207
|
+
const order = input.order === 'desc' ? 'DESC' : 'ASC';
|
|
208
|
+
const connectors = input.connectors?.filter(Boolean) ?? [];
|
|
209
|
+
const params = [input.fromMs, input.toMs, ...connectors, limit];
|
|
210
|
+
const rows = adapter
|
|
211
|
+
.prepare(`
|
|
212
|
+
SELECT *
|
|
213
|
+
FROM connector_event_index
|
|
214
|
+
WHERE event_datetime >= ?
|
|
215
|
+
AND event_datetime <= ?
|
|
216
|
+
${connectorFilterSql(connectors)}
|
|
217
|
+
ORDER BY event_datetime ${order}, event_index_id ASC
|
|
218
|
+
LIMIT ?
|
|
219
|
+
`)
|
|
220
|
+
.all(...params);
|
|
221
|
+
return rows.map(mapConnectorEventIndexRow);
|
|
222
|
+
}
|
|
223
|
+
function searchConnectorEventsByFTS(adapter, query, options = {}) {
|
|
224
|
+
const normalizedQuery = query.trim();
|
|
225
|
+
if (normalizedQuery.length === 0) {
|
|
226
|
+
return [];
|
|
227
|
+
}
|
|
228
|
+
const limit = positiveLimit(options.limit, DEFAULT_SEARCH_LIMIT, 100);
|
|
229
|
+
if (limit === 0) {
|
|
230
|
+
return [];
|
|
231
|
+
}
|
|
232
|
+
const connectors = options.connectors?.filter(Boolean) ?? [];
|
|
233
|
+
const params = [normalizedQuery, ...connectors, limit];
|
|
234
|
+
const rows = adapter
|
|
235
|
+
.prepare(`
|
|
236
|
+
SELECT e.*, bm25(connector_event_index_fts) AS rank
|
|
237
|
+
FROM connector_event_index_fts
|
|
238
|
+
JOIN connector_event_index e
|
|
239
|
+
ON e.event_index_id = connector_event_index_fts.event_index_id
|
|
240
|
+
WHERE connector_event_index_fts MATCH ?
|
|
241
|
+
${connectorFilterSql(connectors, 'e')}
|
|
242
|
+
ORDER BY rank ASC, e.event_datetime DESC, e.event_index_id ASC
|
|
243
|
+
LIMIT ?
|
|
244
|
+
`)
|
|
245
|
+
.all(...params);
|
|
246
|
+
return rows.map((row) => {
|
|
247
|
+
const rank = Number(row.rank);
|
|
248
|
+
return {
|
|
249
|
+
...mapConnectorEventIndexRow(row),
|
|
250
|
+
rank,
|
|
251
|
+
score: Number.isFinite(rank) ? 1 / (1 + Math.exp(rank)) : 0,
|
|
252
|
+
};
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
function readConnectorCursor(adapter, connectorName) {
|
|
256
|
+
const row = adapter
|
|
257
|
+
.prepare(`
|
|
258
|
+
SELECT *
|
|
259
|
+
FROM connector_event_index_cursors
|
|
260
|
+
WHERE connector_name = ?
|
|
261
|
+
LIMIT 1
|
|
262
|
+
`)
|
|
263
|
+
.get(connectorName);
|
|
264
|
+
return row ? mapConnectorCursorRow(row) : null;
|
|
265
|
+
}
|
|
266
|
+
function upsertConnectorCursor(adapter, input) {
|
|
267
|
+
const current = readConnectorCursor(adapter, input.connector_name);
|
|
268
|
+
const merged = {
|
|
269
|
+
connector_name: input.connector_name,
|
|
270
|
+
last_seen_timestamp_ms: input.last_seen_timestamp_ms ?? current?.last_seen_timestamp_ms ?? 0,
|
|
271
|
+
last_seen_source_id: input.last_seen_source_id ?? current?.last_seen_source_id ?? '',
|
|
272
|
+
last_sweep_at: hasOwn(input, 'last_sweep_at')
|
|
273
|
+
? (input.last_sweep_at ?? null)
|
|
274
|
+
: (current?.last_sweep_at ?? null),
|
|
275
|
+
last_success_at: hasOwn(input, 'last_success_at')
|
|
276
|
+
? (input.last_success_at ?? null)
|
|
277
|
+
: (current?.last_success_at ?? null),
|
|
278
|
+
last_error: hasOwn(input, 'last_error')
|
|
279
|
+
? (input.last_error ?? null)
|
|
280
|
+
: (current?.last_error ?? null),
|
|
281
|
+
last_error_at: hasOwn(input, 'last_error_at')
|
|
282
|
+
? (input.last_error_at ?? null)
|
|
283
|
+
: (current?.last_error_at ?? null),
|
|
284
|
+
indexed_count: input.indexed_count ?? current?.indexed_count ?? 0,
|
|
285
|
+
};
|
|
286
|
+
adapter
|
|
287
|
+
.prepare(`
|
|
288
|
+
INSERT INTO connector_event_index_cursors (
|
|
289
|
+
connector_name, last_seen_timestamp_ms, last_seen_source_id, last_sweep_at,
|
|
290
|
+
last_success_at, last_error, last_error_at, indexed_count
|
|
291
|
+
)
|
|
292
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
293
|
+
ON CONFLICT(connector_name) DO UPDATE SET
|
|
294
|
+
last_seen_timestamp_ms = excluded.last_seen_timestamp_ms,
|
|
295
|
+
last_seen_source_id = excluded.last_seen_source_id,
|
|
296
|
+
last_sweep_at = excluded.last_sweep_at,
|
|
297
|
+
last_success_at = excluded.last_success_at,
|
|
298
|
+
last_error = excluded.last_error,
|
|
299
|
+
last_error_at = excluded.last_error_at,
|
|
300
|
+
indexed_count = excluded.indexed_count
|
|
301
|
+
`)
|
|
302
|
+
.run(merged.connector_name, merged.last_seen_timestamp_ms, merged.last_seen_source_id, merged.last_sweep_at, merged.last_success_at, merged.last_error, merged.last_error_at, merged.indexed_count);
|
|
303
|
+
const saved = readConnectorCursor(adapter, input.connector_name);
|
|
304
|
+
if (!saved) {
|
|
305
|
+
throw new Error(`Failed to read connector cursor after upsert: ${input.connector_name}`);
|
|
306
|
+
}
|
|
307
|
+
return saved;
|
|
308
|
+
}
|
|
309
|
+
function deleteExpiredConnectorEvents(adapter, input) {
|
|
310
|
+
if (!Number.isFinite(input.nowMs)) {
|
|
311
|
+
throw new Error('deleteExpiredConnectorEvents.nowMs must be a finite number.');
|
|
312
|
+
}
|
|
313
|
+
if (!Number.isFinite(input.retentionMs) || input.retentionMs < 0) {
|
|
314
|
+
throw new Error('deleteExpiredConnectorEvents.retentionMs must be a non-negative finite number.');
|
|
315
|
+
}
|
|
316
|
+
const cutoffMs = input.nowMs - input.retentionMs;
|
|
317
|
+
const result = input.connectorName
|
|
318
|
+
? adapter
|
|
319
|
+
.prepare(`
|
|
320
|
+
DELETE FROM connector_event_index
|
|
321
|
+
WHERE source_connector = ?
|
|
322
|
+
AND event_datetime IS NOT NULL
|
|
323
|
+
AND event_datetime < ?
|
|
324
|
+
AND artifact_locator IS NULL
|
|
325
|
+
`)
|
|
326
|
+
.run(input.connectorName, cutoffMs)
|
|
327
|
+
: adapter
|
|
328
|
+
.prepare(`
|
|
329
|
+
DELETE FROM connector_event_index
|
|
330
|
+
WHERE event_datetime IS NOT NULL
|
|
331
|
+
AND event_datetime < ?
|
|
332
|
+
AND artifact_locator IS NULL
|
|
333
|
+
`)
|
|
334
|
+
.run(cutoffMs);
|
|
335
|
+
return { rows_deleted: result.changes };
|
|
336
|
+
}
|
|
337
|
+
function computeStalenessStatus(cursor, options) {
|
|
338
|
+
if (!cursor) {
|
|
339
|
+
return 'never_swept';
|
|
340
|
+
}
|
|
341
|
+
if (cursor.last_error !== null) {
|
|
342
|
+
return 'unhealthy';
|
|
343
|
+
}
|
|
344
|
+
if (cursor.last_success_at === null) {
|
|
345
|
+
return 'never_swept';
|
|
346
|
+
}
|
|
347
|
+
const lastSuccessMs = Date.parse(cursor.last_success_at);
|
|
348
|
+
if (!Number.isFinite(lastSuccessMs)) {
|
|
349
|
+
return 'unhealthy';
|
|
350
|
+
}
|
|
351
|
+
const elapsedMs = options.nowMs - lastSuccessMs;
|
|
352
|
+
const staleAfterMs = options.staleAfterMs ?? FIVE_MINUTES_MS;
|
|
353
|
+
const warnAfterMs = options.stalenessWarnAfterMs ?? FIFTEEN_MINUTES_MS;
|
|
354
|
+
const unhealthyAfterMs = options.unhealthyAfterMs ?? SIXTY_MINUTES_MS;
|
|
355
|
+
if (elapsedMs > unhealthyAfterMs) {
|
|
356
|
+
return 'unhealthy';
|
|
357
|
+
}
|
|
358
|
+
if (elapsedMs > warnAfterMs) {
|
|
359
|
+
return 'warn';
|
|
360
|
+
}
|
|
361
|
+
if (elapsedMs > staleAfterMs) {
|
|
362
|
+
return 'stale-but-warming';
|
|
363
|
+
}
|
|
364
|
+
return 'healthy';
|
|
365
|
+
}
|
|
366
|
+
//# sourceMappingURL=event-index.js.map
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import type { Buffer } from 'node:buffer';
|
|
2
|
+
export interface ConnectorEventIndexRecord {
|
|
3
|
+
event_index_id: string;
|
|
4
|
+
source_connector: string;
|
|
5
|
+
source_type: string;
|
|
6
|
+
source_id: string;
|
|
7
|
+
source_locator: string | null;
|
|
8
|
+
channel: string | null;
|
|
9
|
+
author: string | null;
|
|
10
|
+
title: string | null;
|
|
11
|
+
content: string;
|
|
12
|
+
event_datetime: number | null;
|
|
13
|
+
event_date: string | null;
|
|
14
|
+
source_timestamp_ms: number;
|
|
15
|
+
metadata_json: string | null;
|
|
16
|
+
artifact_locator: string | null;
|
|
17
|
+
artifact_title: string | null;
|
|
18
|
+
content_hash: Buffer;
|
|
19
|
+
indexed_at: string;
|
|
20
|
+
updated_at: string;
|
|
21
|
+
expires_at: string | null;
|
|
22
|
+
}
|
|
23
|
+
export interface UpsertConnectorEventIndexInput {
|
|
24
|
+
source_connector: string;
|
|
25
|
+
source_type: string;
|
|
26
|
+
source_id: string;
|
|
27
|
+
source_locator?: string | null;
|
|
28
|
+
channel?: string | null;
|
|
29
|
+
author?: string | null;
|
|
30
|
+
title?: string | null;
|
|
31
|
+
content: string;
|
|
32
|
+
event_datetime?: number | null;
|
|
33
|
+
event_date?: string | null;
|
|
34
|
+
source_timestamp_ms?: number | null;
|
|
35
|
+
metadata_json?: string | null;
|
|
36
|
+
metadata?: unknown;
|
|
37
|
+
artifact_locator?: string | null;
|
|
38
|
+
artifact_title?: string | null;
|
|
39
|
+
content_hash?: Buffer | Uint8Array | null;
|
|
40
|
+
indexed_at?: string;
|
|
41
|
+
updated_at?: string;
|
|
42
|
+
expires_at?: string | null;
|
|
43
|
+
}
|
|
44
|
+
export interface ConnectorEventIndexCursorRecord {
|
|
45
|
+
connector_name: string;
|
|
46
|
+
last_seen_timestamp_ms: number;
|
|
47
|
+
last_seen_source_id: string;
|
|
48
|
+
last_sweep_at: string | null;
|
|
49
|
+
last_success_at: string | null;
|
|
50
|
+
last_error: string | null;
|
|
51
|
+
last_error_at: string | null;
|
|
52
|
+
indexed_count: number;
|
|
53
|
+
}
|
|
54
|
+
export interface UpsertConnectorEventIndexCursorInput {
|
|
55
|
+
connector_name: string;
|
|
56
|
+
last_seen_timestamp_ms?: number;
|
|
57
|
+
last_seen_source_id?: string;
|
|
58
|
+
last_sweep_at?: string | null;
|
|
59
|
+
last_success_at?: string | null;
|
|
60
|
+
last_error?: string | null;
|
|
61
|
+
last_error_at?: string | null;
|
|
62
|
+
indexed_count?: number;
|
|
63
|
+
}
|
|
64
|
+
export interface ConnectorEventSearchHit extends ConnectorEventIndexRecord {
|
|
65
|
+
rank: number;
|
|
66
|
+
score: number;
|
|
67
|
+
}
|
|
68
|
+
export type ConnectorEventStalenessStatus = 'healthy' | 'stale-but-warming' | 'warn' | 'unhealthy' | 'never_swept';
|
|
69
|
+
//# sourceMappingURL=types.d.ts.map
|
package/dist/db-manager.d.ts
CHANGED
|
@@ -57,6 +57,8 @@ export interface DecisionRecord {
|
|
|
57
57
|
edges?: DecisionEdgeRow[];
|
|
58
58
|
/** ISO 8601 date when the event actually occurred. Null if not set. */
|
|
59
59
|
event_date?: string | null;
|
|
60
|
+
/** Source event timestamp in milliseconds when known. Null if not set. */
|
|
61
|
+
event_datetime?: number | null;
|
|
60
62
|
}
|
|
61
63
|
export interface OutcomeData {
|
|
62
64
|
outcome?: string | null;
|
|
@@ -202,6 +204,8 @@ export interface DecisionInput {
|
|
|
202
204
|
risks?: string | null;
|
|
203
205
|
/** ISO 8601 date string for when the event actually occurred (e.g. "2023-01-15") */
|
|
204
206
|
event_date?: string | null;
|
|
207
|
+
/** Source event timestamp in milliseconds when known. */
|
|
208
|
+
event_datetime?: number | null;
|
|
205
209
|
}
|
|
206
210
|
/**
|
|
207
211
|
* Insert decision with embedding
|
package/dist/db-manager.js
CHANGED
|
@@ -267,6 +267,13 @@ async function insertDecisionWithEmbedding(decision) {
|
|
|
267
267
|
throw new Error(`Invalid event_date: must be ISO 8601 YYYY-MM-DD (got: ${d})`);
|
|
268
268
|
}
|
|
269
269
|
}
|
|
270
|
+
if (decision.event_datetime !== null && decision.event_datetime !== undefined) {
|
|
271
|
+
if (typeof decision.event_datetime !== 'number' ||
|
|
272
|
+
!Number.isFinite(decision.event_datetime) ||
|
|
273
|
+
decision.event_datetime <= 0) {
|
|
274
|
+
throw new Error(`Invalid event_datetime: must be a positive millisecond timestamp (got: ${decision.event_datetime})`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
270
277
|
try {
|
|
271
278
|
// Generate embedding BEFORE transaction (required for SQLite's sync transaction)
|
|
272
279
|
// Note: Redact topic for privacy - only log length
|
|
@@ -299,14 +306,14 @@ async function insertDecisionWithEmbedding(decision) {
|
|
|
299
306
|
confidence, created_at, updated_at,
|
|
300
307
|
needs_validation, validation_attempts, last_validated_at, usage_count,
|
|
301
308
|
trust_context, usage_success, usage_failure, time_saved,
|
|
302
|
-
evidence, alternatives, risks, event_date
|
|
303
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
309
|
+
evidence, alternatives, risks, event_date, event_datetime
|
|
310
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
304
311
|
`);
|
|
305
312
|
const insertResult = stmt.run(decision.id, decision.topic, decision.decision, decision.reasoning || null, decision.outcome || null, decision.failure_reason || null, decision.limitation || null, decision.user_involvement || null, decision.session_id || null, decision.supersedes || null, decision.superseded_by || null, decision.refined_from ? JSON.stringify(decision.refined_from) : null, decision.confidence !== undefined ? decision.confidence : 0.5,
|
|
306
313
|
// IMPORTANT: All timestamps are stored in milliseconds (Date.now()).
|
|
307
314
|
// The schema DEFAULT uses unixepoch() (seconds) but is never used
|
|
308
315
|
// since all inserts go through this function which always provides ms.
|
|
309
|
-
decision.created_at || Date.now(), decision.updated_at || Date.now(), decision.needs_validation !== undefined ? decision.needs_validation : 0, decision.validation_attempts || 0, decision.last_validated_at || null, decision.usage_count || 0, decision.trust_context || null, decision.usage_success || 0, decision.usage_failure || 0, decision.time_saved || 0, decision.evidence || null, decision.alternatives || null, decision.risks || null, decision.event_date || null);
|
|
316
|
+
decision.created_at || Date.now(), decision.updated_at || Date.now(), decision.needs_validation !== undefined ? decision.needs_validation : 0, decision.validation_attempts || 0, decision.last_validated_at || null, decision.usage_count || 0, decision.trust_context || null, decision.usage_success || 0, decision.usage_failure || 0, decision.time_saved || 0, decision.evidence || null, decision.alternatives || null, decision.risks || null, decision.event_date || null, decision.event_datetime ?? null);
|
|
310
317
|
const rowid = Number(insertResult.lastInsertRowid);
|
|
311
318
|
// Insert embedding in same transaction to ensure rowid matching
|
|
312
319
|
(0, debug_logger_js_1.info)(`[db-manager] Vector search enabled: ${adapter.vectorSearchEnabled}`);
|
|
@@ -38,6 +38,8 @@ export interface DecisionForFormat {
|
|
|
38
38
|
risks?: string;
|
|
39
39
|
/** ISO 8601 date when the event actually occurred. Null if not set. */
|
|
40
40
|
event_date?: string | null;
|
|
41
|
+
/** Source event timestamp in milliseconds when known. Null if not set. */
|
|
42
|
+
event_datetime?: number | null;
|
|
41
43
|
}
|
|
42
44
|
/**
|
|
43
45
|
* Trust context object
|