@jungjaehoon/mama-core 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/026-create-canonical-entity-tables.sql +99 -0
- package/db/migrations/027-create-entity-review-support.sql +44 -0
- package/db/migrations/028-create-entity-audit-tables.sql +45 -0
- package/db/migrations/029-extend-entity-observations-observation-type.sql +60 -0
- package/db/migrations/030-case-first-memory-substrate.sql +901 -0
- package/db/migrations/031-wiki-page-index-provenance-columns.sql +2 -0
- package/dist/canonicalize.d.ts +38 -0
- package/dist/canonicalize.js +106 -0
- package/dist/cases/case-links.d.ts +116 -0
- package/dist/cases/case-links.js +794 -0
- package/dist/cases/composition-overrides.d.ts +90 -0
- package/dist/cases/composition-overrides.js +724 -0
- package/dist/cases/corrections.d.ts +156 -0
- package/dist/cases/corrections.js +1045 -0
- package/dist/cases/freshness.d.ts +57 -0
- package/dist/cases/freshness.js +283 -0
- package/dist/cases/live-state.d.ts +48 -0
- package/dist/cases/live-state.js +256 -0
- package/dist/cases/membership-explain.d.ts +56 -0
- package/dist/cases/membership-explain.js +140 -0
- package/dist/cases/membership-matcher.d.ts +23 -0
- package/dist/cases/membership-matcher.js +193 -0
- package/dist/cases/merge-split.d.ts +61 -0
- package/dist/cases/merge-split.js +368 -0
- package/dist/cases/role-inference.d.ts +10 -0
- package/dist/cases/role-inference.js +85 -0
- package/dist/cases/search-rollup.d.ts +23 -0
- package/dist/cases/search-rollup.js +225 -0
- package/dist/cases/sqlite-transaction.d.ts +10 -0
- package/dist/cases/sqlite-transaction.js +62 -0
- package/dist/cases/store.d.ts +74 -0
- package/dist/cases/store.js +816 -0
- package/dist/cases/target-ref.d.ts +25 -0
- package/dist/cases/target-ref.js +22 -0
- package/dist/cases/timeline-range.d.ts +6 -0
- package/dist/cases/timeline-range.js +322 -0
- package/dist/cases/tombstone-sweeper.d.ts +13 -0
- package/dist/cases/tombstone-sweeper.js +128 -0
- package/dist/cases/types.d.ts +244 -0
- package/dist/cases/types.js +47 -0
- package/dist/cases/wiki-page-index.d.ts +50 -0
- package/dist/cases/wiki-page-index.js +350 -0
- package/dist/connectors/event-index.d.ts +44 -0
- package/dist/connectors/event-index.js +366 -0
- package/dist/connectors/types.d.ts +69 -0
- package/dist/connectors/types.js +3 -0
- package/dist/db-manager.d.ts +4 -0
- package/dist/db-manager.js +10 -3
- package/dist/decision-formatter.d.ts +2 -0
- package/dist/decision-formatter.js +11 -8
- package/dist/embedding-server/mobile/daemon.d.ts +1 -0
- package/dist/embedding-server/mobile/daemon.js +11 -1
- package/dist/entities/audit-metrics.d.ts +73 -0
- package/dist/entities/audit-metrics.js +128 -0
- package/dist/entities/candidate-generator.d.ts +9 -0
- package/dist/entities/candidate-generator.js +436 -0
- package/dist/entities/entity-impact.d.ts +41 -0
- package/dist/entities/entity-impact.js +140 -0
- package/dist/entities/entity-linked-decision-counts.d.ts +2 -0
- package/dist/entities/entity-linked-decision-counts.js +29 -0
- package/dist/entities/entity-list.d.ts +23 -0
- package/dist/entities/entity-list.js +99 -0
- package/dist/entities/entity-orphan-list.d.ts +26 -0
- package/dist/entities/entity-orphan-list.js +91 -0
- package/dist/entities/entity-search.d.ts +21 -0
- package/dist/entities/entity-search.js +150 -0
- package/dist/entities/errors.d.ts +65 -0
- package/dist/entities/errors.js +124 -0
- package/dist/entities/exact-merge-backfill.d.ts +11 -0
- package/dist/entities/exact-merge-backfill.js +161 -0
- package/dist/entities/lineage-backfill.d.ts +12 -0
- package/dist/entities/lineage-backfill.js +101 -0
- package/dist/entities/lineage-store.d.ts +51 -0
- package/dist/entities/lineage-store.js +308 -0
- package/dist/entities/normalization.d.ts +14 -0
- package/dist/entities/normalization.js +63 -0
- package/dist/entities/policy-store.d.ts +17 -0
- package/dist/entities/policy-store.js +263 -0
- package/dist/entities/policy-types.d.ts +72 -0
- package/dist/entities/policy-types.js +59 -0
- package/dist/entities/projection.d.ts +8 -0
- package/dist/entities/projection.js +82 -0
- package/dist/entities/provenance-query.d.ts +33 -0
- package/dist/entities/provenance-query.js +167 -0
- package/dist/entities/read-identity.d.ts +31 -0
- package/dist/entities/read-identity.js +264 -0
- package/dist/entities/recall-bridge.d.ts +5 -0
- package/dist/entities/recall-bridge.js +171 -0
- package/dist/entities/resolution-engine.d.ts +15 -0
- package/dist/entities/resolution-engine.js +69 -0
- package/dist/entities/rollback-preview.d.ts +39 -0
- package/dist/entities/rollback-preview.js +235 -0
- package/dist/entities/source-locator.d.ts +3 -0
- package/dist/entities/source-locator.js +32 -0
- package/dist/entities/store.d.ts +75 -0
- package/dist/entities/store.js +465 -0
- package/dist/entities/types.d.ts +162 -0
- package/dist/entities/types.js +30 -0
- package/dist/index.d.ts +47 -2
- package/dist/index.js +66 -25
- package/dist/mama-api.d.ts +22 -1
- package/dist/mama-api.js +272 -64
- package/dist/memory/api.d.ts +28 -1
- package/dist/memory/api.js +293 -31
- package/dist/memory/types.d.ts +11 -2
- package/dist/search/feedback-store.d.ts +59 -0
- package/dist/search/feedback-store.js +211 -0
- package/dist/search/question-type.d.ts +5 -0
- package/dist/search/question-type.js +41 -0
- package/dist/search/ranker-features.d.ts +23 -0
- package/dist/search/ranker-features.js +104 -0
- package/dist/search/ranker-rescore.d.ts +31 -0
- package/dist/search/ranker-rescore.js +226 -0
- package/dist/search/ranker-trainer.d.ts +94 -0
- package/dist/search/ranker-trainer.js +860 -0
- package/dist/test-utils.d.ts +14 -0
- package/dist/test-utils.js +52 -0
- package/package.json +21 -3
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.hashQuery = hashQuery;
|
|
4
|
+
exports.getFeedbackRetentionDays = getFeedbackRetentionDays;
|
|
5
|
+
exports.compactShownSearchFeedback = compactShownSearchFeedback;
|
|
6
|
+
exports.recordSearchFeedback = recordSearchFeedback;
|
|
7
|
+
exports.listSearchFeedback = listSearchFeedback;
|
|
8
|
+
const node_crypto_1 = require("node:crypto");
|
|
9
|
+
const question_type_js_1 = require("./question-type.js");
|
|
10
|
+
const DEFAULT_RETENTION_DAYS = 180;
|
|
11
|
+
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
12
|
+
function normalizedQuery(query) {
|
|
13
|
+
return query.trim().toLowerCase();
|
|
14
|
+
}
|
|
15
|
+
function fullQueryHashHex(query) {
|
|
16
|
+
return (0, node_crypto_1.createHash)('sha256').update(normalizedQuery(query)).digest('hex');
|
|
17
|
+
}
|
|
18
|
+
function fullQueryHashBuffer(query) {
|
|
19
|
+
return Buffer.from(fullQueryHashHex(query), 'hex');
|
|
20
|
+
}
|
|
21
|
+
function hashQuery(query) {
|
|
22
|
+
return fullQueryHashHex(query).slice(0, 16);
|
|
23
|
+
}
|
|
24
|
+
function assertNonEmpty(value, field) {
|
|
25
|
+
if (!value || value.trim().length === 0) {
|
|
26
|
+
throw new Error(`${field} is required`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function assertRank(value, field) {
|
|
30
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
31
|
+
throw new Error(`${field} must be a non-negative integer`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function nowIso() {
|
|
35
|
+
return new Date().toISOString();
|
|
36
|
+
}
|
|
37
|
+
function cutoffIso(retentionDays, now = new Date()) {
|
|
38
|
+
return new Date(now.getTime() - retentionDays * MS_PER_DAY).toISOString();
|
|
39
|
+
}
|
|
40
|
+
function stableObject(value) {
|
|
41
|
+
if (Array.isArray(value)) {
|
|
42
|
+
return value.map((item) => stableObject(item));
|
|
43
|
+
}
|
|
44
|
+
if (typeof value !== 'object' || value === null) {
|
|
45
|
+
return value;
|
|
46
|
+
}
|
|
47
|
+
const input = value;
|
|
48
|
+
const output = {};
|
|
49
|
+
for (const key of Object.keys(input).sort((left, right) => left.localeCompare(right))) {
|
|
50
|
+
const item = input[key];
|
|
51
|
+
if (item !== undefined) {
|
|
52
|
+
output[key] = stableObject(item);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return output;
|
|
56
|
+
}
|
|
57
|
+
function canonicalJson(value) {
|
|
58
|
+
return JSON.stringify(stableObject(value));
|
|
59
|
+
}
|
|
60
|
+
function runTransaction(adapter, fn) {
|
|
61
|
+
if (!adapter.transaction) {
|
|
62
|
+
throw new Error('recordSearchFeedback requires adapter.transaction for atomic writes.');
|
|
63
|
+
}
|
|
64
|
+
const result = adapter.transaction(fn);
|
|
65
|
+
if (typeof result === 'function') {
|
|
66
|
+
return result();
|
|
67
|
+
}
|
|
68
|
+
return result;
|
|
69
|
+
}
|
|
70
|
+
function parseRetentionValue(value) {
|
|
71
|
+
const parsed = JSON.parse(value);
|
|
72
|
+
const numeric = typeof parsed === 'number' ? parsed : Number(parsed);
|
|
73
|
+
if (!Number.isFinite(numeric) || numeric <= 0) {
|
|
74
|
+
throw new Error(`Invalid search_feedback_retention_days value: ${value}`);
|
|
75
|
+
}
|
|
76
|
+
return Math.floor(numeric);
|
|
77
|
+
}
|
|
78
|
+
function getFeedbackRetentionDays(adapter) {
|
|
79
|
+
const row = adapter
|
|
80
|
+
.prepare(`
|
|
81
|
+
SELECT value_json
|
|
82
|
+
FROM search_ranker_settings
|
|
83
|
+
WHERE key = 'search_feedback_retention_days'
|
|
84
|
+
`)
|
|
85
|
+
.get();
|
|
86
|
+
if (!row?.value_json) {
|
|
87
|
+
return DEFAULT_RETENTION_DAYS;
|
|
88
|
+
}
|
|
89
|
+
return parseRetentionValue(row.value_json);
|
|
90
|
+
}
|
|
91
|
+
function compactShownSearchFeedback(adapter, retentionDays = getFeedbackRetentionDays(adapter)) {
|
|
92
|
+
const result = adapter
|
|
93
|
+
.prepare(`
|
|
94
|
+
DELETE FROM search_feedback
|
|
95
|
+
WHERE feedback_kind = 'shown'
|
|
96
|
+
AND created_at < ?
|
|
97
|
+
`)
|
|
98
|
+
.run(cutoffIso(retentionDays));
|
|
99
|
+
return result.changes;
|
|
100
|
+
}
|
|
101
|
+
function sessionForInput(input, queryHash) {
|
|
102
|
+
if (input.session_id && input.session_id.trim().length > 0) {
|
|
103
|
+
return input.session_id;
|
|
104
|
+
}
|
|
105
|
+
if (input.feedback_kind === 'shown') {
|
|
106
|
+
return `anonymous:${queryHash}`;
|
|
107
|
+
}
|
|
108
|
+
return input.session_id ?? null;
|
|
109
|
+
}
|
|
110
|
+
function rankPositionForInput(input) {
|
|
111
|
+
if (input.feedback_kind === 'click' && input.clicked_result_rank !== undefined) {
|
|
112
|
+
const rank = input.clicked_result_rank ?? input.shown_index;
|
|
113
|
+
assertRank(rank, 'clicked_result_rank');
|
|
114
|
+
return rank;
|
|
115
|
+
}
|
|
116
|
+
assertRank(input.shown_index, 'shown_index');
|
|
117
|
+
return input.shown_index;
|
|
118
|
+
}
|
|
119
|
+
function questionTypeForInput(input) {
|
|
120
|
+
if (input.question_type && (0, question_type_js_1.isQuestionType)(input.question_type)) {
|
|
121
|
+
return input.question_type;
|
|
122
|
+
}
|
|
123
|
+
return (0, question_type_js_1.classifyQuestionType)(input.query);
|
|
124
|
+
}
|
|
125
|
+
function recordSearchFeedback(adapter, input) {
|
|
126
|
+
assertNonEmpty(input.result_id, 'result_id');
|
|
127
|
+
assertNonEmpty(input.query, 'query');
|
|
128
|
+
assertNonEmpty(input.result_source_type, 'result_source_type');
|
|
129
|
+
assertNonEmpty(input.result_source_id, 'result_source_id');
|
|
130
|
+
const queryHashHex = fullQueryHashHex(input.query);
|
|
131
|
+
const queryHash = fullQueryHashBuffer(input.query);
|
|
132
|
+
const queryHashShort = queryHashHex.slice(0, 16);
|
|
133
|
+
const sessionId = sessionForInput(input, queryHashShort);
|
|
134
|
+
const questionType = questionTypeForInput(input);
|
|
135
|
+
const rankPosition = rankPositionForInput(input);
|
|
136
|
+
const createdAt = input.created_at ?? nowIso();
|
|
137
|
+
const feedbackId = `search_feedback_${(0, node_crypto_1.randomUUID)()}`;
|
|
138
|
+
const metadataJson = canonicalJson({
|
|
139
|
+
clicked_result_rank: input.clicked_result_rank ?? null,
|
|
140
|
+
input_result_id: input.result_id,
|
|
141
|
+
});
|
|
142
|
+
return runTransaction(adapter, () => {
|
|
143
|
+
let deduped = false;
|
|
144
|
+
if (input.feedback_kind === 'shown') {
|
|
145
|
+
compactShownSearchFeedback(adapter);
|
|
146
|
+
if (sessionId) {
|
|
147
|
+
const deleted = adapter
|
|
148
|
+
.prepare(`
|
|
149
|
+
DELETE FROM search_feedback
|
|
150
|
+
WHERE feedback_kind = 'shown'
|
|
151
|
+
AND session_id = ?
|
|
152
|
+
AND query_hash = ?
|
|
153
|
+
AND result_source_type = ?
|
|
154
|
+
AND result_source_id = ?
|
|
155
|
+
`)
|
|
156
|
+
.run(sessionId, queryHash, input.result_source_type, input.result_source_id);
|
|
157
|
+
deduped = deleted.changes > 0;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
adapter
|
|
161
|
+
.prepare(`
|
|
162
|
+
INSERT INTO search_feedback (
|
|
163
|
+
feedback_id, query, query_hash, question_type, result_source_type, result_source_id,
|
|
164
|
+
case_id, feedback_kind, rank_position, score_before, score_after, session_id,
|
|
165
|
+
actor, metadata_json, created_at, updated_at
|
|
166
|
+
)
|
|
167
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, 'system:search-feedback', ?, ?, NULL)
|
|
168
|
+
`)
|
|
169
|
+
.run(feedbackId, input.query, queryHash, questionType, input.result_source_type, input.result_source_id, input.result_case_id ?? null, input.feedback_kind, rankPosition, sessionId, metadataJson, createdAt);
|
|
170
|
+
return {
|
|
171
|
+
feedback_id: feedbackId,
|
|
172
|
+
query_hash: queryHashShort,
|
|
173
|
+
deduped,
|
|
174
|
+
};
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
function listSearchFeedback(adapter, options = {}) {
|
|
178
|
+
const retentionDays = getFeedbackRetentionDays(adapter);
|
|
179
|
+
const until = options.until ?? nowIso();
|
|
180
|
+
const since = options.since ?? cutoffIso(retentionDays, new Date(until));
|
|
181
|
+
const limit = options.limit ?? 10_000;
|
|
182
|
+
assertRank(limit, 'limit');
|
|
183
|
+
const rows = adapter
|
|
184
|
+
.prepare(`
|
|
185
|
+
SELECT
|
|
186
|
+
feedback_id,
|
|
187
|
+
query,
|
|
188
|
+
lower(hex(query_hash)) AS query_hash_hex,
|
|
189
|
+
question_type,
|
|
190
|
+
result_source_type,
|
|
191
|
+
result_source_id,
|
|
192
|
+
case_id,
|
|
193
|
+
feedback_kind,
|
|
194
|
+
rank_position,
|
|
195
|
+
score_before,
|
|
196
|
+
score_after,
|
|
197
|
+
session_id,
|
|
198
|
+
actor,
|
|
199
|
+
metadata_json,
|
|
200
|
+
created_at,
|
|
201
|
+
updated_at
|
|
202
|
+
FROM search_feedback
|
|
203
|
+
WHERE created_at >= ?
|
|
204
|
+
AND created_at <= ?
|
|
205
|
+
ORDER BY created_at ASC, feedback_id ASC
|
|
206
|
+
LIMIT ?
|
|
207
|
+
`)
|
|
208
|
+
.all(since, until, limit);
|
|
209
|
+
return rows;
|
|
210
|
+
}
|
|
211
|
+
//# sourceMappingURL=feedback-store.js.map
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export type QuestionType = 'correction' | 'artifact' | 'timeline' | 'status' | 'decision_reason' | 'how_to' | 'unknown';
|
|
2
|
+
export declare const QUESTION_TYPES: readonly QuestionType[];
|
|
3
|
+
export declare function classifyQuestionType(query: string): QuestionType;
|
|
4
|
+
export declare function isQuestionType(value: string): value is QuestionType;
|
|
5
|
+
//# sourceMappingURL=question-type.d.ts.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.QUESTION_TYPES = void 0;
|
|
4
|
+
exports.classifyQuestionType = classifyQuestionType;
|
|
5
|
+
exports.isQuestionType = isQuestionType;
|
|
6
|
+
exports.QUESTION_TYPES = [
|
|
7
|
+
'correction',
|
|
8
|
+
'artifact',
|
|
9
|
+
'timeline',
|
|
10
|
+
'status',
|
|
11
|
+
'decision_reason',
|
|
12
|
+
'how_to',
|
|
13
|
+
'unknown',
|
|
14
|
+
];
|
|
15
|
+
// Korean: patterns for classifyQuestionType() multilingual query routing.
|
|
16
|
+
const CORRECTION_PATTERN = /\b(fix|revert|correct|correction|supersede|superseded|revise|revision|수정|되돌|정정|교정)\b/i; // Korean: keywords
|
|
17
|
+
const ARTIFACT_PATTERN = /\b(file|doc|docs|document|image|video|pdf|drive|obsidian|attachment|artifact|파일|문서|이미지|영상|첨부|드라이브|옵시디언)\b/i; // Korean: keywords
|
|
18
|
+
const TIMELINE_PATTERN = /\b(when|history|before|after|around|timeline|chronology|언제|이력|히스토리|전|후|즈음)\b|\bon\s+\d{4}(?:-\d{1,2})?(?:-\d{1,2})?\b|\b\d{4}-\d{1,2}-\d{1,2}\b/i; // Korean: keywords
|
|
19
|
+
const STATUS_PATTERN = /\b(status|state|current|currently|now|latest|progress|blocked|done|상태|현재|최신|진행|진척)\b/i; // Korean: keywords
|
|
20
|
+
const DECISION_REASON_PATTERN = /\b(why|reason|because|rationale|근거|이유|왜|때문)\b/i; // Korean: keywords
|
|
21
|
+
const HOW_TO_PATTERN = /\b(how\s+to|how\s+do|setup|set\s+up|configure|configuration|install|설정|구성|어떻게)\b/i; // Korean: keywords
|
|
22
|
+
function classifyQuestionType(query) {
|
|
23
|
+
const normalized = query.trim();
|
|
24
|
+
if (CORRECTION_PATTERN.test(normalized))
|
|
25
|
+
return 'correction';
|
|
26
|
+
if (ARTIFACT_PATTERN.test(normalized))
|
|
27
|
+
return 'artifact';
|
|
28
|
+
if (TIMELINE_PATTERN.test(normalized))
|
|
29
|
+
return 'timeline';
|
|
30
|
+
if (STATUS_PATTERN.test(normalized))
|
|
31
|
+
return 'status';
|
|
32
|
+
if (DECISION_REASON_PATTERN.test(normalized))
|
|
33
|
+
return 'decision_reason';
|
|
34
|
+
if (HOW_TO_PATTERN.test(normalized))
|
|
35
|
+
return 'how_to';
|
|
36
|
+
return 'unknown';
|
|
37
|
+
}
|
|
38
|
+
function isQuestionType(value) {
|
|
39
|
+
return exports.QUESTION_TYPES.includes(value);
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=question-type.js.map
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type QuestionType } from './question-type.js';
|
|
2
|
+
export declare const SEARCH_RANKER_FEATURE_SET_VERSION = "v1.0.0";
|
|
3
|
+
export type RankerSourceType = 'decision' | 'event' | 'observation' | 'artifact' | 'connector_event';
|
|
4
|
+
export declare const RANKER_SOURCE_TYPES: readonly RankerSourceType[];
|
|
5
|
+
export interface RankerFeatureVector {
|
|
6
|
+
bm25_score: number;
|
|
7
|
+
vector_score: number;
|
|
8
|
+
case_rollup_score: number;
|
|
9
|
+
recency_score: number;
|
|
10
|
+
same_case_boost: number;
|
|
11
|
+
source_type_onehot: Record<RankerSourceType, number>;
|
|
12
|
+
question_type_onehot: Record<QuestionType, number>;
|
|
13
|
+
}
|
|
14
|
+
export interface RankerQueryMeta {
|
|
15
|
+
question_type?: QuestionType;
|
|
16
|
+
case_id?: string | null;
|
|
17
|
+
}
|
|
18
|
+
type FeatureRow = Record<string, unknown>;
|
|
19
|
+
export declare function extractFeatures(row: FeatureRow, query: string, queryMeta?: RankerQueryMeta): RankerFeatureVector;
|
|
20
|
+
export declare function serializeFeatures(vec: RankerFeatureVector): number[];
|
|
21
|
+
export declare function featureSetVersion(): string;
|
|
22
|
+
export {};
|
|
23
|
+
//# sourceMappingURL=ranker-features.d.ts.map
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RANKER_SOURCE_TYPES = exports.SEARCH_RANKER_FEATURE_SET_VERSION = void 0;
|
|
4
|
+
exports.extractFeatures = extractFeatures;
|
|
5
|
+
exports.serializeFeatures = serializeFeatures;
|
|
6
|
+
exports.featureSetVersion = featureSetVersion;
|
|
7
|
+
const question_type_js_1 = require("./question-type.js");
|
|
8
|
+
exports.SEARCH_RANKER_FEATURE_SET_VERSION = 'v1.0.0';
|
|
9
|
+
exports.RANKER_SOURCE_TYPES = [
|
|
10
|
+
'decision',
|
|
11
|
+
'event',
|
|
12
|
+
'observation',
|
|
13
|
+
'artifact',
|
|
14
|
+
'connector_event',
|
|
15
|
+
];
|
|
16
|
+
function numberValue(value, fallback = 0) {
|
|
17
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
18
|
+
return fallback;
|
|
19
|
+
}
|
|
20
|
+
return value;
|
|
21
|
+
}
|
|
22
|
+
function stringValue(value) {
|
|
23
|
+
if (value === null || value === undefined) {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
return String(value);
|
|
27
|
+
}
|
|
28
|
+
function sourceTypeKey(row) {
|
|
29
|
+
const sourceType = stringValue(row.source_type ?? row.result_source_type ?? row.kind ?? row.page_type);
|
|
30
|
+
switch (sourceType) {
|
|
31
|
+
case 'decision':
|
|
32
|
+
return 'decision';
|
|
33
|
+
case 'event':
|
|
34
|
+
case 'checkpoint':
|
|
35
|
+
return 'event';
|
|
36
|
+
case 'observation':
|
|
37
|
+
case 'case':
|
|
38
|
+
return 'observation';
|
|
39
|
+
case 'artifact':
|
|
40
|
+
case 'wiki_page':
|
|
41
|
+
return 'artifact';
|
|
42
|
+
case 'connector_event':
|
|
43
|
+
case 'standalone_connector_hit':
|
|
44
|
+
return 'connector_event';
|
|
45
|
+
default:
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
function zeroSourceOnehot() {
|
|
50
|
+
return {
|
|
51
|
+
decision: 0,
|
|
52
|
+
event: 0,
|
|
53
|
+
observation: 0,
|
|
54
|
+
artifact: 0,
|
|
55
|
+
connector_event: 0,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function zeroQuestionOnehot() {
|
|
59
|
+
return {
|
|
60
|
+
correction: 0,
|
|
61
|
+
artifact: 0,
|
|
62
|
+
timeline: 0,
|
|
63
|
+
status: 0,
|
|
64
|
+
decision_reason: 0,
|
|
65
|
+
how_to: 0,
|
|
66
|
+
unknown: 0,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
function extractFeatures(row, query, queryMeta = {}) {
|
|
70
|
+
const sourceOnehot = zeroSourceOnehot();
|
|
71
|
+
const sourceType = sourceTypeKey(row);
|
|
72
|
+
if (sourceType) {
|
|
73
|
+
sourceOnehot[sourceType] = 1;
|
|
74
|
+
}
|
|
75
|
+
const questionType = queryMeta.question_type ?? (0, question_type_js_1.classifyQuestionType)(query);
|
|
76
|
+
const questionOnehot = zeroQuestionOnehot();
|
|
77
|
+
questionOnehot[questionType] = 1;
|
|
78
|
+
const rowCaseId = stringValue(row.case_id ?? row.result_case_id);
|
|
79
|
+
const queryCaseId = queryMeta.case_id ?? null;
|
|
80
|
+
return {
|
|
81
|
+
bm25_score: numberValue(row.bm25_score ?? row.bm25 ?? row.fts_score),
|
|
82
|
+
vector_score: numberValue(row.vector_score ?? row.similarity),
|
|
83
|
+
case_rollup_score: numberValue(row.case_rollup_score ?? row.retrieval_score ?? row.final_score ?? row.score),
|
|
84
|
+
recency_score: numberValue(row.recency_score),
|
|
85
|
+
same_case_boost: queryCaseId && rowCaseId && queryCaseId === rowCaseId ? 1 : 0,
|
|
86
|
+
source_type_onehot: sourceOnehot,
|
|
87
|
+
question_type_onehot: questionOnehot,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
function serializeFeatures(vec) {
|
|
91
|
+
return [
|
|
92
|
+
vec.bm25_score,
|
|
93
|
+
vec.vector_score,
|
|
94
|
+
vec.case_rollup_score,
|
|
95
|
+
vec.recency_score,
|
|
96
|
+
vec.same_case_boost,
|
|
97
|
+
...exports.RANKER_SOURCE_TYPES.map((sourceType) => vec.source_type_onehot[sourceType]),
|
|
98
|
+
...question_type_js_1.QUESTION_TYPES.map((questionType) => vec.question_type_onehot[questionType]),
|
|
99
|
+
];
|
|
100
|
+
}
|
|
101
|
+
function featureSetVersion() {
|
|
102
|
+
return exports.SEARCH_RANKER_FEATURE_SET_VERSION;
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=ranker-features.js.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { RunResult } from '../db-adapter/statement.js';
|
|
2
|
+
import { type SearchRankerModel } from './ranker-trainer.js';
|
|
3
|
+
export type SearchRankerSkippedReason = 'feature_disabled' | 'no_active_model' | 'feature_set_mismatch' | 'quality_gate_failed' | 'insufficient_result_count' | 'llm_reranking_requested';
|
|
4
|
+
export interface RankerRescoreAdapter {
|
|
5
|
+
prepare(sql: string): {
|
|
6
|
+
run(...params: unknown[]): RunResult;
|
|
7
|
+
get(...params: unknown[]): unknown;
|
|
8
|
+
all(...params: unknown[]): unknown[];
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
export interface RankerScoredFields {
|
|
12
|
+
ranker_score?: number;
|
|
13
|
+
score_before_ranker?: number;
|
|
14
|
+
final_score?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface RescoreSearchResultsOptions<T> {
|
|
17
|
+
query: string;
|
|
18
|
+
results: T[];
|
|
19
|
+
featureEnabled?: boolean;
|
|
20
|
+
useReranking?: boolean;
|
|
21
|
+
}
|
|
22
|
+
export interface RescoreSearchResultsResult<T> {
|
|
23
|
+
results: Array<T & RankerScoredFields>;
|
|
24
|
+
model_id: string | null;
|
|
25
|
+
skipped_reason?: SearchRankerSkippedReason;
|
|
26
|
+
}
|
|
27
|
+
export declare function loadActiveSearchRankerModel(adapter: RankerRescoreAdapter): SearchRankerModel | null;
|
|
28
|
+
export declare function isSearchRankerEnabled(adapter: RankerRescoreAdapter): boolean;
|
|
29
|
+
export declare function rescoreSearchResults<T extends Record<string, unknown>>(adapter: RankerRescoreAdapter, options: RescoreSearchResultsOptions<T>): RescoreSearchResultsResult<T>;
|
|
30
|
+
export declare function logisticScoreForRow(model: SearchRankerModel, row: Record<string, unknown>, query: string): number;
|
|
31
|
+
//# sourceMappingURL=ranker-rescore.d.ts.map
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.loadActiveSearchRankerModel = loadActiveSearchRankerModel;
|
|
4
|
+
exports.isSearchRankerEnabled = isSearchRankerEnabled;
|
|
5
|
+
exports.rescoreSearchResults = rescoreSearchResults;
|
|
6
|
+
exports.logisticScoreForRow = logisticScoreForRow;
|
|
7
|
+
const ranker_features_js_1 = require("./ranker-features.js");
|
|
8
|
+
const ranker_trainer_js_1 = require("./ranker-trainer.js");
|
|
9
|
+
const question_type_js_1 = require("./question-type.js");
|
|
10
|
+
function parseJsonObject(value) {
|
|
11
|
+
const parsed = JSON.parse(value);
|
|
12
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
|
|
13
|
+
throw new Error('Expected JSON object');
|
|
14
|
+
}
|
|
15
|
+
return parsed;
|
|
16
|
+
}
|
|
17
|
+
function parseNumberArray(value) {
|
|
18
|
+
if (!Array.isArray(value))
|
|
19
|
+
return [];
|
|
20
|
+
return value.map((item) => (typeof item === 'number' && Number.isFinite(item) ? item : 0));
|
|
21
|
+
}
|
|
22
|
+
function parseQuestionWeights(value, fallback) {
|
|
23
|
+
const object = typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
24
|
+
? value
|
|
25
|
+
: {};
|
|
26
|
+
return {
|
|
27
|
+
correction: parseNumberArray(object.correction).length > 0
|
|
28
|
+
? parseNumberArray(object.correction)
|
|
29
|
+
: [...fallback],
|
|
30
|
+
artifact: parseNumberArray(object.artifact).length > 0
|
|
31
|
+
? parseNumberArray(object.artifact)
|
|
32
|
+
: [...fallback],
|
|
33
|
+
timeline: parseNumberArray(object.timeline).length > 0
|
|
34
|
+
? parseNumberArray(object.timeline)
|
|
35
|
+
: [...fallback],
|
|
36
|
+
status: parseNumberArray(object.status).length > 0 ? parseNumberArray(object.status) : [...fallback],
|
|
37
|
+
decision_reason: parseNumberArray(object.decision_reason).length > 0
|
|
38
|
+
? parseNumberArray(object.decision_reason)
|
|
39
|
+
: [...fallback],
|
|
40
|
+
how_to: parseNumberArray(object.how_to).length > 0 ? parseNumberArray(object.how_to) : [...fallback],
|
|
41
|
+
unknown: parseNumberArray(object.unknown).length > 0
|
|
42
|
+
? parseNumberArray(object.unknown)
|
|
43
|
+
: [...fallback],
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function modelFromRow(row) {
|
|
47
|
+
const payload = parseJsonObject(row.coefficients_json);
|
|
48
|
+
const coefficients = parseNumberArray(payload.coefficients);
|
|
49
|
+
const intercept = typeof payload.intercept === 'number' ? payload.intercept : 0;
|
|
50
|
+
const trainingRowsCount = typeof payload.training_rows_count === 'number' ? payload.training_rows_count : 0;
|
|
51
|
+
return {
|
|
52
|
+
model_id: row.model_id,
|
|
53
|
+
feature_set_version: row.feature_set_version,
|
|
54
|
+
trained_at: row.trained_at,
|
|
55
|
+
training_rows_count: trainingRowsCount,
|
|
56
|
+
coefficients,
|
|
57
|
+
intercept,
|
|
58
|
+
question_type_weights: parseQuestionWeights(payload.question_type_weights, coefficients),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
function loadActiveRow(adapter) {
|
|
62
|
+
const row = adapter
|
|
63
|
+
.prepare(`
|
|
64
|
+
SELECT model_id, feature_set_version, coefficients_json, quality_gate_status, trained_at
|
|
65
|
+
FROM ranker_model_versions
|
|
66
|
+
WHERE active = 1
|
|
67
|
+
ORDER BY trained_at DESC
|
|
68
|
+
LIMIT 1
|
|
69
|
+
`)
|
|
70
|
+
.get();
|
|
71
|
+
return row ?? null;
|
|
72
|
+
}
|
|
73
|
+
function logSkipped(reason, modelId) {
|
|
74
|
+
console.warn(JSON.stringify({
|
|
75
|
+
event: 'search_ranker_skipped',
|
|
76
|
+
skipped_reason: reason,
|
|
77
|
+
model_id: modelId,
|
|
78
|
+
feature_set_version: ranker_features_js_1.SEARCH_RANKER_FEATURE_SET_VERSION,
|
|
79
|
+
}));
|
|
80
|
+
}
|
|
81
|
+
function loadActiveSearchRankerModel(adapter) {
|
|
82
|
+
const row = loadActiveRow(adapter);
|
|
83
|
+
if (!row) {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
if (row.feature_set_version !== ranker_features_js_1.SEARCH_RANKER_FEATURE_SET_VERSION) {
|
|
87
|
+
logSkipped('feature_set_mismatch', row.model_id);
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
return modelFromRow(row);
|
|
91
|
+
}
|
|
92
|
+
function isSearchRankerEnabled(adapter) {
|
|
93
|
+
const row = adapter
|
|
94
|
+
.prepare(`
|
|
95
|
+
SELECT value_json
|
|
96
|
+
FROM search_ranker_settings
|
|
97
|
+
WHERE key = 'search_ranker_enabled'
|
|
98
|
+
`)
|
|
99
|
+
.get();
|
|
100
|
+
if (!row?.value_json) {
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
const parsed = JSON.parse(row.value_json);
|
|
104
|
+
return parsed !== false;
|
|
105
|
+
}
|
|
106
|
+
function baseScore(row) {
|
|
107
|
+
const candidates = [
|
|
108
|
+
row.final_score,
|
|
109
|
+
row.retrieval_score,
|
|
110
|
+
row.score,
|
|
111
|
+
row.similarity,
|
|
112
|
+
row.confidence,
|
|
113
|
+
row.case_rollup_score,
|
|
114
|
+
row.vector_score,
|
|
115
|
+
row.bm25_score,
|
|
116
|
+
];
|
|
117
|
+
for (const candidate of candidates) {
|
|
118
|
+
if (typeof candidate === 'number' && Number.isFinite(candidate)) {
|
|
119
|
+
return candidate;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return 0;
|
|
123
|
+
}
|
|
124
|
+
function resultId(row, fallback) {
|
|
125
|
+
const id = row.id ?? row.source_id ?? row.result_source_id;
|
|
126
|
+
return id === undefined || id === null ? String(fallback) : String(id);
|
|
127
|
+
}
|
|
128
|
+
function rescoreSearchResults(adapter, options) {
|
|
129
|
+
const passthrough = options.results;
|
|
130
|
+
if (options.useReranking) {
|
|
131
|
+
return {
|
|
132
|
+
results: passthrough,
|
|
133
|
+
model_id: null,
|
|
134
|
+
skipped_reason: 'llm_reranking_requested',
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
if (options.featureEnabled === false) {
|
|
138
|
+
return {
|
|
139
|
+
results: passthrough,
|
|
140
|
+
model_id: null,
|
|
141
|
+
skipped_reason: 'feature_disabled',
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
if (options.results.length < 2) {
|
|
145
|
+
return {
|
|
146
|
+
results: passthrough,
|
|
147
|
+
model_id: null,
|
|
148
|
+
skipped_reason: 'insufficient_result_count',
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
const row = loadActiveRow(adapter);
|
|
152
|
+
if (!row) {
|
|
153
|
+
return {
|
|
154
|
+
results: passthrough,
|
|
155
|
+
model_id: null,
|
|
156
|
+
skipped_reason: 'no_active_model',
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
if (row.feature_set_version !== ranker_features_js_1.SEARCH_RANKER_FEATURE_SET_VERSION) {
|
|
160
|
+
logSkipped('feature_set_mismatch', row.model_id);
|
|
161
|
+
return {
|
|
162
|
+
results: passthrough,
|
|
163
|
+
model_id: row.model_id,
|
|
164
|
+
skipped_reason: 'feature_set_mismatch',
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
if (row.quality_gate_status === 'failed') {
|
|
168
|
+
return {
|
|
169
|
+
results: passthrough,
|
|
170
|
+
model_id: row.model_id,
|
|
171
|
+
skipped_reason: 'quality_gate_failed',
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
if (row.quality_gate_status !== 'passed') {
|
|
175
|
+
return {
|
|
176
|
+
results: passthrough,
|
|
177
|
+
model_id: row.model_id,
|
|
178
|
+
skipped_reason: 'quality_gate_failed',
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
const model = modelFromRow(row);
|
|
182
|
+
const questionType = (0, question_type_js_1.classifyQuestionType)(options.query);
|
|
183
|
+
const scored = options.results.map((result, index) => {
|
|
184
|
+
const scoreBeforeRanker = baseScore(result);
|
|
185
|
+
const rankerScore = (0, ranker_trainer_js_1.scoreWithRankerModel)(model, result, options.query, questionType);
|
|
186
|
+
const finalScore = 0.8 * scoreBeforeRanker + 0.2 * rankerScore;
|
|
187
|
+
return {
|
|
188
|
+
result: {
|
|
189
|
+
...result,
|
|
190
|
+
ranker_score: rankerScore,
|
|
191
|
+
score_before_ranker: scoreBeforeRanker,
|
|
192
|
+
final_score: finalScore,
|
|
193
|
+
},
|
|
194
|
+
index,
|
|
195
|
+
id: resultId(result, index),
|
|
196
|
+
finalScore,
|
|
197
|
+
};
|
|
198
|
+
});
|
|
199
|
+
scored.sort((left, right) => {
|
|
200
|
+
const diff = right.finalScore - left.finalScore;
|
|
201
|
+
if (Math.abs(diff) > 1e-12)
|
|
202
|
+
return diff;
|
|
203
|
+
if (left.index !== right.index)
|
|
204
|
+
return left.index - right.index;
|
|
205
|
+
return left.id.localeCompare(right.id);
|
|
206
|
+
});
|
|
207
|
+
return {
|
|
208
|
+
results: scored.map((entry) => entry.result),
|
|
209
|
+
model_id: model.model_id,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
function logisticScoreForRow(model, row, query) {
|
|
213
|
+
const questionType = (0, question_type_js_1.classifyQuestionType)(query);
|
|
214
|
+
const features = (0, ranker_features_js_1.serializeFeatures)((0, ranker_features_js_1.extractFeatures)(row, query, { question_type: questionType }));
|
|
215
|
+
const coefficients = model.question_type_weights[questionType] ?? model.coefficients;
|
|
216
|
+
let score = model.intercept;
|
|
217
|
+
for (let index = 0; index < Math.min(coefficients.length, features.length); index += 1) {
|
|
218
|
+
score += coefficients[index] * features[index];
|
|
219
|
+
}
|
|
220
|
+
if (score >= 35)
|
|
221
|
+
return 1;
|
|
222
|
+
if (score <= -35)
|
|
223
|
+
return 0;
|
|
224
|
+
return 1 / (1 + Math.exp(-score));
|
|
225
|
+
}
|
|
226
|
+
//# sourceMappingURL=ranker-rescore.js.map
|