@memberjunction/search-engine 5.32.0 → 5.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/generic/BaseReRanker.d.ts +164 -0
- package/dist/generic/BaseReRanker.d.ts.map +1 -0
- package/dist/generic/BaseReRanker.js +209 -0
- package/dist/generic/BaseReRanker.js.map +1 -0
- package/dist/generic/EntitySearchProvider.d.ts +51 -3
- package/dist/generic/EntitySearchProvider.d.ts.map +1 -1
- package/dist/generic/EntitySearchProvider.js +130 -17
- package/dist/generic/EntitySearchProvider.js.map +1 -1
- package/dist/generic/FullTextSearchProvider.d.ts +7 -2
- package/dist/generic/FullTextSearchProvider.d.ts.map +1 -1
- package/dist/generic/FullTextSearchProvider.js +25 -4
- package/dist/generic/FullTextSearchProvider.js.map +1 -1
- package/dist/generic/ISearchProvider.d.ts +44 -2
- package/dist/generic/ISearchProvider.d.ts.map +1 -1
- package/dist/generic/ISearchProvider.js +35 -1
- package/dist/generic/ISearchProvider.js.map +1 -1
- package/dist/generic/NoopReRanker.d.ts +28 -0
- package/dist/generic/NoopReRanker.d.ts.map +1 -0
- package/dist/generic/NoopReRanker.js +49 -0
- package/dist/generic/NoopReRanker.js.map +1 -0
- package/dist/generic/ScopeTemplateRenderer.d.ts +36 -0
- package/dist/generic/ScopeTemplateRenderer.d.ts.map +1 -0
- package/dist/generic/ScopeTemplateRenderer.js +110 -0
- package/dist/generic/ScopeTemplateRenderer.js.map +1 -0
- package/dist/generic/SearchEngine.d.ts +180 -12
- package/dist/generic/SearchEngine.d.ts.map +1 -1
- package/dist/generic/SearchEngine.js +737 -35
- package/dist/generic/SearchEngine.js.map +1 -1
- package/dist/generic/SearchFusion.d.ts +40 -6
- package/dist/generic/SearchFusion.d.ts.map +1 -1
- package/dist/generic/SearchFusion.js +139 -18
- package/dist/generic/SearchFusion.js.map +1 -1
- package/dist/generic/StorageSearchProvider.d.ts +9 -2
- package/dist/generic/StorageSearchProvider.d.ts.map +1 -1
- package/dist/generic/StorageSearchProvider.js +44 -12
- package/dist/generic/StorageSearchProvider.js.map +1 -1
- package/dist/generic/VectorSearchProvider.d.ts +9 -2
- package/dist/generic/VectorSearchProvider.d.ts.map +1 -1
- package/dist/generic/VectorSearchProvider.js +83 -13
- package/dist/generic/VectorSearchProvider.js.map +1 -1
- package/dist/generic/search.types.d.ts +206 -0
- package/dist/generic/search.types.d.ts.map +1 -1
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -0
- package/dist/index.js.map +1 -1
- package/dist/permissions/SearchScopePermissionResolver.d.ts +109 -0
- package/dist/permissions/SearchScopePermissionResolver.d.ts.map +1 -0
- package/dist/permissions/SearchScopePermissionResolver.js +159 -0
- package/dist/permissions/SearchScopePermissionResolver.js.map +1 -0
- package/dist/providers/AzureAISearchProvider.d.ts +37 -0
- package/dist/providers/AzureAISearchProvider.d.ts.map +1 -0
- package/dist/providers/AzureAISearchProvider.js +180 -0
- package/dist/providers/AzureAISearchProvider.js.map +1 -0
- package/dist/providers/ElasticsearchSearchProvider.d.ts +43 -0
- package/dist/providers/ElasticsearchSearchProvider.d.ts.map +1 -0
- package/dist/providers/ElasticsearchSearchProvider.js +200 -0
- package/dist/providers/ElasticsearchSearchProvider.js.map +1 -0
- package/dist/providers/OpenSearchSearchProvider.d.ts +36 -0
- package/dist/providers/OpenSearchSearchProvider.d.ts.map +1 -0
- package/dist/providers/OpenSearchSearchProvider.js +167 -0
- package/dist/providers/OpenSearchSearchProvider.js.map +1 -0
- package/dist/providers/TypesenseSearchProvider.d.ts +36 -0
- package/dist/providers/TypesenseSearchProvider.d.ts.map +1 -0
- package/dist/providers/TypesenseSearchProvider.js +161 -0
- package/dist/providers/TypesenseSearchProvider.js.map +1 -0
- package/dist/rerankers/BGEReRanker.d.ts +57 -0
- package/dist/rerankers/BGEReRanker.d.ts.map +1 -0
- package/dist/rerankers/BGEReRanker.js +193 -0
- package/dist/rerankers/BGEReRanker.js.map +1 -0
- package/dist/rerankers/CohereReRanker.d.ts +65 -0
- package/dist/rerankers/CohereReRanker.d.ts.map +1 -0
- package/dist/rerankers/CohereReRanker.js +155 -0
- package/dist/rerankers/CohereReRanker.js.map +1 -0
- package/dist/rerankers/OpenAIReRanker.d.ts +62 -0
- package/dist/rerankers/OpenAIReRanker.d.ts.map +1 -0
- package/dist/rerankers/OpenAIReRanker.js +197 -0
- package/dist/rerankers/OpenAIReRanker.js.map +1 -0
- package/dist/rerankers/RerankerBudgetGuard.d.ts +54 -0
- package/dist/rerankers/RerankerBudgetGuard.d.ts.map +1 -0
- package/dist/rerankers/RerankerBudgetGuard.js +67 -0
- package/dist/rerankers/RerankerBudgetGuard.js.map +1 -0
- package/dist/rerankers/VoyageReRanker.d.ts +59 -0
- package/dist/rerankers/VoyageReRanker.d.ts.map +1 -0
- package/dist/rerankers/VoyageReRanker.js +184 -0
- package/dist/rerankers/VoyageReRanker.js.map +1 -0
- package/package.json +13 -8
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Lightweight Nunjucks renderer for scope configuration values.
|
|
3
|
+
*
|
|
4
|
+
* `SearchScope*` tables store template strings (MetadataFilter, ExtraFilter,
|
|
5
|
+
* UserSearchString, FolderPath) that embed `SearchContext` variables — for example:
|
|
6
|
+
*
|
|
7
|
+
* `OrganizationID='{{ context.PrimaryScopeRecordID }}' AND DepartmentID='{{ context.SecondaryScopes.dept.value }}'`
|
|
8
|
+
*
|
|
9
|
+
* This module renders those strings with a minimal, pre-configured Nunjucks environment
|
|
10
|
+
* so the SearchEngine does not have to bootstrap the full `@memberjunction/templates`
|
|
11
|
+
* engine (which is intended for stored, managed templates, not ad-hoc config values).
|
|
12
|
+
*
|
|
13
|
+
* For stored template resolution (e.g., `AIAgentSearchScope.QueryTemplateID`,
|
|
14
|
+
* `SearchScopeProvider.QueryTransformTemplateID`) use `@memberjunction/templates` with
|
|
15
|
+
* `TemplateEngineServer` — that path runs in Phase 1C (AgentPreExecutionRAG) where we
|
|
16
|
+
* already have a stored-template workflow.
|
|
17
|
+
*
|
|
18
|
+
* @module @memberjunction/search-engine
|
|
19
|
+
*/
|
|
20
|
+
import nunjucks from 'nunjucks';
|
|
21
|
+
import { LogError } from '@memberjunction/core';
|
|
22
|
+
/** Environment used for all ad-hoc scope template rendering. */
|
|
23
|
+
const env = new nunjucks.Environment(null, {
|
|
24
|
+
autoescape: false, // values go into filters / metadata, not HTML
|
|
25
|
+
throwOnUndefined: false,
|
|
26
|
+
trimBlocks: true,
|
|
27
|
+
lstripBlocks: true
|
|
28
|
+
});
|
|
29
|
+
// Add the same JSON-oriented filters exposed by @memberjunction/templates so scope
|
|
30
|
+
// authors can use familiar helpers (`{{ foo | json }}`, `{{ raw | jsonparse }}`) when
|
|
31
|
+
// composing MetadataFilter strings.
|
|
32
|
+
env.addFilter('json', (value, indent = 2) => {
|
|
33
|
+
if (value === undefined || value === null)
|
|
34
|
+
return '';
|
|
35
|
+
try {
|
|
36
|
+
return JSON.stringify(value, null, indent);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return String(value);
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
env.addFilter('jsoninline', (value) => {
|
|
43
|
+
if (value === undefined || value === null)
|
|
44
|
+
return '';
|
|
45
|
+
try {
|
|
46
|
+
return JSON.stringify(value);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return String(value);
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
env.addFilter('jsonparse', (value) => {
|
|
53
|
+
if (typeof value !== 'string')
|
|
54
|
+
return value;
|
|
55
|
+
try {
|
|
56
|
+
return JSON.parse(value);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return value;
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
/**
|
|
63
|
+
* Render a scope template string with the supplied SearchContext. Returns the original
|
|
64
|
+
* string unchanged when `template` is null/empty. Returns the original string on render
|
|
65
|
+
* failure (logged via LogError) so a single bad template does not bring down a search.
|
|
66
|
+
*/
|
|
67
|
+
export function RenderScopeTemplate(template, context, extraData) {
|
|
68
|
+
if (!template)
|
|
69
|
+
return '';
|
|
70
|
+
if (!template.includes('{{') && !template.includes('{%')) {
|
|
71
|
+
// No templating syntax — skip the renderer entirely
|
|
72
|
+
return template;
|
|
73
|
+
}
|
|
74
|
+
const data = {
|
|
75
|
+
context: context ?? {},
|
|
76
|
+
...(extraData ?? {})
|
|
77
|
+
};
|
|
78
|
+
try {
|
|
79
|
+
return env.renderString(template, data);
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
83
|
+
LogError(`SearchEngine: Scope template render failed — returning raw template. Error: ${msg}. Template: ${template.substring(0, 200)}`);
|
|
84
|
+
return template;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Render and parse a JSON-valued scope template (used for `MetadataFilter` values).
|
|
89
|
+
* Returns:
|
|
90
|
+
* - `undefined` when the template is null/empty
|
|
91
|
+
* - parsed object when render output is valid JSON
|
|
92
|
+
* - the raw rendered string when it is non-empty but not JSON (provider can decide what to do)
|
|
93
|
+
* - `undefined` when render fails outright
|
|
94
|
+
*/
|
|
95
|
+
export function RenderScopeJsonTemplate(template, context, extraData) {
|
|
96
|
+
if (!template)
|
|
97
|
+
return undefined;
|
|
98
|
+
const rendered = RenderScopeTemplate(template, context, extraData);
|
|
99
|
+
const trimmed = rendered.trim();
|
|
100
|
+
if (!trimmed)
|
|
101
|
+
return undefined;
|
|
102
|
+
try {
|
|
103
|
+
return JSON.parse(trimmed);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
// Return the raw rendered string — providers can still interpret
|
|
107
|
+
return trimmed;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
//# sourceMappingURL=ScopeTemplateRenderer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ScopeTemplateRenderer.js","sourceRoot":"","sources":["../../src/generic/ScopeTemplateRenderer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,gEAAgE;AAChE,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,IAAmC,EAAE;IACtE,UAAU,EAAE,KAAK,EAAE,8CAA8C;IACjE,gBAAgB,EAAE,KAAK;IACvB,UAAU,EAAE,IAAI;IAChB,YAAY,EAAE,IAAI;CACrB,CAAC,CAAC;AAEH,mFAAmF;AACnF,sFAAsF;AACtF,oCAAoC;AACpC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,KAAc,EAAE,SAAiB,CAAC,EAAU,EAAE;IACjE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACrD,IAAI,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,KAAc,EAAU,EAAE;IACnD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACrD,IAAI,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,KAAc,EAAW,EAAE;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAC/B,QAAmC,EACnC,OAAkC,EAClC,SAAmC;IAEnC,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,CAAC;IACzB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACvD,oDAAoD;QACpD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,IAAI,GAA4B;QAClC,OAAO,EAAE,OAAO,IAAI,EAAE;QACtB,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC;KACvB,CAAC;IAEF,IAAI,CAAC;QACD,OAAO,GAAG,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,QAAQ,CAAC,+EAA+E,GAAG,eAAe,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACxI,OAAO,QAAQ,CAAC;IACpB,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CACnC,QAAmC,EACnC,OAAkC,EAClC,SAAmC;IAEnC,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,IAAI,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACL,iEAAiE;QACjE,OAAO,OAAO,CAAC;IACnB,CAAC;AACL,CAAC"}
|
|
@@ -5,7 +5,14 @@
|
|
|
5
5
|
* fuses results with Reciprocal Rank Fusion (RRF), applies enrichment
|
|
6
6
|
* (entity icons, record names, tags), and filters by minimum score.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
8
|
+
* When `SearchParams.ScopeIDs` is provided, the engine resolves each scope against
|
|
9
|
+
* `SearchEngineBase` (which caches all `MJ: Search Scope*` metadata), builds a
|
|
10
|
+
* `ScopeConstraints` object per scope (including Nunjucks-rendered MetadataFilter,
|
|
11
|
+
* ExtraFilter, UserSearchString, and FolderPath values), runs each scope's providers
|
|
12
|
+
* in parallel, then fuses the per-scope results via cross-scope RRF. An optional
|
|
13
|
+
* re-ranker stage (`BaseReRanker`) runs after fusion when configured.
|
|
14
|
+
*
|
|
15
|
+
* Providers are loaded via `@RegisterClass(BaseSearchProvider, DriverClass)` and
|
|
9
16
|
* instantiated using the MJ ClassFactory based on active SearchProvider records.
|
|
10
17
|
*
|
|
11
18
|
* Uses BaseSingleton from @memberjunction/global for a truly global instance.
|
|
@@ -15,32 +22,64 @@
|
|
|
15
22
|
import { IMetadataProvider, UserInfo } from '@memberjunction/core';
|
|
16
23
|
import { SearchEngineBase } from '@memberjunction/core-entities';
|
|
17
24
|
import { BaseSingleton } from '@memberjunction/global';
|
|
18
|
-
import { SearchParams, SearchResult } from './search.types.js';
|
|
25
|
+
import { SearchParams, SearchResult, SearchResultItem, SearchStreamEvent } from './search.types.js';
|
|
19
26
|
/**
|
|
20
27
|
* Configuration options for the SearchEngine.
|
|
21
28
|
*/
|
|
22
29
|
export interface SearchEngineConfig {
|
|
23
30
|
/** Default maximum results if not specified in SearchParams (default: 20) */
|
|
24
31
|
DefaultMaxResults?: number;
|
|
32
|
+
/**
|
|
33
|
+
* Default multiplier applied to per-provider `topK` to compensate for residual
|
|
34
|
+
* late permission filtering. Individual calls can override via
|
|
35
|
+
* `SearchParams.PermissionOverfetchFactor`. Default: 2.
|
|
36
|
+
*/
|
|
37
|
+
DefaultPermissionOverfetchFactor?: number;
|
|
25
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* Callback fired the moment an individual provider's `Search()` resolves —
|
|
41
|
+
* before fusion, dedup, permission filtering, or rerank. Used by
|
|
42
|
+
* {@link SearchEngine.streamSearch} to emit `provider` events as each
|
|
43
|
+
* provider returns rather than waiting for the whole pipeline. The callback
|
|
44
|
+
* runs inside the provider's promise chain, so any throw it raises will
|
|
45
|
+
* cancel that provider's contribution but won't take down the search.
|
|
46
|
+
*/
|
|
47
|
+
export type OnProviderResolved = (event: {
|
|
48
|
+
/** Source type as reported by the provider (e.g. 'vector', 'fulltext'). */
|
|
49
|
+
sourceType: string;
|
|
50
|
+
/** Result rows from this provider, with metadata already stamped. */
|
|
51
|
+
results: SearchResultItem[];
|
|
52
|
+
/** Wall-clock time spent inside `Provider.Search()` for this invocation. */
|
|
53
|
+
durationMs: number;
|
|
54
|
+
/** Scope ID when running per-scope; undefined when unconstrained. */
|
|
55
|
+
scopeID?: string;
|
|
56
|
+
}) => void;
|
|
26
57
|
/**
|
|
27
58
|
* Singleton search engine that orchestrates multi-source search with RRF fusion.
|
|
28
59
|
*
|
|
29
60
|
* Providers are discovered from the MJ: Search Providers entity. Each active
|
|
30
61
|
* provider's DriverClass is resolved via ClassFactory to create an instance,
|
|
31
|
-
* which is then initialized with the provider's config from the DB.
|
|
62
|
+
* which is then initialized with the provider's config from the DB record.
|
|
32
63
|
*
|
|
33
64
|
* Usage:
|
|
34
65
|
* ```typescript
|
|
35
66
|
* // Initialize once at server startup
|
|
36
67
|
* await SearchEngine.Instance.Config({}, contextUser);
|
|
37
68
|
*
|
|
38
|
-
* // Execute searches
|
|
69
|
+
* // Execute searches (unscoped — original behavior)
|
|
39
70
|
* const result = await SearchEngine.Instance.Search({
|
|
40
71
|
* Query: 'quarterly revenue',
|
|
41
72
|
* MaxResults: 20,
|
|
42
73
|
* MinScore: 0.1
|
|
43
74
|
* }, contextUser);
|
|
75
|
+
*
|
|
76
|
+
* // Scoped search against two scopes with multi-tenant context
|
|
77
|
+
* const scopedResult = await SearchEngine.Instance.Search({
|
|
78
|
+
* Query: 'refund policy',
|
|
79
|
+
* MaxResults: 20,
|
|
80
|
+
* ScopeIDs: ['hr-scope-id', 'legal-scope-id'],
|
|
81
|
+
* SearchContext: { PrimaryScopeRecordID: 'tenant-a' }
|
|
82
|
+
* }, contextUser);
|
|
44
83
|
* ```
|
|
45
84
|
*/
|
|
46
85
|
export declare class SearchEngine extends BaseSingleton<SearchEngine> {
|
|
@@ -52,6 +91,23 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
|
|
|
52
91
|
private _fusion;
|
|
53
92
|
private _enricher;
|
|
54
93
|
private _defaultMaxResults;
|
|
94
|
+
private _defaultOverfetchFactor;
|
|
95
|
+
/**
|
|
96
|
+
* Minimum trimmed query length we accept. One- and two-character queries against
|
|
97
|
+
* a `LIKE '%term%'` fan-out are essentially full-database scans with negligible
|
|
98
|
+
* relevance — the providers also enforce this, but we short-circuit here to
|
|
99
|
+
* avoid the cache lookup and provider dispatch overhead too.
|
|
100
|
+
*/
|
|
101
|
+
private static readonly MIN_TERM_LENGTH;
|
|
102
|
+
/**
|
|
103
|
+
* Result cache TTL. 30s balances "user resubmits the same prefix" wins against
|
|
104
|
+
* "results stay reasonably fresh after a write". Cache key includes the user's
|
|
105
|
+
* ID so two users with different RLS scopes never share an entry.
|
|
106
|
+
*/
|
|
107
|
+
private static readonly CACHE_TTL_MS;
|
|
108
|
+
/** Maximum cached entries across all users. LRU-evicted on overflow. */
|
|
109
|
+
private static readonly CACHE_MAX_ENTRIES;
|
|
110
|
+
private _cache;
|
|
55
111
|
/** Access the cached provider metadata from SearchEngineBase */
|
|
56
112
|
protected get Base(): SearchEngineBase;
|
|
57
113
|
/** Resolve the metadata provider via SearchEngineBase (which extends BaseEngine and tracks ProviderToUse). */
|
|
@@ -71,19 +127,68 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
|
|
|
71
127
|
/**
|
|
72
128
|
* Execute a multi-source search with RRF fusion and optional enrichment.
|
|
73
129
|
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
* 2. Fuse results with RRF
|
|
77
|
-
* 3. Deduplicate by EntityName+RecordID
|
|
78
|
-
* 4. Exclude redundant entity-sourced Content Items
|
|
79
|
-
* 5. Apply minimum score threshold
|
|
80
|
-
* 6. Enrich with icons, names, and tags (skipped in preview mode)
|
|
130
|
+
* When `params.ScopeIDs` is provided, each scope runs independently and the results
|
|
131
|
+
* are combined via cross-scope RRF before deduplication, re-ranking, and enrichment.
|
|
81
132
|
*
|
|
82
133
|
* @param params - Search parameters
|
|
83
134
|
* @param contextUser - The user performing the search
|
|
84
135
|
* @returns Aggregated search result
|
|
85
136
|
*/
|
|
86
137
|
Search(params: SearchParams, contextUser: UserInfo): Promise<SearchResult>;
|
|
138
|
+
/**
|
|
139
|
+
* Internal search implementation that optionally fires `onProviderResolved`
|
|
140
|
+
* as each provider's promise settles. Exposed via the public {@link Search}
|
|
141
|
+
* (no callback) and {@link streamSearch} (queue-backed callback that
|
|
142
|
+
* yields `provider` events to the caller).
|
|
143
|
+
*/
|
|
144
|
+
private searchInternal;
|
|
145
|
+
/**
|
|
146
|
+
* Streaming variant of {@link Search}. Yields events as each pipeline
|
|
147
|
+
* stage produces output so the caller can emit partials to the UI / agent
|
|
148
|
+
* before fusion + reranking complete.
|
|
149
|
+
*
|
|
150
|
+
* **Phase 2C v1 semantics:** runs the same internal pipeline as
|
|
151
|
+
* {@link Search} and emits synthetic events at each transition. This
|
|
152
|
+
* preserves all existing fusion / permission / dedup / enrich behavior
|
|
153
|
+
* — important because those steps have subtle correctness rules that
|
|
154
|
+
* we don't want to re-implement in a parallel code path. Per-provider
|
|
155
|
+
* partials are reconstructed from the final SourceCounts; a future
|
|
156
|
+
* refactor (Phase 2C v2) can split provider emission to true real-time
|
|
157
|
+
* concurrent emission once we measure that the synthetic phase is the
|
|
158
|
+
* actual bottleneck.
|
|
159
|
+
*
|
|
160
|
+
* Cancellation: the consumer can stop iterating at any point — the
|
|
161
|
+
* underlying Search() will run to completion but its result is
|
|
162
|
+
* discarded. AbortSignal-based mid-pipeline cancellation is a Phase 2C
|
|
163
|
+
* v2 concern.
|
|
164
|
+
*
|
|
165
|
+
* Event ordering:
|
|
166
|
+
* 1. Zero or more `provider` events (one per non-empty source)
|
|
167
|
+
* 2. Exactly one `fused` event
|
|
168
|
+
* 3. Optional one `reranked` event (when a reranker is configured)
|
|
169
|
+
* 4. Exactly one `final` event
|
|
170
|
+
* 5. On error: a single `error` event in place of `final`.
|
|
171
|
+
*
|
|
172
|
+
* @example
|
|
173
|
+
* for await (const ev of SearchEngine.Instance.streamSearch(params, user)) {
|
|
174
|
+
* switch (ev.phase) {
|
|
175
|
+
* case 'provider': scratchpad.append(`${ev.providerName}: ${ev.results.length} hits`); break;
|
|
176
|
+
* case 'final': scratchpad.commit(ev.results); break;
|
|
177
|
+
* case 'error': scratchpad.fail(ev.error); break;
|
|
178
|
+
* }
|
|
179
|
+
* }
|
|
180
|
+
*/
|
|
181
|
+
streamSearch(params: SearchParams, contextUser: UserInfo): AsyncIterable<SearchStreamEvent>;
|
|
182
|
+
/**
|
|
183
|
+
* Build a stable cache key for a search. Includes the user identity so RLS
|
|
184
|
+
* scopes never bleed across users, plus the trimmed query, MaxResults,
|
|
185
|
+
* MinScore, and a deterministic projection of Filters.
|
|
186
|
+
*/
|
|
187
|
+
private buildCacheKey;
|
|
188
|
+
/** Insert into the LRU cache, evicting oldest entries when over capacity. */
|
|
189
|
+
private cachePut;
|
|
190
|
+
/** Test / admin hook: clear the result cache. */
|
|
191
|
+
ClearResultCache(): void;
|
|
87
192
|
/**
|
|
88
193
|
* Quick preview search optimized for autocomplete / typeahead.
|
|
89
194
|
* Uses preview mode (no enrichment), limited to 8 results by default.
|
|
@@ -95,6 +200,36 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
|
|
|
95
200
|
* @returns Search result in preview mode
|
|
96
201
|
*/
|
|
97
202
|
PreviewSearch(query: string, maxResults: number, contextUser: UserInfo): Promise<SearchResult>;
|
|
203
|
+
/**
|
|
204
|
+
* Load `ScopeBundle`s for each requested scope ID, filtering out inactive / expired.
|
|
205
|
+
* Returns an empty array when no scope IDs are supplied (caller treats as Global).
|
|
206
|
+
*/
|
|
207
|
+
private resolveScopes;
|
|
208
|
+
/**
|
|
209
|
+
* Execute all scoped providers for a single scope bundle and return per-scope fused results.
|
|
210
|
+
*/
|
|
211
|
+
private executeScopeBundle;
|
|
212
|
+
/**
|
|
213
|
+
* Assemble a `ScopeConstraints` for a single scope: Nunjucks-render each template
|
|
214
|
+
* field against the `SearchContext`, then hand the rendered values to providers.
|
|
215
|
+
*/
|
|
216
|
+
private buildScopeConstraints;
|
|
217
|
+
/** Resolve the EntityID → EntityName via MJ Metadata (for passing to providers that key by name). */
|
|
218
|
+
private lookupEntityName;
|
|
219
|
+
/**
|
|
220
|
+
* Pick a re-ranker config for this search. When multiple scopes are in play, we
|
|
221
|
+
* use the first scope's config (matching task 1B.17: the re-rank stage is one
|
|
222
|
+
* call applied AFTER cross-scope fusion). A future enhancement could merge
|
|
223
|
+
* per-scope re-rankers, but the current plan keeps it simple.
|
|
224
|
+
*/
|
|
225
|
+
private pickReRankerConfig;
|
|
226
|
+
/**
|
|
227
|
+
* Pick the first scope's `RerankerBudgetCents` value to apply to the reranker
|
|
228
|
+
* run. Mirrors `pickReRankerConfig` — the leading scope's policy wins. NULL
|
|
229
|
+
* (uncapped) is the default when no scope sets a budget.
|
|
230
|
+
*/
|
|
231
|
+
private pickRerankerBudgetCents;
|
|
232
|
+
private runReRanker;
|
|
98
233
|
/**
|
|
99
234
|
* Instantiate a single provider from its SearchProvider metadata record,
|
|
100
235
|
* initialize it, check availability, and add to the active list if available.
|
|
@@ -112,13 +247,18 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
|
|
|
112
247
|
/**
|
|
113
248
|
* Filter search results by entity-level and row-level security permissions.
|
|
114
249
|
*
|
|
250
|
+
* **This is a safety net.** Providers are expected to do per-provider permission
|
|
251
|
+
* push-down (Section 3.6 of plans/search-scopes-rag-plus.md). If this filter is
|
|
252
|
+
* removing more than a handful of results in practice, the responsible provider's
|
|
253
|
+
* push-down is incomplete and should be fixed.
|
|
254
|
+
*
|
|
115
255
|
* Groups results by entity for efficient permission checking:
|
|
116
256
|
* 1. Unknown entities are excluded (fail closed).
|
|
117
257
|
* 2. If the user lacks entity-level CanRead, all results for that entity are dropped.
|
|
118
258
|
* 3. If the user is exempt from RLS, all results pass through.
|
|
119
259
|
* 4. If RLS applies, a RunView validates which record IDs the user can read.
|
|
120
260
|
*/
|
|
121
|
-
|
|
261
|
+
protected filterByPermissions(results: SearchResultItem[], contextUser: UserInfo): Promise<SearchResultItem[]>;
|
|
122
262
|
/**
|
|
123
263
|
* Group search result items by EntityName for batch permission checking.
|
|
124
264
|
*/
|
|
@@ -135,8 +275,36 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
|
|
|
135
275
|
*/
|
|
136
276
|
private filterByRowLevelSecurity;
|
|
137
277
|
/** Build an error SearchResult */
|
|
278
|
+
/**
|
|
279
|
+
* Public hook for callers (e.g. the GraphQL resolver) to emit a
|
|
280
|
+
* Status='Forbidden' SearchExecutionLog row when they reject a request
|
|
281
|
+
* before delegating to {@link Search}. Without this, forbidden invocations
|
|
282
|
+
* never reach the analytics dashboard — exactly the signal admins need
|
|
283
|
+
* to spot users / agents trying to access scopes they shouldn't.
|
|
284
|
+
*/
|
|
285
|
+
LogForbiddenSearch(input: {
|
|
286
|
+
Query: string;
|
|
287
|
+
ScopeIDs?: string[];
|
|
288
|
+
FailureReason: string;
|
|
289
|
+
StartTime: number;
|
|
290
|
+
ContextUser: UserInfo;
|
|
291
|
+
AIAgentID?: string | null;
|
|
292
|
+
}): Promise<void>;
|
|
293
|
+
/**
|
|
294
|
+
* Best-effort hook (P3.2) that writes one MJSearchExecutionLog row per
|
|
295
|
+
* SearchEngine.Search call. Captures query, timing, scope, result count,
|
|
296
|
+
* reranker info, status, and a per-source-count breakdown for the analytics
|
|
297
|
+
* dashboard (P3.3) and tuning CSV export (P3.4).
|
|
298
|
+
*
|
|
299
|
+
* Errors during the write are swallowed and logged — observability is the
|
|
300
|
+
* point of this hook, not a load-bearing dependency. A logger that brings
|
|
301
|
+
* down search would be the worst possible outcome.
|
|
302
|
+
*/
|
|
303
|
+
private logSearchExecution;
|
|
138
304
|
private buildErrorResult;
|
|
139
305
|
/** Build the list of active provider metadata for the response */
|
|
140
306
|
private buildProviderInfoList;
|
|
307
|
+
/** Defensive JSON parse that never throws. Returns `null` on any failure. */
|
|
308
|
+
private parseJson;
|
|
141
309
|
}
|
|
142
310
|
//# sourceMappingURL=SearchEngine.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SearchEngine.d.ts","sourceRoot":"","sources":["../../src/generic/SearchEngine.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"SearchEngine.d.ts","sourceRoot":"","sources":["../../src/generic/SearchEngine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAoC,iBAAiB,EAA0C,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC7I,OAAO,EACH,gBAAgB,EAKnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,aAAa,EAAuC,MAAM,wBAAwB,CAAC;AAC5F,OAAO,EACH,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EASpB,MAAM,gBAAgB,CAAC;AAaxB;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,6EAA6E;IAC7E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,gCAAgC,CAAC,EAAE,MAAM,CAAC;CAC7C;AA4BD;;;;;;;GAOG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACrC,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,IAAI,CAAC;AAEX;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,YAAa,SAAQ,aAAa,CAAC,YAAY,CAAC;;IAMzD,wDAAwD;IACxD,WAAkB,QAAQ,IAAI,YAAY,CAEzC;IAED,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,SAAS,CAAwB;IACzC,OAAO,CAAC,kBAAkB,CAAM;IAChC,OAAO,CAAC,uBAAuB,CAAK;IAEpC;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAK;IAE5C;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAU;IAE9C,wEAAwE;IACxE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAO;IAEhD,OAAO,CAAC,MAAM,CAAqE;IAEnF,gEAAgE;IAChE,SAAS,KAAK,IAAI,IAAI,gBAAgB,CAErC;IAED,8GAA8G;IAC9G,SAAS,KAAK,aAAa,IAAI,iBAAiB,CAE/C;IAED;;;;;;;;;;OAUG;IACU,MAAM,CACf,MAAM,EAAE,kBAAuB,EAC/B,WAAW,EAAE,QAAQ,EACrB,YAAY,GAAE,OAAe,GAC9B,OAAO,CAAC,IAAI,CAAC;IAkChB;;;;;;;;;OASG;IACU,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;IAIvF;;;;;OAKG;YACW,cAAc;IAwO5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACW,YAAY,CACtB,MAAM,EAAE,YAAY,EACpB,WAAW,EAAE,QAAQ,GACtB,aAAa,CAAC,iBAAiB,CAAC;IA8GnC;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAWrB,6EAA6E;IAC7E,OAAO,CAAC,QAAQ;IAShB,iDAAiD;IAC1C,gBAAgB,IAAI,IAAI;IAI/B;;;;;;;;;OASG;IACU,aAAa,CACtB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAU,EACtB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,YAAY,CAAC;IAYxB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;OAEG;YACW,kBAAkB;IAmHhC;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IA+C7B,qGAAqG;IACrG,OAAO,CAAC,gBAAgB;IAaxB;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAS1B;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;YAQjB,WAAW;IAgDzB;;;OAGG;YACW,kBAAkB;IA6EhC;;;OAGG;YACW,gBAAgB;IAgE9B;;OAEG;IACH,OAAO,CAAC,YAAY;IAyBpB;;;;;;;;;;;;;OAaG;cACa,mBAAmB,CAC/B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA6B9B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAa5B;;;;OAIG;YACW,mBAAmB;IAiDjC;;;OAGG;YACW,wBAAwB;IA2CtC,kCAAkC;IAClC;;;;;;OAMG;IACU,kBAAkB,CAAC,KAAK,EAAE;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,QAAQ,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBjB;;;;;;;;;OASG;YACW,kBAAkB;IAoDhC,OAAO,CAAC,gBAAgB;IAYxB,kEAAkE;IAClE,OAAO,CAAC,qBAAqB;IAW7B,6EAA6E;IAC7E,OAAO,CAAC,SAAS;CAUpB"}
|