@memberjunction/web-search-engine 0.0.1-seed.2 → 6.2.0-edge.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +183 -0
- package/README.md +343 -3
- package/dist/BaseWebSearchProvider.d.ts +89 -0
- package/dist/BaseWebSearchProvider.d.ts.map +1 -0
- package/dist/BaseWebSearchProvider.js +130 -0
- package/dist/BaseWebSearchProvider.js.map +1 -0
- package/dist/WebSearchEngine.d.ts +107 -0
- package/dist/WebSearchEngine.d.ts.map +1 -0
- package/dist/WebSearchEngine.js +396 -0
- package/dist/WebSearchEngine.js.map +1 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +43 -0
- package/dist/index.js.map +1 -0
- package/dist/operations/WebSearchQueryOperation.d.ts +8 -0
- package/dist/operations/WebSearchQueryOperation.d.ts.map +1 -0
- package/dist/operations/WebSearchQueryOperation.js +72 -0
- package/dist/operations/WebSearchQueryOperation.js.map +1 -0
- package/dist/providers/BraveWebSearchProvider.d.ts +41 -0
- package/dist/providers/BraveWebSearchProvider.d.ts.map +1 -0
- package/dist/providers/BraveWebSearchProvider.js +169 -0
- package/dist/providers/BraveWebSearchProvider.js.map +1 -0
- package/dist/providers/DuckDuckGoWebSearchProvider.d.ts +45 -0
- package/dist/providers/DuckDuckGoWebSearchProvider.d.ts.map +1 -0
- package/dist/providers/DuckDuckGoWebSearchProvider.js +116 -0
- package/dist/providers/DuckDuckGoWebSearchProvider.js.map +1 -0
- package/dist/providers/GoogleCustomSearchWebSearchProvider.d.ts +36 -0
- package/dist/providers/GoogleCustomSearchWebSearchProvider.d.ts.map +1 -0
- package/dist/providers/GoogleCustomSearchWebSearchProvider.js +125 -0
- package/dist/providers/GoogleCustomSearchWebSearchProvider.js.map +1 -0
- package/dist/providers/PerplexityWebSearchProvider.d.ts +40 -0
- package/dist/providers/PerplexityWebSearchProvider.d.ts.map +1 -0
- package/dist/providers/PerplexityWebSearchProvider.js +148 -0
- package/dist/providers/PerplexityWebSearchProvider.js.map +1 -0
- package/dist/providers/TavilyWebSearchProvider.d.ts +30 -0
- package/dist/providers/TavilyWebSearchProvider.d.ts.map +1 -0
- package/dist/providers/TavilyWebSearchProvider.js +126 -0
- package/dist/providers/TavilyWebSearchProvider.js.map +1 -0
- package/dist/providers/httpFailure.d.ts +29 -0
- package/dist/providers/httpFailure.d.ts.map +1 -0
- package/dist/providers/httpFailure.js +136 -0
- package/dist/providers/httpFailure.js.map +1 -0
- package/dist/types.d.ts +214 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +17 -0
- package/dist/types.js.map +1 -0
- package/package.json +28 -5
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Shared HTTP failure classification for web search drivers.
|
|
3
|
+
* @module @memberjunction/web-search-engine
|
|
4
|
+
*/
|
|
5
|
+
import { IsHttpError } from '@memberjunction/network-utils';
|
|
6
|
+
/**
|
|
7
|
+
* Phrases a vendor uses when the *credential* was rejected, matched against the response body.
|
|
8
|
+
*
|
|
9
|
+
* Lowercased, and compared with `String.includes` rather than a regular expression — this package
|
|
10
|
+
* deliberately runs no regex over remote input, and that property is worth more than the brevity.
|
|
11
|
+
*
|
|
12
|
+
* Kept specific enough not to fire on a genuine query rejection: "api key" alone would match a
|
|
13
|
+
* search *for* the phrase, so each entry carries the verb that makes it a credential verdict.
|
|
14
|
+
*/
|
|
15
|
+
const CREDENTIAL_REJECTION_MARKERS = [
|
|
16
|
+
// Google Custom Search: HTTP 400 with reason `keyInvalid`
|
|
17
|
+
'api key not valid',
|
|
18
|
+
'keyinvalid',
|
|
19
|
+
'keyexpired',
|
|
20
|
+
'api key expired',
|
|
21
|
+
// Widely used phrasings across the other vendors
|
|
22
|
+
'invalid api key',
|
|
23
|
+
'api key is invalid',
|
|
24
|
+
'invalid_api_key',
|
|
25
|
+
'api_key_invalid',
|
|
26
|
+
'missing api key',
|
|
27
|
+
'no api key',
|
|
28
|
+
'invalid authentication',
|
|
29
|
+
'unauthorized',
|
|
30
|
+
];
|
|
31
|
+
/**
|
|
32
|
+
* Whether a 400-class body is really an authentication verdict rather than a bad query.
|
|
33
|
+
*
|
|
34
|
+
* Both the extracted detail and the raw serialized body are searched, because the marker is not
|
|
35
|
+
* always in the field {@link describeErrorBody} picks: Google puts `"message": "API key not
|
|
36
|
+
* valid…"` at `error.message` but the machine-readable `"reason": "keyInvalid"` inside
|
|
37
|
+
* `error.errors[]`, which the extractor never reaches.
|
|
38
|
+
*/
|
|
39
|
+
function isCredentialRejection(detail, error) {
|
|
40
|
+
const haystack = `${detail} ${serializeBody(error.Data)}`.toLowerCase();
|
|
41
|
+
return CREDENTIAL_REJECTION_MARKERS.some((marker) => haystack.includes(marker));
|
|
42
|
+
}
|
|
43
|
+
/** Flatten a response body to searchable text, bounded so a large body cannot dominate the scan. */
|
|
44
|
+
function serializeBody(data) {
|
|
45
|
+
if (typeof data === 'string') {
|
|
46
|
+
return data.slice(0, 2000);
|
|
47
|
+
}
|
|
48
|
+
if (data && typeof data === 'object') {
|
|
49
|
+
try {
|
|
50
|
+
return JSON.stringify(data).slice(0, 2000);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
// Circular or otherwise unserialisable — the extracted detail still carries a verdict.
|
|
54
|
+
return '';
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return '';
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Classify an error thrown by a vendor call into the kind that drives engine failover.
|
|
61
|
+
*
|
|
62
|
+
* The distinction is about **the request, not the vendor**: a 400/422 normally means this query is
|
|
63
|
+
* malformed and every other provider will reject it too, so the engine must stop rather than
|
|
64
|
+
* spend four more paid calls learning the same thing. Everything else — auth, quota, 5xx,
|
|
65
|
+
* timeouts — is specific to this vendor and another may well succeed.
|
|
66
|
+
*
|
|
67
|
+
* A missing or invalid API key is deliberately `transient` in that sense: the request is fine,
|
|
68
|
+
* *this* provider just cannot serve it, so moving on is exactly right.
|
|
69
|
+
*
|
|
70
|
+
* **Status alone does not decide that**, which is why a 400 is inspected before it is trusted.
|
|
71
|
+
* Google Custom Search answers an invalid key with HTTP 400 (`reason: keyInvalid`), not 401 — so
|
|
72
|
+
* classifying by status alone made a dead Google key stop the whole run, on precisely the vendor
|
|
73
|
+
* whose keys are expected to stop working when the API is discontinued on 2027-01-01. Perplexity
|
|
74
|
+
* answers the same condition with 401 and failed over correctly, which is what made the
|
|
75
|
+
* inconsistency easy to miss.
|
|
76
|
+
*/
|
|
77
|
+
export function classifyHttpFailure(error, vendor) {
|
|
78
|
+
// Status 0 is HttpError's "the request never produced a response" — a timeout or network
|
|
79
|
+
// failure, which is transient by definition.
|
|
80
|
+
if (IsHttpError(error) && error.Status > 0) {
|
|
81
|
+
const status = error.Status;
|
|
82
|
+
const detail = describeErrorBody(error) || error.message;
|
|
83
|
+
if (status === 400 || status === 422) {
|
|
84
|
+
// An auth verdict dressed as a bad request: another provider may well serve this query.
|
|
85
|
+
if (isCredentialRejection(detail, error)) {
|
|
86
|
+
return failure('transient', `${vendor} rejected the API key (HTTP ${status}): ${detail}`);
|
|
87
|
+
}
|
|
88
|
+
return failure('permanent', `${vendor} rejected the request (HTTP ${status}): ${detail}`);
|
|
89
|
+
}
|
|
90
|
+
if (status === 401 || status === 403) {
|
|
91
|
+
return failure('transient', `${vendor} rejected the API key (HTTP ${status}): ${detail}`);
|
|
92
|
+
}
|
|
93
|
+
if (status === 429) {
|
|
94
|
+
return failure('transient', `${vendor} rate limit or quota exceeded: ${detail}`);
|
|
95
|
+
}
|
|
96
|
+
return failure('transient', `${vendor} API error (HTTP ${status}): ${detail}`);
|
|
97
|
+
}
|
|
98
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
99
|
+
return failure('transient', `${vendor} request failed: ${message}`);
|
|
100
|
+
}
|
|
101
|
+
/** Build a failed provider response. */
|
|
102
|
+
export function failure(kind, message) {
|
|
103
|
+
return { Success: false, Hits: [], FailureKind: kind, ErrorMessage: message };
|
|
104
|
+
}
|
|
105
|
+
/** Pull whatever explanation an error body carries, without assuming a shape. */
|
|
106
|
+
function describeErrorBody(error) {
|
|
107
|
+
const data = error.Data;
|
|
108
|
+
if (typeof data === 'string') {
|
|
109
|
+
return data.slice(0, 500);
|
|
110
|
+
}
|
|
111
|
+
if (data && typeof data === 'object') {
|
|
112
|
+
const record = data;
|
|
113
|
+
for (const key of ['detail', 'message', 'error']) {
|
|
114
|
+
const value = record[key];
|
|
115
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
116
|
+
return value;
|
|
117
|
+
}
|
|
118
|
+
// Several vendors nest as { error: { message | detail } }.
|
|
119
|
+
if (value && typeof value === 'object') {
|
|
120
|
+
const nested = value;
|
|
121
|
+
for (const nestedKey of ['message', 'detail', 'code']) {
|
|
122
|
+
const nestedValue = nested[nestedKey];
|
|
123
|
+
if (typeof nestedValue === 'string' && nestedValue.length > 0) {
|
|
124
|
+
return nestedValue;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return '';
|
|
131
|
+
}
|
|
132
|
+
/** Map a relative freshness window onto a vendor's own code table. */
|
|
133
|
+
export function mapFreshnessWindow(window, codes) {
|
|
134
|
+
return codes[window];
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=httpFailure.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"httpFailure.js","sourceRoot":"","sources":["../../src/providers/httpFailure.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAa,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAGvE;;;;;;;;GAQG;AACH,MAAM,4BAA4B,GAAsB;IACpD,0DAA0D;IAC1D,mBAAmB;IACnB,YAAY;IACZ,YAAY;IACZ,iBAAiB;IACjB,iDAAiD;IACjD,iBAAiB;IACjB,oBAAoB;IACpB,iBAAiB;IACjB,iBAAiB;IACjB,iBAAiB;IACjB,YAAY;IACZ,wBAAwB;IACxB,cAAc;CACjB,CAAC;AAEF;;;;;;;GAOG;AACH,SAAS,qBAAqB,CAAC,MAAc,EAAE,KAAgB;IAC3D,MAAM,QAAQ,GAAG,GAAG,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;IACxE,OAAO,4BAA4B,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AACpF,CAAC;AAED,oGAAoG;AACpG,SAAS,aAAa,CAAC,IAAa;IAChC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACL,uFAAuF;YACvF,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IACD,OAAO,EAAE,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAc,EAAE,MAAc;IAC9D,yFAAyF;IACzF,6CAA6C;IAC7C,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5B,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC;QAEzD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnC,wFAAwF;YACxF,IAAI,qBAAqB,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;gBACvC,OAAO,OAAO,CACV,WAAW,EACX,GAAG,MAAM,+BAA+B,MAAM,MAAM,MAAM,EAAE,CAC/D,CAAC;YACN,CAAC;YACD,OAAO,OAAO,CAAC,WAAW,EAAE,GAAG,MAAM,+BAA+B,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;QAC9F,CAAC;QACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnC,OAAO,OAAO,CAAC,WAAW,EAAE,GAAG,MAAM,+BAA+B,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;QAC9F,CAAC;QACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACjB,OAAO,OAAO,CAAC,WAAW,EAAE,GAAG,MAAM,kCAAkC,MAAM,EAAE,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,OAAO,CAAC,WAAW,EAAE,GAAG,MAAM,oBAAoB,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,OAAO,CAAC,WAAW,EAAE,GAAG,MAAM,oBAAoB,OAAO,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,wCAAwC;AACxC,MAAM,UAAU,OAAO,CAAC,IAA0B,EAAE,OAAe;IAC/D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AAClF,CAAC;AAED,iFAAiF;AACjF,SAAS,iBAAiB,CAAC,KAAgB;IACvC,MAAM,IAAI,GAAY,KAAK,CAAC,IAAI,CAAC;IACjC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,IAA+B,CAAC;QAC/C,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChD,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,2DAA2D;YAC3D,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,KAAgC,CAAC;gBAChD,KAAK,MAAM,SAAS,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;oBACpD,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;oBACtC,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5D,OAAO,WAAW,CAAC;oBACvB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,EAAE,CAAC;AACd,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,kBAAkB,CAC9B,MAAyC,EACzC,KAAmD;IAEnD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC;AACzB,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Public types for the MemberJunction web search engine.
|
|
3
|
+
*
|
|
4
|
+
* Web search is deliberately a **separate concern** from `@memberjunction/search-engine`.
|
|
5
|
+
* That engine searches content MJ owns — entity records, vectors, full-text indexes, storage
|
|
6
|
+
* files — and its `SearchResultItem` requires `EntityName` and `RecordID`, the primary key of a
|
|
7
|
+
* source record. A web result has a URL and none of those. Fitting one in would mean lying in
|
|
8
|
+
* the type and dragging the hit through enrichment built to decorate entity records.
|
|
9
|
+
*
|
|
10
|
+
* What the two DO share is the plugin *pattern*: a provider table carrying `DriverClass`,
|
|
11
|
+
* `Priority`, `Status` and `CredentialID`; ClassFactory resolution; and the
|
|
12
|
+
* `Initialize()` → `CheckAvailability()` → `IsAvailable()` lifecycle.
|
|
13
|
+
*
|
|
14
|
+
* @module @memberjunction/web-search-engine
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* One result from a web search provider, normalised across vendors.
|
|
18
|
+
*
|
|
19
|
+
* The first three fields are the contract every provider must satisfy, and they are
|
|
20
|
+
* deliberately the same shape the retiring `Google Custom Search` action returns — which is
|
|
21
|
+
* what makes migrating off it a rename rather than a rewrite.
|
|
22
|
+
*/
|
|
23
|
+
export interface WebSearchHit {
|
|
24
|
+
/** Page title as the provider reports it. */
|
|
25
|
+
Title: string;
|
|
26
|
+
/** Absolute URL of the result. */
|
|
27
|
+
URL: string;
|
|
28
|
+
/** Snippet or extracted content. Length and style vary by provider. */
|
|
29
|
+
Snippet: string;
|
|
30
|
+
/** Host as the provider displays it, e.g. `irs.gov`. Absent when the provider omits it. */
|
|
31
|
+
DisplayURL?: string;
|
|
32
|
+
/** Publication or last-modified date, when the provider resolved one. */
|
|
33
|
+
PublishedAt?: Date;
|
|
34
|
+
/**
|
|
35
|
+
* The provider's own relevance score for this hit.
|
|
36
|
+
*
|
|
37
|
+
* **Provider-relative and not comparable across providers** — Brave, Tavily and Perplexity
|
|
38
|
+
* each compute it differently and none is a calibrated probability. Use it to order hits
|
|
39
|
+
* within one response; never threshold on it, and never compare it between providers.
|
|
40
|
+
*/
|
|
41
|
+
Score?: number;
|
|
42
|
+
/**
|
|
43
|
+
* Provider-specific extras that do not fit the normalised shape — Brave's extra snippets,
|
|
44
|
+
* Tavily's raw page content, Perplexity's last-updated stamp.
|
|
45
|
+
*
|
|
46
|
+
* This exists so normalisation does not become a lowest-common-denominator: a caller that
|
|
47
|
+
* knows which provider answered (via {@link WebSearchResult.ProviderUsed}) can reach for
|
|
48
|
+
* what makes that provider worth having.
|
|
49
|
+
*/
|
|
50
|
+
ProviderExtras?: Record<string, unknown>;
|
|
51
|
+
}
|
|
52
|
+
/** Relative freshness windows every provider can express in some form. */
|
|
53
|
+
export type WebSearchFreshnessWindow = 'day' | 'week' | 'month' | 'year';
|
|
54
|
+
/** An absolute date range. Providers that cannot express one fall back to their nearest window. */
|
|
55
|
+
export interface WebSearchDateRange {
|
|
56
|
+
From: Date;
|
|
57
|
+
To: Date;
|
|
58
|
+
}
|
|
59
|
+
export type WebSearchFreshness = WebSearchFreshnessWindow | WebSearchDateRange;
|
|
60
|
+
/** Adult-content filtering level. */
|
|
61
|
+
export type WebSearchSafeSearch = 'off' | 'moderate' | 'strict';
|
|
62
|
+
/**
|
|
63
|
+
* What a driver can do. Declared on the **class**, not stored in the database.
|
|
64
|
+
*
|
|
65
|
+
* Capability is a property of the driver implementation; the database row holds *configuration*
|
|
66
|
+
* — on/off, order, credentials, limits. This mirrors how `search-engine` declares `SourceType`
|
|
67
|
+
* on `BaseSearchProvider` rather than in the `SearchProvider` table.
|
|
68
|
+
*/
|
|
69
|
+
export interface WebSearchCapabilities {
|
|
70
|
+
/** Can return a synthesized answer alongside hits (Tavily, Perplexity — not Brave). */
|
|
71
|
+
Answer: boolean;
|
|
72
|
+
/** Honours `IncludeDomains` / `ExcludeDomains`. */
|
|
73
|
+
DomainFilter: boolean;
|
|
74
|
+
/** Honours `Freshness`. */
|
|
75
|
+
Freshness: boolean;
|
|
76
|
+
/** Honours `Country` / `Language`. */
|
|
77
|
+
Region: boolean;
|
|
78
|
+
/** The vendor's hard cap on results per request. Requests above it are clamped, not rejected. */
|
|
79
|
+
MaxResultsCap: number;
|
|
80
|
+
}
|
|
81
|
+
/** Parameters for a web search. Only `Query` is required. */
|
|
82
|
+
export interface WebSearchParams {
|
|
83
|
+
/** The search query. Required, non-empty. */
|
|
84
|
+
Query: string;
|
|
85
|
+
/** Desired result count. Clamped to the serving provider's cap. Default 10. */
|
|
86
|
+
MaxResults?: number;
|
|
87
|
+
/**
|
|
88
|
+
* Pin the search to one provider, by `Name` (case-insensitive) or `DriverClass`.
|
|
89
|
+
*
|
|
90
|
+
* **When set, there is no failover.** If the named provider is missing, inactive,
|
|
91
|
+
* unavailable or incapable of what was asked, the call fails with a specific result code.
|
|
92
|
+
* A silent substitution is how you discover in production that a "semantic research"
|
|
93
|
+
* search has been served by a keyword engine for a month.
|
|
94
|
+
*/
|
|
95
|
+
Provider?: string;
|
|
96
|
+
/** Restrict results to these domains, where the provider supports it. */
|
|
97
|
+
IncludeDomains?: string[];
|
|
98
|
+
/** Exclude these domains, where the provider supports it. */
|
|
99
|
+
ExcludeDomains?: string[];
|
|
100
|
+
/** Restrict by recency. */
|
|
101
|
+
Freshness?: WebSearchFreshness;
|
|
102
|
+
/** Two-letter country code for localisation, e.g. `US`, `GB`. */
|
|
103
|
+
Country?: string;
|
|
104
|
+
/** Language code for results, e.g. `en`. */
|
|
105
|
+
Language?: string;
|
|
106
|
+
/** Adult-content filter. Default `moderate`. */
|
|
107
|
+
SafeSearch?: WebSearchSafeSearch;
|
|
108
|
+
/**
|
|
109
|
+
* Ask for a synthesized answer as well as hits.
|
|
110
|
+
*
|
|
111
|
+
* Restricts selection to providers whose {@link WebSearchCapabilities.Answer} is true — so
|
|
112
|
+
* it changes *which* provider serves the request, not just what comes back.
|
|
113
|
+
*/
|
|
114
|
+
IncludeAnswer?: boolean;
|
|
115
|
+
}
|
|
116
|
+
/** Outcome codes. `SUCCESS` is the only one where {@link WebSearchResult.Success} is true. */
|
|
117
|
+
export type WebSearchResultCode = 'SUCCESS'
|
|
118
|
+
/** `Query` was missing or blank. */
|
|
119
|
+
| 'MISSING_QUERY'
|
|
120
|
+
/** No web search providers exist in metadata at all — a setup problem, not a search failure. */
|
|
121
|
+
| 'NO_PROVIDERS_CONFIGURED'
|
|
122
|
+
/**
|
|
123
|
+
* The provider list could not be READ — the view query failed, the entity is absent from
|
|
124
|
+
* metadata, or the caller lacks permission. Deliberately distinct from
|
|
125
|
+
* `NO_PROVIDERS_CONFIGURED`: that one tells an operator to add a record, which is the wrong
|
|
126
|
+
* instruction (and an actively misleading one) when the read itself never succeeded.
|
|
127
|
+
*/
|
|
128
|
+
| 'PROVIDER_LOAD_FAILED'
|
|
129
|
+
/** Providers exist, but none is Active, available and capable of this request. */
|
|
130
|
+
| 'NO_ELIGIBLE_PROVIDER'
|
|
131
|
+
/** An explicit `Provider` was named and no such provider exists in metadata. */
|
|
132
|
+
| 'PROVIDER_NOT_FOUND'
|
|
133
|
+
/** The named provider exists but its `Status` is not `Active`. */
|
|
134
|
+
| 'PROVIDER_NOT_ACTIVE'
|
|
135
|
+
/** The named provider is Active but not operational — usually a missing credential. */
|
|
136
|
+
| 'PROVIDER_UNAVAILABLE'
|
|
137
|
+
/** The named provider cannot do what was asked, e.g. `IncludeAnswer` on a hits-only provider. */
|
|
138
|
+
| 'PROVIDER_LACKS_CAPABILITY'
|
|
139
|
+
/** Every eligible provider was tried and each failed. See `Attempts`. */
|
|
140
|
+
| 'ALL_PROVIDERS_FAILED'
|
|
141
|
+
/** The request itself was rejected and retrying elsewhere cannot help. */
|
|
142
|
+
| 'INVALID_REQUEST';
|
|
143
|
+
/**
|
|
144
|
+
* Why a provider call failed, which decides whether the engine tries the next provider.
|
|
145
|
+
*
|
|
146
|
+
* - `transient` — rate limit, 5xx, timeout, network. Another provider may well succeed.
|
|
147
|
+
* - `permanent` — the *request* is the problem (query too long, malformed parameters). Every
|
|
148
|
+
* other provider will reject it too, so the engine stops rather than burning paid calls on
|
|
149
|
+
* a request that cannot succeed and burying the real error behind N identical failures.
|
|
150
|
+
*/
|
|
151
|
+
export type WebSearchFailureKind = 'transient' | 'permanent';
|
|
152
|
+
/** One provider's turn, recorded whether it succeeded or not. */
|
|
153
|
+
export interface WebSearchAttempt {
|
|
154
|
+
/** The provider's `Name` from metadata. */
|
|
155
|
+
ProviderName: string;
|
|
156
|
+
/** The provider's `DriverClass`. */
|
|
157
|
+
DriverClass: string;
|
|
158
|
+
Succeeded: boolean;
|
|
159
|
+
/** Wall-clock duration of the provider call. */
|
|
160
|
+
DurationMs: number;
|
|
161
|
+
/** How many hits it returned. Zero is a legitimate answer, not a failure. */
|
|
162
|
+
HitCount?: number;
|
|
163
|
+
FailureKind?: WebSearchFailureKind;
|
|
164
|
+
ErrorMessage?: string;
|
|
165
|
+
}
|
|
166
|
+
/** The result of {@link WebSearchEngine.Search}. */
|
|
167
|
+
export interface WebSearchResult {
|
|
168
|
+
Success: boolean;
|
|
169
|
+
ResultCode: WebSearchResultCode;
|
|
170
|
+
/** Normalised hits. Empty on failure, and legitimately empty on a narrow query. */
|
|
171
|
+
Hits: WebSearchHit[];
|
|
172
|
+
/** Synthesized answer, only when `IncludeAnswer` was requested and the provider produced one. */
|
|
173
|
+
Answer?: string;
|
|
174
|
+
/** `Name` of the provider that actually served the result. Always set when `Success`. */
|
|
175
|
+
ProviderUsed?: string;
|
|
176
|
+
/**
|
|
177
|
+
* Every provider tried, in order.
|
|
178
|
+
*
|
|
179
|
+
* **Populated on the success path too**, deliberately. If the primary provider rate-limits
|
|
180
|
+
* every call and the secondary quietly serves everything, the system looks healthy while
|
|
181
|
+
* the bill moves to a vendor nobody chose. This is what makes that visible.
|
|
182
|
+
*/
|
|
183
|
+
Attempts: WebSearchAttempt[];
|
|
184
|
+
ErrorMessage?: string;
|
|
185
|
+
}
|
|
186
|
+
/** Configuration handed to a driver during {@link BaseWebSearchProvider.Initialize}. */
|
|
187
|
+
export interface WebSearchProviderConfig {
|
|
188
|
+
/** The provider's `Name` from its metadata record. */
|
|
189
|
+
Name: string;
|
|
190
|
+
/** Parsed `ProviderConfig` JSON, or null when the column was empty or unparseable. */
|
|
191
|
+
ProviderConfig: Record<string, unknown> | null;
|
|
192
|
+
/** Decrypted credential values, when the record carried a `CredentialID`. */
|
|
193
|
+
CredentialValues: Record<string, unknown> | null;
|
|
194
|
+
/** Per-provider result cap from metadata, or null to use the driver's own cap. */
|
|
195
|
+
MaxResultsOverride: number | null;
|
|
196
|
+
/** Ordering value from metadata; lower is higher priority. */
|
|
197
|
+
Priority: number;
|
|
198
|
+
}
|
|
199
|
+
/** What a driver returns from {@link BaseWebSearchProvider.ExecuteSearch}. */
|
|
200
|
+
export interface WebSearchProviderResponse {
|
|
201
|
+
Success: boolean;
|
|
202
|
+
Hits: WebSearchHit[];
|
|
203
|
+
Answer?: string;
|
|
204
|
+
/** Required when `Success` is false — decides failover. */
|
|
205
|
+
FailureKind?: WebSearchFailureKind;
|
|
206
|
+
ErrorMessage?: string;
|
|
207
|
+
}
|
|
208
|
+
/** Catalog entry for a registered driver, for admin UI population. */
|
|
209
|
+
export interface RegisteredWebSearchProviderInfo {
|
|
210
|
+
/** ClassFactory registration key — the `WebSearchProvider.DriverClass` column value. */
|
|
211
|
+
DriverClass: string;
|
|
212
|
+
Capabilities: WebSearchCapabilities;
|
|
213
|
+
}
|
|
214
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IACzB,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,uEAAuE;IACvE,OAAO,EAAE,MAAM,CAAC;IAChB,2FAA2F;IAC3F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED,0EAA0E;AAC1E,MAAM,MAAM,wBAAwB,GAAG,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAEzE,mGAAmG;AACnG,MAAM,WAAW,kBAAkB;IAC/B,IAAI,EAAE,IAAI,CAAC;IACX,EAAE,EAAE,IAAI,CAAC;CACZ;AAED,MAAM,MAAM,kBAAkB,GAAG,wBAAwB,GAAG,kBAAkB,CAAC;AAE/E,qCAAqC;AACrC,MAAM,MAAM,mBAAmB,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,CAAC;AAEhE;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IAClC,uFAAuF;IACvF,MAAM,EAAE,OAAO,CAAC;IAChB,mDAAmD;IACnD,YAAY,EAAE,OAAO,CAAC;IACtB,2BAA2B;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,sCAAsC;IACtC,MAAM,EAAE,OAAO,CAAC;IAChB,iGAAiG;IACjG,aAAa,EAAE,MAAM,CAAC;CACzB;AAED,6DAA6D;AAC7D,MAAM,WAAW,eAAe;IAC5B,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,+EAA+E;IAC/E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,6DAA6D;IAC7D,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,2BAA2B;IAC3B,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,iEAAiE;IACjE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,UAAU,CAAC,EAAE,mBAAmB,CAAC;IACjC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,8FAA8F;AAC9F,MAAM,MAAM,mBAAmB,GACzB,SAAS;AACX,oCAAoC;GAClC,eAAe;AACjB,gGAAgG;GAC9F,yBAAyB;AAC3B;;;;;GAKG;GACD,sBAAsB;AACxB,kFAAkF;GAChF,sBAAsB;AACxB,gFAAgF;GAC9E,oBAAoB;AACtB,kEAAkE;GAChE,qBAAqB;AACvB,uFAAuF;GACrF,sBAAsB;AACxB,iGAAiG;GAC/F,2BAA2B;AAC7B,yEAAyE;GACvE,sBAAsB;AACxB,0EAA0E;GACxE,iBAAiB,CAAC;AAExB;;;;;;;GAOG;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,GAAG,WAAW,CAAC;AAE7D,iEAAiE;AACjE,MAAM,WAAW,gBAAgB;IAC7B,2CAA2C;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;IACnB,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,oBAAoB,CAAC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,oDAAoD;AACpD,MAAM,WAAW,eAAe;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,mBAAmB,CAAC;IAChC,mFAAmF;IACnF,IAAI,EAAE,YAAY,EAAE,CAAC;IACrB,iGAAiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,wFAAwF;AACxF,MAAM,WAAW,uBAAuB;IACpC,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,sFAAsF;IACtF,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC/C,6EAA6E;IAC7E,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,kFAAkF;IAClF,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,8EAA8E;AAC9E,MAAM,WAAW,yBAAyB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,YAAY,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,WAAW,CAAC,EAAE,oBAAoB,CAAC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,sEAAsE;AACtE,MAAM,WAAW,+BAA+B;IAC5C,wFAAwF;IACxF,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,qBAAqB,CAAC;CACvC"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Public types for the MemberJunction web search engine.
|
|
3
|
+
*
|
|
4
|
+
* Web search is deliberately a **separate concern** from `@memberjunction/search-engine`.
|
|
5
|
+
* That engine searches content MJ owns — entity records, vectors, full-text indexes, storage
|
|
6
|
+
* files — and its `SearchResultItem` requires `EntityName` and `RecordID`, the primary key of a
|
|
7
|
+
* source record. A web result has a URL and none of those. Fitting one in would mean lying in
|
|
8
|
+
* the type and dragging the hit through enrichment built to decorate entity records.
|
|
9
|
+
*
|
|
10
|
+
* What the two DO share is the plugin *pattern*: a provider table carrying `DriverClass`,
|
|
11
|
+
* `Priority`, `Status` and `CredentialID`; ClassFactory resolution; and the
|
|
12
|
+
* `Initialize()` → `CheckAvailability()` → `IsAvailable()` lifecycle.
|
|
13
|
+
*
|
|
14
|
+
* @module @memberjunction/web-search-engine
|
|
15
|
+
*/
|
|
16
|
+
export {};
|
|
17
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG"}
|
package/package.json
CHANGED
|
@@ -1,10 +1,33 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memberjunction/web-search-engine",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "6.2.0-edge.0",
|
|
5
|
+
"description": "MemberJunction: Metadata-driven web search engine — pluggable external search providers with priority failover",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"/dist"
|
|
10
|
+
],
|
|
11
|
+
"author": "MemberJunction.com",
|
|
12
|
+
"license": "BUSL-1.1",
|
|
13
|
+
"dependencies": {
|
|
14
|
+
"@memberjunction/core": "6.2.0-edge.0",
|
|
15
|
+
"@memberjunction/core-entities": "6.2.0-edge.0",
|
|
16
|
+
"@memberjunction/credentials": "6.2.0-edge.0",
|
|
17
|
+
"@memberjunction/global": "6.2.0-edge.0",
|
|
18
|
+
"@memberjunction/network-utils": "6.2.0-edge.0"
|
|
19
|
+
},
|
|
20
|
+
"devDependencies": {
|
|
21
|
+
"@types/node": "24.10.11",
|
|
22
|
+
"typescript": "^5.9.3"
|
|
23
|
+
},
|
|
6
24
|
"repository": {
|
|
7
25
|
"type": "git",
|
|
8
|
-
"url": "
|
|
26
|
+
"url": "https://github.com/MemberJunction/MJ"
|
|
27
|
+
},
|
|
28
|
+
"scripts": {
|
|
29
|
+
"build": "tsc && tsc-alias -f",
|
|
30
|
+
"test": "vitest run",
|
|
31
|
+
"test:watch": "vitest"
|
|
9
32
|
}
|
|
10
|
-
}
|
|
33
|
+
}
|