@open-mercato/web-research-searxng 0.7.1-develop.7194.1.ab4fc81f82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +2 -0
- package/build.mjs +10 -0
- package/dist/adapter.js +128 -0
- package/dist/adapter.js.map +7 -0
- package/dist/index.js +13 -0
- package/dist/index.js.map +7 -0
- package/jest.config.cjs +28 -0
- package/package.json +59 -0
- package/src/__tests__/searxng.test.ts +75 -0
- package/src/adapter.ts +145 -0
- package/src/index.ts +8 -0
- package/tsconfig.json +10 -0
- package/watch.mjs +7 -0
package/build.mjs
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { dirname } from 'node:path'
|
|
2
|
+
import { fileURLToPath } from 'node:url'
|
|
3
|
+
import { buildPackage } from '../../scripts/build-package.mjs'
|
|
4
|
+
|
|
5
|
+
const packageDir = dirname(fileURLToPath(import.meta.url))
|
|
6
|
+
|
|
7
|
+
await buildPackage(packageDir, {
|
|
8
|
+
name: 'web-research-searxng',
|
|
9
|
+
entryPoints: 'src/**/*.ts',
|
|
10
|
+
})
|
package/dist/adapter.js
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CONTRACT_VERSION,
|
|
3
|
+
READY,
|
|
4
|
+
defineAdapterModule,
|
|
5
|
+
notReady,
|
|
6
|
+
searchOk,
|
|
7
|
+
stripTrailingSlashes,
|
|
8
|
+
toOutcomeFailure
|
|
9
|
+
} from "@open-mercato/web-research";
|
|
10
|
+
import { z } from "zod";
|
|
11
|
+
const searxngOptionsSchema = z.object({
|
|
12
|
+
baseUrl: z.url().optional(),
|
|
13
|
+
/** SearXNG engine names to restrict the query to, e.g. ["duckduckgo","brave"]. */
|
|
14
|
+
engines: z.array(z.string().min(1)).optional(),
|
|
15
|
+
language: z.string().min(2).max(16).optional(),
|
|
16
|
+
timeoutMs: z.number().int().min(1e3).max(12e4).optional()
|
|
17
|
+
});
|
|
18
|
+
const DEFAULT_TIMEOUT_MS = 12e3;
|
|
19
|
+
const responseSchema = z.object({
|
|
20
|
+
results: z.array(
|
|
21
|
+
z.object({
|
|
22
|
+
url: z.string().nullish(),
|
|
23
|
+
title: z.string().nullish(),
|
|
24
|
+
content: z.string().nullish(),
|
|
25
|
+
score: z.number().nullish(),
|
|
26
|
+
publishedDate: z.string().nullish()
|
|
27
|
+
})
|
|
28
|
+
).nullish()
|
|
29
|
+
});
|
|
30
|
+
const TIME_RANGE = {
|
|
31
|
+
day: "day",
|
|
32
|
+
week: "week",
|
|
33
|
+
month: "month",
|
|
34
|
+
year: "year"
|
|
35
|
+
};
|
|
36
|
+
function parseJson(body) {
|
|
37
|
+
try {
|
|
38
|
+
return JSON.parse(body);
|
|
39
|
+
} catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function createSearxngAdapter(options) {
|
|
44
|
+
const baseUrl = options.baseUrl ? stripTrailingSlashes(options.baseUrl) : "";
|
|
45
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
46
|
+
return {
|
|
47
|
+
id: "searxng",
|
|
48
|
+
kind: "serp",
|
|
49
|
+
capabilities: {
|
|
50
|
+
search: true,
|
|
51
|
+
fetch: false,
|
|
52
|
+
snippets: true,
|
|
53
|
+
content: false,
|
|
54
|
+
freshness: true,
|
|
55
|
+
siteFilter: true,
|
|
56
|
+
cost: "free"
|
|
57
|
+
},
|
|
58
|
+
/** `free` — pings /healthz on infrastructure the tenant runs itself. */
|
|
59
|
+
probeCost: "free",
|
|
60
|
+
readiness: () => baseUrl.length > 0 ? READY : notReady("SearXNG base URL is not set"),
|
|
61
|
+
async search(request, context) {
|
|
62
|
+
if (baseUrl.length === 0) return { status: "unavailable", reason: "SearXNG base URL is not set" };
|
|
63
|
+
try {
|
|
64
|
+
const url = new URL(`${baseUrl}/search`);
|
|
65
|
+
url.searchParams.set("q", request.site ? `${request.query} site:${request.site}` : request.query);
|
|
66
|
+
url.searchParams.set("format", "json");
|
|
67
|
+
if (options.engines?.length) url.searchParams.set("engines", options.engines.join(","));
|
|
68
|
+
if (options.language ?? request.locale) {
|
|
69
|
+
url.searchParams.set("language", options.language ?? request.locale);
|
|
70
|
+
}
|
|
71
|
+
if (request.freshness && TIME_RANGE[request.freshness]) {
|
|
72
|
+
url.searchParams.set("time_range", TIME_RANGE[request.freshness]);
|
|
73
|
+
}
|
|
74
|
+
const response = await context.http.request(url.toString(), {
|
|
75
|
+
accept: ["application/json"],
|
|
76
|
+
timeoutMs,
|
|
77
|
+
signal: context.signal
|
|
78
|
+
});
|
|
79
|
+
const parsed = responseSchema.safeParse(parseJson(response.body));
|
|
80
|
+
if (!parsed.success) {
|
|
81
|
+
return {
|
|
82
|
+
status: "error",
|
|
83
|
+
reason: "SearXNG returned an unrecognized response shape (is the JSON format enabled?)",
|
|
84
|
+
retriable: false
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
const results = [];
|
|
88
|
+
for (const item of parsed.data.results ?? []) {
|
|
89
|
+
if (!item.url) continue;
|
|
90
|
+
results.push({
|
|
91
|
+
url: item.url,
|
|
92
|
+
title: item.title ?? null,
|
|
93
|
+
snippet: item.content ?? null,
|
|
94
|
+
publishedAt: item.publishedDate ?? null,
|
|
95
|
+
score: item.score ?? null
|
|
96
|
+
});
|
|
97
|
+
if (results.length >= request.limit) break;
|
|
98
|
+
}
|
|
99
|
+
context.report("ok", `SearXNG returned ${results.length} result(s)`, { resultCount: results.length });
|
|
100
|
+
return searchOk(results);
|
|
101
|
+
} catch (error) {
|
|
102
|
+
return toOutcomeFailure(error);
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
async healthCheck(context) {
|
|
106
|
+
if (baseUrl.length === 0) return { ok: false, detail: "SearXNG base URL is not set" };
|
|
107
|
+
try {
|
|
108
|
+
await context.http.request(`${baseUrl}/healthz`, { timeoutMs, signal: context.signal });
|
|
109
|
+
return { ok: true };
|
|
110
|
+
} catch (error) {
|
|
111
|
+
return { ok: false, detail: error instanceof Error ? error.message : String(error) };
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
const searxngAdapterModule = defineAdapterModule({
|
|
117
|
+
id: "searxng",
|
|
118
|
+
kind: "serp",
|
|
119
|
+
contractVersion: CONTRACT_VERSION,
|
|
120
|
+
optionsSchema: searxngOptionsSchema,
|
|
121
|
+
createAdapter: createSearxngAdapter
|
|
122
|
+
});
|
|
123
|
+
export {
|
|
124
|
+
createSearxngAdapter,
|
|
125
|
+
searxngAdapterModule,
|
|
126
|
+
searxngOptionsSchema
|
|
127
|
+
};
|
|
128
|
+
//# sourceMappingURL=adapter.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/adapter.ts"],
|
|
4
|
+
"sourcesContent": ["import {\n CONTRACT_VERSION,\n READY,\n defineAdapterModule,\n notReady,\n searchOk,\n stripTrailingSlashes,\n toOutcomeFailure,\n type RawResult,\n type SearchAdapter,\n type SearchOutcome,\n} from '@open-mercato/web-research'\nimport { z } from 'zod'\n\n/**\n * Client for an operator's OWN SearXNG instance. Only the client ships here \u2014\n * SearXNG is AGPL-3.0, so calling one over HTTP is fine but bundling the server\n * would be distribution.\n */\nexport const searxngOptionsSchema = z.object({\n baseUrl: z.url().optional(),\n /** SearXNG engine names to restrict the query to, e.g. [\"duckduckgo\",\"brave\"]. */\n engines: z.array(z.string().min(1)).optional(),\n language: z.string().min(2).max(16).optional(),\n timeoutMs: z.number().int().min(1000).max(120_000).optional(),\n})\n\nexport type SearxngOptions = z.infer<typeof searxngOptionsSchema>\n\nconst DEFAULT_TIMEOUT_MS = 12_000\n\nconst responseSchema = z.object({\n results: z\n .array(\n z.object({\n url: z.string().nullish(),\n title: z.string().nullish(),\n content: z.string().nullish(),\n score: z.number().nullish(),\n publishedDate: z.string().nullish(),\n }),\n )\n .nullish(),\n})\n\nconst TIME_RANGE: Readonly<Record<string, string>> = {\n day: 'day',\n week: 'week',\n month: 'month',\n year: 'year',\n}\n\nfunction parseJson(body: string): unknown {\n try {\n return JSON.parse(body)\n } catch {\n return null\n }\n}\n\nexport function createSearxngAdapter(options: SearxngOptions): SearchAdapter {\n const baseUrl = options.baseUrl ? stripTrailingSlashes(options.baseUrl) : ''\n const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS\n\n return {\n id: 'searxng',\n kind: 'serp',\n capabilities: {\n search: true,\n fetch: false,\n snippets: true,\n content: false,\n freshness: true,\n siteFilter: true,\n cost: 'free',\n },\n /** `free` \u2014 pings /healthz on infrastructure the tenant runs itself. */\n probeCost: 'free' as const,\n readiness: () => (baseUrl.length > 0 ? READY : notReady('SearXNG base URL is not set')),\n\n async search(request, context): Promise<SearchOutcome> {\n if (baseUrl.length === 0) return { status: 'unavailable', reason: 'SearXNG base URL is not set' }\n try {\n const url = new URL(`${baseUrl}/search`)\n url.searchParams.set('q', request.site ? `${request.query} site:${request.site}` : request.query)\n url.searchParams.set('format', 'json')\n if (options.engines?.length) url.searchParams.set('engines', options.engines.join(','))\n if (options.language ?? request.locale) {\n url.searchParams.set('language', options.language ?? (request.locale as string))\n }\n if (request.freshness && TIME_RANGE[request.freshness]) {\n url.searchParams.set('time_range', TIME_RANGE[request.freshness])\n }\n\n const response = await context.http.request(url.toString(), {\n accept: ['application/json'],\n timeoutMs,\n signal: context.signal,\n })\n const parsed = responseSchema.safeParse(parseJson(response.body))\n if (!parsed.success) {\n return {\n status: 'error',\n reason: 'SearXNG returned an unrecognized response shape (is the JSON format enabled?)',\n retriable: false,\n }\n }\n const results: RawResult[] = []\n for (const item of parsed.data.results ?? []) {\n if (!item.url) continue\n results.push({\n url: item.url,\n title: item.title ?? null,\n snippet: item.content ?? null,\n publishedAt: item.publishedDate ?? null,\n score: item.score ?? null,\n })\n if (results.length >= request.limit) break\n }\n context.report('ok', `SearXNG returned ${results.length} result(s)`, { resultCount: results.length })\n return searchOk(results)\n } catch (error) {\n return toOutcomeFailure(error)\n }\n },\n\n async healthCheck(context) {\n if (baseUrl.length === 0) return { ok: false, detail: 'SearXNG base URL is not set' }\n try {\n await context.http.request(`${baseUrl}/healthz`, { timeoutMs, signal: context.signal })\n return { ok: true }\n } catch (error) {\n return { ok: false, detail: error instanceof Error ? error.message : String(error) }\n }\n },\n }\n}\n\nexport const searxngAdapterModule = defineAdapterModule({\n id: 'searxng',\n kind: 'serp',\n contractVersion: CONTRACT_VERSION,\n optionsSchema: searxngOptionsSchema,\n createAdapter: createSearxngAdapter,\n})\n"],
|
|
5
|
+
"mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAIK;AACP,SAAS,SAAS;AAOX,MAAM,uBAAuB,EAAE,OAAO;AAAA,EAC3C,SAAS,EAAE,IAAI,EAAE,SAAS;AAAA;AAAA,EAE1B,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,EAAE,SAAS;AAAA,EAC7C,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS;AAAA,EAC7C,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,GAAI,EAAE,IAAI,IAAO,EAAE,SAAS;AAC9D,CAAC;AAID,MAAM,qBAAqB;AAE3B,MAAM,iBAAiB,EAAE,OAAO;AAAA,EAC9B,SAAS,EACN;AAAA,IACC,EAAE,OAAO;AAAA,MACP,KAAK,EAAE,OAAO,EAAE,QAAQ;AAAA,MACxB,OAAO,EAAE,OAAO,EAAE,QAAQ;AAAA,MAC1B,SAAS,EAAE,OAAO,EAAE,QAAQ;AAAA,MAC5B,OAAO,EAAE,OAAO,EAAE,QAAQ;AAAA,MAC1B,eAAe,EAAE,OAAO,EAAE,QAAQ;AAAA,IACpC,CAAC;AAAA,EACH,EACC,QAAQ;AACb,CAAC;AAED,MAAM,aAA+C;AAAA,EACnD,KAAK;AAAA,EACL,MAAM;AAAA,EACN,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,UAAU,MAAuB;AACxC,MAAI;AACF,WAAO,KAAK,MAAM,IAAI;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,qBAAqB,SAAwC;AAC3E,QAAM,UAAU,QAAQ,UAAU,qBAAqB,QAAQ,OAAO,IAAI;AAC1E,QAAM,YAAY,QAAQ,aAAa;AAEvC,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,cAAc;AAAA,MACZ,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,UAAU;AAAA,MACV,SAAS;AAAA,MACT,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,MAAM;AAAA,IACR;AAAA;AAAA,IAEA,WAAW;AAAA,IACX,WAAW,MAAO,QAAQ,SAAS,IAAI,QAAQ,SAAS,6BAA6B;AAAA,IAErF,MAAM,OAAO,SAAS,SAAiC;AACrD,UAAI,QAAQ,WAAW,EAAG,QAAO,EAAE,QAAQ,eAAe,QAAQ,8BAA8B;AAChG,UAAI;AACF,cAAM,MAAM,IAAI,IAAI,GAAG,OAAO,SAAS;AACvC,YAAI,aAAa,IAAI,KAAK,QAAQ,OAAO,GAAG,QAAQ,KAAK,SAAS,QAAQ,IAAI,KAAK,QAAQ,KAAK;AAChG,YAAI,aAAa,IAAI,UAAU,MAAM;AACrC,YAAI,QAAQ,SAAS,OAAQ,KAAI,aAAa,IAAI,WAAW,QAAQ,QAAQ,KAAK,GAAG,CAAC;AACtF,YAAI,QAAQ,YAAY,QAAQ,QAAQ;AACtC,cAAI,aAAa,IAAI,YAAY,QAAQ,YAAa,QAAQ,MAAiB;AAAA,QACjF;AACA,YAAI,QAAQ,aAAa,WAAW,QAAQ,SAAS,GAAG;AACtD,cAAI,aAAa,IAAI,cAAc,WAAW,QAAQ,SAAS,CAAC;AAAA,QAClE;AAEA,cAAM,WAAW,MAAM,QAAQ,KAAK,QAAQ,IAAI,SAAS,GAAG;AAAA,UAC1D,QAAQ,CAAC,kBAAkB;AAAA,UAC3B;AAAA,UACA,QAAQ,QAAQ;AAAA,QAClB,CAAC;AACD,cAAM,SAAS,eAAe,UAAU,UAAU,SAAS,IAAI,CAAC;AAChE,YAAI,CAAC,OAAO,SAAS;AACnB,iBAAO;AAAA,YACL,QAAQ;AAAA,YACR,QAAQ;AAAA,YACR,WAAW;AAAA,UACb;AAAA,QACF;AACA,cAAM,UAAuB,CAAC;AAC9B,mBAAW,QAAQ,OAAO,KAAK,WAAW,CAAC,GAAG;AAC5C,cAAI,CAAC,KAAK,IAAK;AACf,kBAAQ,KAAK;AAAA,YACX,KAAK,KAAK;AAAA,YACV,OAAO,KAAK,SAAS;AAAA,YACrB,SAAS,KAAK,WAAW;AAAA,YACzB,aAAa,KAAK,iBAAiB;AAAA,YACnC,OAAO,KAAK,SAAS;AAAA,UACvB,CAAC;AACD,cAAI,QAAQ,UAAU,QAAQ,MAAO;AAAA,QACvC;AACA,gBAAQ,OAAO,MAAM,oBAAoB,QAAQ,MAAM,cAAc,EAAE,aAAa,QAAQ,OAAO,CAAC;AACpG,eAAO,SAAS,OAAO;AAAA,MACzB,SAAS,OAAO;AACd,eAAO,iBAAiB,KAAK;AAAA,MAC/B;AAAA,IACF;AAAA,IAEA,MAAM,YAAY,SAAS;AACzB,UAAI,QAAQ,WAAW,EAAG,QAAO,EAAE,IAAI,OAAO,QAAQ,8BAA8B;AACpF,UAAI;AACF,cAAM,QAAQ,KAAK,QAAQ,GAAG,OAAO,YAAY,EAAE,WAAW,QAAQ,QAAQ,OAAO,CAAC;AACtF,eAAO,EAAE,IAAI,KAAK;AAAA,MACpB,SAAS,OAAO;AACd,eAAO,EAAE,IAAI,OAAO,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;AAAA,MACrF;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,uBAAuB,oBAAoB;AAAA,EACtD,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,iBAAiB;AAAA,EACjB,eAAe;AAAA,EACf,eAAe;AACjB,CAAC;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createSearxngAdapter,
|
|
3
|
+
searxngAdapterModule,
|
|
4
|
+
searxngOptionsSchema
|
|
5
|
+
} from "./adapter.js";
|
|
6
|
+
import { searxngAdapterModule as searxngAdapterModule2 } from "./adapter.js";
|
|
7
|
+
export {
|
|
8
|
+
createSearxngAdapter,
|
|
9
|
+
searxngAdapterModule2 as default,
|
|
10
|
+
searxngAdapterModule,
|
|
11
|
+
searxngOptionsSchema
|
|
12
|
+
};
|
|
13
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/index.ts"],
|
|
4
|
+
"sourcesContent": ["export {\n createSearxngAdapter,\n searxngAdapterModule,\n searxngOptionsSchema,\n type SearxngOptions,\n} from './adapter'\n\nexport { searxngAdapterModule as default } from './adapter'\n"],
|
|
5
|
+
"mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAEP,SAAiC,wBAAxBA,6BAAuC;",
|
|
6
|
+
"names": ["searxngAdapterModule"]
|
|
7
|
+
}
|
package/jest.config.cjs
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** @type {import('jest').Config} */
|
|
2
|
+
const base = require('../../jest.config.base.cjs')
|
|
3
|
+
|
|
4
|
+
module.exports = {
|
|
5
|
+
...base,
|
|
6
|
+
preset: 'ts-jest',
|
|
7
|
+
testEnvironment: 'node',
|
|
8
|
+
watchman: false,
|
|
9
|
+
rootDir: '.',
|
|
10
|
+
moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
|
|
11
|
+
moduleNameMapper: {
|
|
12
|
+
'^@open-mercato/web-research/(.*)$': '<rootDir>/../web-research/src/$1',
|
|
13
|
+
'^@open-mercato/web-research$': '<rootDir>/../web-research/src/index.ts',
|
|
14
|
+
},
|
|
15
|
+
transform: {
|
|
16
|
+
'^.+\\.(t|j)sx?$': [
|
|
17
|
+
'ts-jest',
|
|
18
|
+
{
|
|
19
|
+
tsconfig: {
|
|
20
|
+
rootDir: '.',
|
|
21
|
+
ignoreDeprecations: '6.0',
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
],
|
|
25
|
+
},
|
|
26
|
+
testMatch: ['<rootDir>/src/**/__tests__/**/*.test.(ts|tsx)'],
|
|
27
|
+
passWithNoTests: true,
|
|
28
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@open-mercato/web-research-searxng",
|
|
3
|
+
"version": "0.7.1-develop.7194.1.ab4fc81f82",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"openMercato": {
|
|
8
|
+
"webResearchAdapter": {
|
|
9
|
+
"id": "searxng"
|
|
10
|
+
}
|
|
11
|
+
},
|
|
12
|
+
"scripts": {
|
|
13
|
+
"build": "node build.mjs",
|
|
14
|
+
"watch": "node watch.mjs",
|
|
15
|
+
"test": "jest --config jest.config.cjs",
|
|
16
|
+
"typecheck": "tsc --noEmit"
|
|
17
|
+
},
|
|
18
|
+
"exports": {
|
|
19
|
+
".": {
|
|
20
|
+
"types": "./src/index.ts",
|
|
21
|
+
"default": "./dist/index.js"
|
|
22
|
+
},
|
|
23
|
+
"./*.ts": {
|
|
24
|
+
"types": "./src/*.ts",
|
|
25
|
+
"default": "./dist/*.js"
|
|
26
|
+
},
|
|
27
|
+
"./*": {
|
|
28
|
+
"types": [
|
|
29
|
+
"./src/*.ts"
|
|
30
|
+
],
|
|
31
|
+
"default": "./dist/*.js"
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"peerDependencies": {
|
|
35
|
+
"@open-mercato/web-research": "^0.6.0"
|
|
36
|
+
},
|
|
37
|
+
"dependencies": {
|
|
38
|
+
"zod": "^4.4.3"
|
|
39
|
+
},
|
|
40
|
+
"devDependencies": {
|
|
41
|
+
"@open-mercato/web-research": "0.7.1-develop.7194.1.ab4fc81f82",
|
|
42
|
+
"@types/jest": "^30.0.0",
|
|
43
|
+
"@types/node": "^26.0.1",
|
|
44
|
+
"esbuild": "^0.28.1",
|
|
45
|
+
"glob": "^13.0.6",
|
|
46
|
+
"jest": "^30.4.2",
|
|
47
|
+
"ts-jest": "^29.4.11",
|
|
48
|
+
"typescript": "^6.0.3"
|
|
49
|
+
},
|
|
50
|
+
"publishConfig": {
|
|
51
|
+
"access": "public"
|
|
52
|
+
},
|
|
53
|
+
"repository": {
|
|
54
|
+
"type": "git",
|
|
55
|
+
"url": "https://github.com/open-mercato/open-mercato",
|
|
56
|
+
"directory": "packages/web-research-searxng"
|
|
57
|
+
},
|
|
58
|
+
"stableVersion": "0.7.0"
|
|
59
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { createStubHttpClient, createTestContext, describeAdapterContract } from '@open-mercato/web-research/testing'
|
|
2
|
+
import { createSearxngAdapter, searxngAdapterModule } from '../adapter'
|
|
3
|
+
|
|
4
|
+
const BODY = JSON.stringify({
|
|
5
|
+
results: [
|
|
6
|
+
{ url: 'https://a.com/1', title: 'Alpha', content: 'Alpha snippet', score: 3.2 },
|
|
7
|
+
{ title: 'No url here', content: 'skipped' },
|
|
8
|
+
{ url: 'https://b.com/2', title: 'Beta', content: 'Beta snippet' },
|
|
9
|
+
],
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
const BASE = { baseUrl: 'https://searx.internal' }
|
|
13
|
+
|
|
14
|
+
describe('searxng adapter', () => {
|
|
15
|
+
it('is unavailable without a base URL and makes no call', async () => {
|
|
16
|
+
const http = createStubHttpClient(() => ({ body: BODY }))
|
|
17
|
+
const outcome = await createSearxngAdapter({}).search({ query: 'x', limit: 5 }, createTestContext({ http }))
|
|
18
|
+
expect(outcome).toMatchObject({ status: 'unavailable' })
|
|
19
|
+
expect(http.calls).toHaveLength(0)
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
it('maps results and skips entries without a url', async () => {
|
|
23
|
+
const http = createStubHttpClient(() => ({ body: BODY }))
|
|
24
|
+
const outcome = await createSearxngAdapter(BASE).search(
|
|
25
|
+
{ query: 'x', limit: 5 },
|
|
26
|
+
createTestContext({ http }),
|
|
27
|
+
)
|
|
28
|
+
expect(outcome.status).toBe('ok')
|
|
29
|
+
if (outcome.status !== 'ok') return
|
|
30
|
+
expect(outcome.results.map((result) => result.url)).toEqual(['https://a.com/1', 'https://b.com/2'])
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
it('asks for JSON and passes engines, language and freshness through', async () => {
|
|
34
|
+
const http = createStubHttpClient(() => ({ body: BODY }))
|
|
35
|
+
await createSearxngAdapter({ ...BASE, engines: ['duckduckgo', 'brave'], language: 'en' }).search(
|
|
36
|
+
{ query: 'pricing', limit: 5, site: 'example.com', freshness: 'week' },
|
|
37
|
+
createTestContext({ http }),
|
|
38
|
+
)
|
|
39
|
+
const requested = decodeURIComponent(http.calls[0])
|
|
40
|
+
expect(requested).toContain('format=json')
|
|
41
|
+
expect(requested).toContain('engines=duckduckgo,brave')
|
|
42
|
+
expect(requested).toContain('language=en')
|
|
43
|
+
expect(requested).toContain('time_range=week')
|
|
44
|
+
expect(requested).toContain('site:example.com')
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
it('explains an unparseable payload as a likely JSON-format problem', async () => {
|
|
48
|
+
const http = createStubHttpClient(() => ({ body: '<html>not json</html>' }))
|
|
49
|
+
const outcome = await createSearxngAdapter(BASE).search(
|
|
50
|
+
{ query: 'x', limit: 5 },
|
|
51
|
+
createTestContext({ http }),
|
|
52
|
+
)
|
|
53
|
+
expect(outcome).toMatchObject({ status: 'error' })
|
|
54
|
+
if (outcome.status === 'error') expect(outcome.reason).toContain('JSON format')
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
it('respects the requested limit', async () => {
|
|
58
|
+
const http = createStubHttpClient(() => ({ body: BODY }))
|
|
59
|
+
const outcome = await createSearxngAdapter(BASE).search(
|
|
60
|
+
{ query: 'x', limit: 1 },
|
|
61
|
+
createTestContext({ http }),
|
|
62
|
+
)
|
|
63
|
+
expect(outcome.status === 'ok' && outcome.results).toHaveLength(1)
|
|
64
|
+
})
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
describeAdapterContract(
|
|
68
|
+
searxngAdapterModule,
|
|
69
|
+
{
|
|
70
|
+
unconfiguredOptions: {},
|
|
71
|
+
configuredOptions: BASE,
|
|
72
|
+
context: () => createTestContext({ http: createStubHttpClient(() => ({ body: BODY })) }),
|
|
73
|
+
},
|
|
74
|
+
{ describe, it, expect: expect as never },
|
|
75
|
+
)
|
package/src/adapter.ts
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CONTRACT_VERSION,
|
|
3
|
+
READY,
|
|
4
|
+
defineAdapterModule,
|
|
5
|
+
notReady,
|
|
6
|
+
searchOk,
|
|
7
|
+
stripTrailingSlashes,
|
|
8
|
+
toOutcomeFailure,
|
|
9
|
+
type RawResult,
|
|
10
|
+
type SearchAdapter,
|
|
11
|
+
type SearchOutcome,
|
|
12
|
+
} from '@open-mercato/web-research'
|
|
13
|
+
import { z } from 'zod'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Client for an operator's OWN SearXNG instance. Only the client ships here —
|
|
17
|
+
* SearXNG is AGPL-3.0, so calling one over HTTP is fine but bundling the server
|
|
18
|
+
* would be distribution.
|
|
19
|
+
*/
|
|
20
|
+
export const searxngOptionsSchema = z.object({
|
|
21
|
+
baseUrl: z.url().optional(),
|
|
22
|
+
/** SearXNG engine names to restrict the query to, e.g. ["duckduckgo","brave"]. */
|
|
23
|
+
engines: z.array(z.string().min(1)).optional(),
|
|
24
|
+
language: z.string().min(2).max(16).optional(),
|
|
25
|
+
timeoutMs: z.number().int().min(1000).max(120_000).optional(),
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
export type SearxngOptions = z.infer<typeof searxngOptionsSchema>
|
|
29
|
+
|
|
30
|
+
const DEFAULT_TIMEOUT_MS = 12_000
|
|
31
|
+
|
|
32
|
+
const responseSchema = z.object({
|
|
33
|
+
results: z
|
|
34
|
+
.array(
|
|
35
|
+
z.object({
|
|
36
|
+
url: z.string().nullish(),
|
|
37
|
+
title: z.string().nullish(),
|
|
38
|
+
content: z.string().nullish(),
|
|
39
|
+
score: z.number().nullish(),
|
|
40
|
+
publishedDate: z.string().nullish(),
|
|
41
|
+
}),
|
|
42
|
+
)
|
|
43
|
+
.nullish(),
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
const TIME_RANGE: Readonly<Record<string, string>> = {
|
|
47
|
+
day: 'day',
|
|
48
|
+
week: 'week',
|
|
49
|
+
month: 'month',
|
|
50
|
+
year: 'year',
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function parseJson(body: string): unknown {
|
|
54
|
+
try {
|
|
55
|
+
return JSON.parse(body)
|
|
56
|
+
} catch {
|
|
57
|
+
return null
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function createSearxngAdapter(options: SearxngOptions): SearchAdapter {
|
|
62
|
+
const baseUrl = options.baseUrl ? stripTrailingSlashes(options.baseUrl) : ''
|
|
63
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS
|
|
64
|
+
|
|
65
|
+
return {
|
|
66
|
+
id: 'searxng',
|
|
67
|
+
kind: 'serp',
|
|
68
|
+
capabilities: {
|
|
69
|
+
search: true,
|
|
70
|
+
fetch: false,
|
|
71
|
+
snippets: true,
|
|
72
|
+
content: false,
|
|
73
|
+
freshness: true,
|
|
74
|
+
siteFilter: true,
|
|
75
|
+
cost: 'free',
|
|
76
|
+
},
|
|
77
|
+
/** `free` — pings /healthz on infrastructure the tenant runs itself. */
|
|
78
|
+
probeCost: 'free' as const,
|
|
79
|
+
readiness: () => (baseUrl.length > 0 ? READY : notReady('SearXNG base URL is not set')),
|
|
80
|
+
|
|
81
|
+
async search(request, context): Promise<SearchOutcome> {
|
|
82
|
+
if (baseUrl.length === 0) return { status: 'unavailable', reason: 'SearXNG base URL is not set' }
|
|
83
|
+
try {
|
|
84
|
+
const url = new URL(`${baseUrl}/search`)
|
|
85
|
+
url.searchParams.set('q', request.site ? `${request.query} site:${request.site}` : request.query)
|
|
86
|
+
url.searchParams.set('format', 'json')
|
|
87
|
+
if (options.engines?.length) url.searchParams.set('engines', options.engines.join(','))
|
|
88
|
+
if (options.language ?? request.locale) {
|
|
89
|
+
url.searchParams.set('language', options.language ?? (request.locale as string))
|
|
90
|
+
}
|
|
91
|
+
if (request.freshness && TIME_RANGE[request.freshness]) {
|
|
92
|
+
url.searchParams.set('time_range', TIME_RANGE[request.freshness])
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const response = await context.http.request(url.toString(), {
|
|
96
|
+
accept: ['application/json'],
|
|
97
|
+
timeoutMs,
|
|
98
|
+
signal: context.signal,
|
|
99
|
+
})
|
|
100
|
+
const parsed = responseSchema.safeParse(parseJson(response.body))
|
|
101
|
+
if (!parsed.success) {
|
|
102
|
+
return {
|
|
103
|
+
status: 'error',
|
|
104
|
+
reason: 'SearXNG returned an unrecognized response shape (is the JSON format enabled?)',
|
|
105
|
+
retriable: false,
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const results: RawResult[] = []
|
|
109
|
+
for (const item of parsed.data.results ?? []) {
|
|
110
|
+
if (!item.url) continue
|
|
111
|
+
results.push({
|
|
112
|
+
url: item.url,
|
|
113
|
+
title: item.title ?? null,
|
|
114
|
+
snippet: item.content ?? null,
|
|
115
|
+
publishedAt: item.publishedDate ?? null,
|
|
116
|
+
score: item.score ?? null,
|
|
117
|
+
})
|
|
118
|
+
if (results.length >= request.limit) break
|
|
119
|
+
}
|
|
120
|
+
context.report('ok', `SearXNG returned ${results.length} result(s)`, { resultCount: results.length })
|
|
121
|
+
return searchOk(results)
|
|
122
|
+
} catch (error) {
|
|
123
|
+
return toOutcomeFailure(error)
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
|
|
127
|
+
async healthCheck(context) {
|
|
128
|
+
if (baseUrl.length === 0) return { ok: false, detail: 'SearXNG base URL is not set' }
|
|
129
|
+
try {
|
|
130
|
+
await context.http.request(`${baseUrl}/healthz`, { timeoutMs, signal: context.signal })
|
|
131
|
+
return { ok: true }
|
|
132
|
+
} catch (error) {
|
|
133
|
+
return { ok: false, detail: error instanceof Error ? error.message : String(error) }
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export const searxngAdapterModule = defineAdapterModule({
|
|
140
|
+
id: 'searxng',
|
|
141
|
+
kind: 'serp',
|
|
142
|
+
contractVersion: CONTRACT_VERSION,
|
|
143
|
+
optionsSchema: searxngOptionsSchema,
|
|
144
|
+
createAdapter: createSearxngAdapter,
|
|
145
|
+
})
|
package/src/index.ts
ADDED
package/tsconfig.json
ADDED