opencode-search 1.0.0-next.0 → 1.0.0-next.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -24
- package/REMOVED.md +13 -3
- package/dist/package.json +20 -22
- package/dist/search.js +1 -1
- package/dist/src/providers/gnews-search.d.ts +1 -1
- package/dist/src/providers/gnews-search.d.ts.map +1 -1
- package/dist/src/providers/gnews-search.js +5 -3
- package/dist/src/providers/gnews-search.js.map +1 -1
- package/dist/src/providers/rfc-search.d.ts +9 -0
- package/dist/src/providers/rfc-search.d.ts.map +1 -0
- package/dist/src/providers/rfc-search.js +38 -0
- package/dist/src/providers/rfc-search.js.map +1 -0
- package/dist/src/providers/w3c-search.d.ts +6 -0
- package/dist/src/providers/w3c-search.d.ts.map +1 -0
- package/dist/src/providers/w3c-search.js +42 -0
- package/dist/src/providers/w3c-search.js.map +1 -0
- package/dist/src/providers/whatwg-search.d.ts +8 -0
- package/dist/src/providers/whatwg-search.d.ts.map +1 -0
- package/dist/src/providers/whatwg-search.js +31 -0
- package/dist/src/providers/whatwg-search.js.map +1 -0
- package/dist/src/renderers.d.ts +6 -2
- package/dist/src/renderers.d.ts.map +1 -1
- package/dist/src/renderers.js +14 -8
- package/dist/src/renderers.js.map +1 -1
- package/dist/src/search.d.ts +3 -1
- package/dist/src/search.d.ts.map +1 -1
- package/dist/src/search.js +123 -61
- package/dist/src/search.js.map +1 -1
- package/package.json +20 -22
- package/search.ts +22 -0
- package/src/cache.test.ts +14 -0
- package/src/cache.ts +61 -0
- package/src/format.test.ts +175 -0
- package/src/format.ts +72 -0
- package/src/html.ts +39 -0
- package/src/normalize-blurb.test.ts +67 -0
- package/src/normalize-blurb.ts +22 -0
- package/src/opencode/notify.ts +16 -0
- package/src/parse-results.test.ts +75 -0
- package/src/parse-results.ts +22 -0
- package/src/providers/brave-search.ts +39 -0
- package/src/providers/bsky-search.test.ts +16 -0
- package/src/providers/bsky-search.ts +88 -0
- package/src/providers/ddg-search.test.ts +15 -0
- package/src/providers/ddg-search.ts +117 -0
- package/src/providers/gnews-search.ts +72 -0
- package/src/providers/mdn-search.test.ts +17 -0
- package/src/providers/mdn-search.ts +83 -0
- package/src/providers/rfc-search.test.ts +31 -0
- package/src/providers/rfc-search.ts +61 -0
- package/src/providers/w3c-search.test.ts +22 -0
- package/src/providers/w3c-search.ts +67 -0
- package/src/providers/whatwg-search.test.ts +21 -0
- package/src/providers/whatwg-search.ts +55 -0
- package/src/providers/wiki-search.test.ts +16 -0
- package/src/providers/wiki-search.ts +72 -0
- package/src/renderers.test.ts +263 -0
- package/src/renderers.ts +62 -0
- package/src/safe.test.ts +87 -0
- package/src/safe.ts +27 -0
- package/src/search.ts +369 -0
- package/src/strategies.test.ts +92 -0
- package/src/strategies.ts +62 -0
- package/src/user-agent.ts +3 -0
- package/src/v1-message.test.ts +17 -0
- package/src/v1-message.ts +10 -0
- package/dist/src/providers/standard-search.d.ts +0 -18
- package/dist/src/providers/standard-search.d.ts.map +0 -1
- package/dist/src/providers/standard-search.js +0 -61
- package/dist/src/providers/standard-search.js.map +0 -1
package/src/search.ts
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
import type { Plugin } from '@opencode/plugin'
|
|
2
|
+
import type * as Tool from '@opencode/plugin/promise/tool'
|
|
3
|
+
import { formatResults } from './format.ts'
|
|
4
|
+
import { sendResult } from './opencode/notify.ts'
|
|
5
|
+
import { braveSearch } from './providers/brave-search.ts'
|
|
6
|
+
import { bskySearch } from './providers/bsky-search.ts'
|
|
7
|
+
import { ddgSearch } from './providers/ddg-search.ts'
|
|
8
|
+
import { gnewsSearch } from './providers/gnews-search.ts'
|
|
9
|
+
import { mdnSearch } from './providers/mdn-search.ts'
|
|
10
|
+
import { rfcSearch } from './providers/rfc-search.ts'
|
|
11
|
+
import { w3cSearch } from './providers/w3c-search.ts'
|
|
12
|
+
import { whatwgSearch } from './providers/whatwg-search.ts'
|
|
13
|
+
import { wikiSearch } from './providers/wiki-search.ts'
|
|
14
|
+
import {
|
|
15
|
+
renderBskyPost,
|
|
16
|
+
renderDdgResult,
|
|
17
|
+
renderGnewsItem,
|
|
18
|
+
renderMdnDoc,
|
|
19
|
+
renderRfc,
|
|
20
|
+
renderW3c,
|
|
21
|
+
renderWhatwg,
|
|
22
|
+
renderWebResult,
|
|
23
|
+
renderWikiPage,
|
|
24
|
+
} from './renderers.ts'
|
|
25
|
+
import { safeAsync } from './safe.ts'
|
|
26
|
+
|
|
27
|
+
const postResult = async (context: Plugin.Context, ctx: Tool.ToolContext, text: string) => {
|
|
28
|
+
await safeAsync(() => sendResult({ context, sessionID: ctx.sessionID, text }))
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// No `output` field: declaring one without a matching output schema in the tool definition makes
|
|
32
|
+
// OpenCode reject the result.
|
|
33
|
+
const textOutput = (text: string): Tool.Result => ({
|
|
34
|
+
content: [{ type: 'text', text }],
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
const isRecord = (value: unknown): value is Record<string, unknown> => (
|
|
38
|
+
typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
const stringField = (input: unknown, key: string): string => {
|
|
42
|
+
if (!isRecord(input)) return ''
|
|
43
|
+
|
|
44
|
+
const value = input[key]
|
|
45
|
+
return typeof value === 'string' ? value : ''
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const numberField = (input: unknown, key: string): number | undefined => {
|
|
49
|
+
if (!isRecord(input)) return undefined
|
|
50
|
+
|
|
51
|
+
const value = input[key]
|
|
52
|
+
return typeof value === 'number' ? value : undefined
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const queryArg = {
|
|
56
|
+
type: 'object',
|
|
57
|
+
properties: {
|
|
58
|
+
query: { type: 'string', description: 'The search query' },
|
|
59
|
+
},
|
|
60
|
+
required: ['query'],
|
|
61
|
+
additionalProperties: false,
|
|
62
|
+
} as const
|
|
63
|
+
|
|
64
|
+
export const createDdgSearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
65
|
+
name: 'ddg_search',
|
|
66
|
+
description: 'Search DuckDuckGo and return results as extracted text content',
|
|
67
|
+
input: queryArg,
|
|
68
|
+
async execute(input, ctx) {
|
|
69
|
+
const query = stringField(input, 'query')
|
|
70
|
+
|
|
71
|
+
const { data, error } = await safeAsync(() => ddgSearch(query))
|
|
72
|
+
if (error) return textOutput('DuckDuckGo search failed: ' + error.message)
|
|
73
|
+
|
|
74
|
+
const formatted = formatResults({
|
|
75
|
+
label: 'DuckDuckGo results',
|
|
76
|
+
items: data,
|
|
77
|
+
total: data.length,
|
|
78
|
+
renderItem: renderDdgResult,
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
await postResult(context, ctx, formatted)
|
|
82
|
+
|
|
83
|
+
return textOutput(formatted)
|
|
84
|
+
},
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
export const createBraveSearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
88
|
+
name: 'brave_search',
|
|
89
|
+
description: 'Search the web via Brave Search, returning titles, URLs, and snippets.',
|
|
90
|
+
input: queryArg,
|
|
91
|
+
async execute(input, ctx) {
|
|
92
|
+
const query = stringField(input, 'query')
|
|
93
|
+
|
|
94
|
+
const { data, error } = await safeAsync(() => braveSearch(query))
|
|
95
|
+
if (error) return textOutput('Brave search failed: ' + error.message)
|
|
96
|
+
|
|
97
|
+
const formatted = formatResults({
|
|
98
|
+
label: 'Brave results',
|
|
99
|
+
items: data,
|
|
100
|
+
total: data.length,
|
|
101
|
+
renderItem: renderWebResult,
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
await postResult(context, ctx, formatted)
|
|
105
|
+
|
|
106
|
+
return textOutput(formatted)
|
|
107
|
+
},
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
export const createBskySearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
111
|
+
name: 'bsky_search',
|
|
112
|
+
description: [
|
|
113
|
+
'Search Bluesky posts via the AT Protocol.',
|
|
114
|
+
'Returns posts with author, text, and engagement counts.',
|
|
115
|
+
].join(' '),
|
|
116
|
+
input: {
|
|
117
|
+
type: 'object',
|
|
118
|
+
properties: {
|
|
119
|
+
query: { type: 'string', description: 'The search query' },
|
|
120
|
+
limit: { type: 'number', description: 'Maximum number of results to return' },
|
|
121
|
+
sort: { type: 'string', description: 'Sort order: top or latest' },
|
|
122
|
+
},
|
|
123
|
+
required: ['query'],
|
|
124
|
+
additionalProperties: false,
|
|
125
|
+
},
|
|
126
|
+
async execute(input, ctx) {
|
|
127
|
+
const query = stringField(input, 'query')
|
|
128
|
+
const limit = numberField(input, 'limit')
|
|
129
|
+
const sort = stringField(input, 'sort') || undefined
|
|
130
|
+
|
|
131
|
+
const { data, error } = await safeAsync(() => (
|
|
132
|
+
bskySearch({ query, limit, sort })
|
|
133
|
+
))
|
|
134
|
+
|
|
135
|
+
if (error) return textOutput('Bluesky search failed: ' + error.message)
|
|
136
|
+
|
|
137
|
+
const formatted = formatResults({
|
|
138
|
+
label: 'Bluesky results',
|
|
139
|
+
items: data.posts,
|
|
140
|
+
total: data.hitsTotal,
|
|
141
|
+
limit,
|
|
142
|
+
offset: 0,
|
|
143
|
+
renderItem: renderBskyPost,
|
|
144
|
+
})
|
|
145
|
+
|
|
146
|
+
await postResult(context, ctx, formatted)
|
|
147
|
+
|
|
148
|
+
return textOutput(formatted)
|
|
149
|
+
},
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
export const createWikiSearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
153
|
+
name: 'wiki_search',
|
|
154
|
+
description: 'Search Wikipedia articles. Returns page titles, descriptions, and excerpts.',
|
|
155
|
+
input: {
|
|
156
|
+
type: 'object',
|
|
157
|
+
properties: {
|
|
158
|
+
query: { type: 'string', description: 'The search query' },
|
|
159
|
+
limit: { type: 'number', description: 'Maximum number of results to return (1-100)' },
|
|
160
|
+
},
|
|
161
|
+
required: ['query'],
|
|
162
|
+
additionalProperties: false,
|
|
163
|
+
},
|
|
164
|
+
async execute(input, ctx) {
|
|
165
|
+
const query = stringField(input, 'query')
|
|
166
|
+
const limit = numberField(input, 'limit')
|
|
167
|
+
|
|
168
|
+
const { data, error } = await safeAsync(() => (
|
|
169
|
+
wikiSearch({ query, limit })
|
|
170
|
+
))
|
|
171
|
+
|
|
172
|
+
if (error) return textOutput('Wikipedia search failed: ' + error.message)
|
|
173
|
+
|
|
174
|
+
const formatted = formatResults({
|
|
175
|
+
label: 'Wikipedia results',
|
|
176
|
+
items: data.pages,
|
|
177
|
+
total: data.pages.length,
|
|
178
|
+
limit,
|
|
179
|
+
offset: 0,
|
|
180
|
+
renderItem: renderWikiPage,
|
|
181
|
+
})
|
|
182
|
+
|
|
183
|
+
await postResult(context, ctx, formatted)
|
|
184
|
+
|
|
185
|
+
return textOutput(formatted)
|
|
186
|
+
},
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
export const createGnewsSearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
190
|
+
name: 'gnews_search',
|
|
191
|
+
description: [
|
|
192
|
+
'Search Google News for recent articles, returning headlines, publishers, and dates.',
|
|
193
|
+
'News and tech media only — not general web search, and not documentation.',
|
|
194
|
+
'Links are news.google.com redirect URLs, not publisher URLs.',
|
|
195
|
+
].join(' '),
|
|
196
|
+
input: {
|
|
197
|
+
type: 'object',
|
|
198
|
+
properties: {
|
|
199
|
+
query: { type: 'string', description: 'The search query' },
|
|
200
|
+
limit: { type: 'number', description: 'Maximum number of results to return (default 20)' },
|
|
201
|
+
},
|
|
202
|
+
required: ['query'],
|
|
203
|
+
additionalProperties: false,
|
|
204
|
+
},
|
|
205
|
+
async execute(input, ctx) {
|
|
206
|
+
const query = stringField(input, 'query')
|
|
207
|
+
const limit = numberField(input, 'limit')
|
|
208
|
+
|
|
209
|
+
const { data, error } = await safeAsync(() => gnewsSearch(query, limit))
|
|
210
|
+
if (error) return textOutput('Google News search failed: ' + error.message)
|
|
211
|
+
|
|
212
|
+
const formatted = formatResults({
|
|
213
|
+
label: 'Google News results',
|
|
214
|
+
items: data,
|
|
215
|
+
total: data.length,
|
|
216
|
+
renderItem: renderGnewsItem,
|
|
217
|
+
})
|
|
218
|
+
|
|
219
|
+
await postResult(context, ctx, formatted)
|
|
220
|
+
|
|
221
|
+
return textOutput(formatted)
|
|
222
|
+
},
|
|
223
|
+
})
|
|
224
|
+
|
|
225
|
+
export const createMdnSearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
226
|
+
name: 'mdn_search',
|
|
227
|
+
description: 'Search MDN Web Docs. Returns documentation pages for web technologies.',
|
|
228
|
+
input: {
|
|
229
|
+
type: 'object',
|
|
230
|
+
properties: {
|
|
231
|
+
query: { type: 'string', description: 'The search query' },
|
|
232
|
+
limit: { type: 'number', description: 'Maximum number of results to return' },
|
|
233
|
+
page: { type: 'number', description: 'Page number for pagination' },
|
|
234
|
+
},
|
|
235
|
+
required: ['query'],
|
|
236
|
+
additionalProperties: false,
|
|
237
|
+
},
|
|
238
|
+
async execute(input, ctx) {
|
|
239
|
+
const query = stringField(input, 'query')
|
|
240
|
+
const limit = numberField(input, 'limit')
|
|
241
|
+
const page = numberField(input, 'page')
|
|
242
|
+
|
|
243
|
+
const { data, error } = await safeAsync(() => (
|
|
244
|
+
mdnSearch({ query, limit, page })
|
|
245
|
+
))
|
|
246
|
+
|
|
247
|
+
if (error) return textOutput('MDN search failed: ' + error.message)
|
|
248
|
+
|
|
249
|
+
const formatted = formatResults({
|
|
250
|
+
label: 'MDN Web Docs results',
|
|
251
|
+
items: data.documents,
|
|
252
|
+
total: data.total,
|
|
253
|
+
limit,
|
|
254
|
+
renderItem: renderMdnDoc,
|
|
255
|
+
})
|
|
256
|
+
|
|
257
|
+
await postResult(context, ctx, formatted)
|
|
258
|
+
|
|
259
|
+
return textOutput(formatted)
|
|
260
|
+
},
|
|
261
|
+
})
|
|
262
|
+
|
|
263
|
+
export const createRfcSearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
264
|
+
name: 'rfc_search',
|
|
265
|
+
description: 'Search IETF RFCs by title. Returns RFC number, title, date, and abstract.',
|
|
266
|
+
input: {
|
|
267
|
+
type: 'object',
|
|
268
|
+
properties: {
|
|
269
|
+
query: { type: 'string', description: 'The search query' },
|
|
270
|
+
limit: { type: 'number', description: 'Maximum number of results to return (default 20)' },
|
|
271
|
+
},
|
|
272
|
+
required: ['query'],
|
|
273
|
+
additionalProperties: false,
|
|
274
|
+
},
|
|
275
|
+
async execute(input, ctx) {
|
|
276
|
+
const query = stringField(input, 'query')
|
|
277
|
+
const limit = numberField(input, 'limit')
|
|
278
|
+
|
|
279
|
+
const { data, error } = await safeAsync(() => rfcSearch(query, limit))
|
|
280
|
+
if (error) return textOutput('RFC search failed: ' + error.message)
|
|
281
|
+
|
|
282
|
+
const formatted = formatResults({
|
|
283
|
+
label: 'RFC results',
|
|
284
|
+
items: data,
|
|
285
|
+
total: data.length,
|
|
286
|
+
renderItem: renderRfc,
|
|
287
|
+
})
|
|
288
|
+
|
|
289
|
+
await postResult(context, ctx, formatted)
|
|
290
|
+
|
|
291
|
+
return textOutput(formatted)
|
|
292
|
+
},
|
|
293
|
+
})
|
|
294
|
+
|
|
295
|
+
export const createW3cSearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
296
|
+
name: 'w3c_search',
|
|
297
|
+
description: 'Search W3C specifications by title. Returns the spec title and its URL.',
|
|
298
|
+
input: {
|
|
299
|
+
type: 'object',
|
|
300
|
+
properties: {
|
|
301
|
+
query: { type: 'string', description: 'The search query' },
|
|
302
|
+
limit: { type: 'number', description: 'Maximum number of results to return (default 20)' },
|
|
303
|
+
},
|
|
304
|
+
required: ['query'],
|
|
305
|
+
additionalProperties: false,
|
|
306
|
+
},
|
|
307
|
+
async execute(input, ctx) {
|
|
308
|
+
const query = stringField(input, 'query')
|
|
309
|
+
const limit = numberField(input, 'limit')
|
|
310
|
+
|
|
311
|
+
const { data, error } = await safeAsync(() => w3cSearch(query, limit))
|
|
312
|
+
if (error) return textOutput('W3C search failed: ' + error.message)
|
|
313
|
+
|
|
314
|
+
const formatted = formatResults({
|
|
315
|
+
label: 'W3C results',
|
|
316
|
+
items: data,
|
|
317
|
+
total: data.length,
|
|
318
|
+
renderItem: renderW3c,
|
|
319
|
+
})
|
|
320
|
+
|
|
321
|
+
await postResult(context, ctx, formatted)
|
|
322
|
+
|
|
323
|
+
return textOutput(formatted)
|
|
324
|
+
},
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
export const createWhatwgSearchTool = (context: Plugin.Context): Tool.Info => ({
|
|
328
|
+
name: 'whatwg_search',
|
|
329
|
+
description: 'Search WHATWG living standards. Returns the standard name, URL, and description.',
|
|
330
|
+
input: {
|
|
331
|
+
type: 'object',
|
|
332
|
+
properties: {
|
|
333
|
+
query: { type: 'string', description: 'The search query' },
|
|
334
|
+
limit: { type: 'number', description: 'Maximum number of results to return (default 20)' },
|
|
335
|
+
},
|
|
336
|
+
required: ['query'],
|
|
337
|
+
additionalProperties: false,
|
|
338
|
+
},
|
|
339
|
+
async execute(input, ctx) {
|
|
340
|
+
const query = stringField(input, 'query')
|
|
341
|
+
const limit = numberField(input, 'limit')
|
|
342
|
+
|
|
343
|
+
const { data, error } = await safeAsync(() => whatwgSearch(query, limit))
|
|
344
|
+
if (error) return textOutput('WHATWG search failed: ' + error.message)
|
|
345
|
+
|
|
346
|
+
const formatted = formatResults({
|
|
347
|
+
label: 'WHATWG results',
|
|
348
|
+
items: data,
|
|
349
|
+
total: data.length,
|
|
350
|
+
renderItem: renderWhatwg,
|
|
351
|
+
})
|
|
352
|
+
|
|
353
|
+
await postResult(context, ctx, formatted)
|
|
354
|
+
|
|
355
|
+
return textOutput(formatted)
|
|
356
|
+
},
|
|
357
|
+
})
|
|
358
|
+
|
|
359
|
+
export const createSearchTools = (context: Plugin.Context): Tool.Info[] => [
|
|
360
|
+
createBraveSearchTool(context),
|
|
361
|
+
createBskySearchTool(context),
|
|
362
|
+
createDdgSearchTool(context),
|
|
363
|
+
createGnewsSearchTool(context),
|
|
364
|
+
createMdnSearchTool(context),
|
|
365
|
+
createRfcSearchTool(context),
|
|
366
|
+
createW3cSearchTool(context),
|
|
367
|
+
createWhatwgSearchTool(context),
|
|
368
|
+
createWikiSearchTool(context),
|
|
369
|
+
]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test'
|
|
2
|
+
import { BlockedError, baseHeaders, detectBlock, nextUserAgent, runStrategies, USER_AGENTS } from './strategies.ts'
|
|
3
|
+
|
|
4
|
+
describe('nextUserAgent', () => {
|
|
5
|
+
it('rotates through the pool', () => {
|
|
6
|
+
const first = nextUserAgent()
|
|
7
|
+
const second = nextUserAgent()
|
|
8
|
+
|
|
9
|
+
expect(USER_AGENTS.includes(first)).toEqual(true)
|
|
10
|
+
expect(USER_AGENTS.includes(second)).toEqual(true)
|
|
11
|
+
expect(first === second).toEqual(false)
|
|
12
|
+
})
|
|
13
|
+
})
|
|
14
|
+
|
|
15
|
+
describe('baseHeaders', () => {
|
|
16
|
+
it('includes the user agent', () => {
|
|
17
|
+
const headers = baseHeaders('test-agent')
|
|
18
|
+
|
|
19
|
+
expect(headers['User-Agent']).toEqual('test-agent')
|
|
20
|
+
expect(typeof headers['Accept']).toEqual('string')
|
|
21
|
+
})
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
describe('detectBlock', () => {
|
|
25
|
+
it('throws BlockedError when a marker is present', () => {
|
|
26
|
+
expect(() => detectBlock('Test', '<div class="anomaly-modal">', ['anomaly-modal'])).toThrow(BlockedError)
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
it('does nothing when no marker is present', () => {
|
|
30
|
+
expect(() => detectBlock('Test', '<div>results</div>', ['anomaly-modal'])).not.toThrow()
|
|
31
|
+
})
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
describe('runStrategies', () => {
|
|
35
|
+
it('returns the first non-empty result', async () => {
|
|
36
|
+
const results = await runStrategies([
|
|
37
|
+
{ name: 'a', run: async () => [] },
|
|
38
|
+
{ name: 'b', run: async () => ['result'] },
|
|
39
|
+
])
|
|
40
|
+
|
|
41
|
+
expect(results).toEqual(['result'])
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
it('falls through when a strategy throws', async () => {
|
|
45
|
+
const results = await runStrategies<string>([{
|
|
46
|
+
name: 'a',
|
|
47
|
+
run: async () => {
|
|
48
|
+
throw new Error('boom')
|
|
49
|
+
},
|
|
50
|
+
}, {
|
|
51
|
+
name: 'b',
|
|
52
|
+
run: async () => ['result'],
|
|
53
|
+
}])
|
|
54
|
+
|
|
55
|
+
expect(results).toEqual(['result'])
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
it('throws the last error when everything fails', async () => {
|
|
59
|
+
const attempt = runStrategies([
|
|
60
|
+
{ name: 'a', run: async () => [] },
|
|
61
|
+
{
|
|
62
|
+
name: 'b',
|
|
63
|
+
run: async () => {
|
|
64
|
+
throw new Error('final failure')
|
|
65
|
+
},
|
|
66
|
+
},
|
|
67
|
+
])
|
|
68
|
+
|
|
69
|
+
await expect(attempt).rejects.toThrow('final failure')
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it('hands each strategy a different user agent', async () => {
|
|
73
|
+
const seen: string[] = []
|
|
74
|
+
|
|
75
|
+
await runStrategies([{
|
|
76
|
+
name: 'a',
|
|
77
|
+
run: async (userAgent) => {
|
|
78
|
+
seen.push(userAgent)
|
|
79
|
+
return []
|
|
80
|
+
},
|
|
81
|
+
}, {
|
|
82
|
+
name: 'b',
|
|
83
|
+
run: async (userAgent) => {
|
|
84
|
+
seen.push(userAgent)
|
|
85
|
+
return ['done']
|
|
86
|
+
},
|
|
87
|
+
}])
|
|
88
|
+
|
|
89
|
+
expect(seen.length).toEqual(2)
|
|
90
|
+
expect(seen[0] === seen[1]).toEqual(false)
|
|
91
|
+
})
|
|
92
|
+
})
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { safeAsync } from './safe.ts'
|
|
2
|
+
|
|
3
|
+
export const USER_AGENTS = [
|
|
4
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
5
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
|
|
6
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15',
|
|
7
|
+
'Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0',
|
|
8
|
+
]
|
|
9
|
+
|
|
10
|
+
let uaIndex = 0
|
|
11
|
+
|
|
12
|
+
// Rotation gives each fallback attempt a fresh identity after a block
|
|
13
|
+
export const nextUserAgent = (): string => {
|
|
14
|
+
const agent = USER_AGENTS[uaIndex % USER_AGENTS.length]
|
|
15
|
+
uaIndex += 1
|
|
16
|
+
|
|
17
|
+
return agent ?? ''
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export const baseHeaders = (userAgent: string): Record<string, string> => ({
|
|
21
|
+
'User-Agent': userAgent,
|
|
22
|
+
'Accept': 'text/html',
|
|
23
|
+
'Accept-Encoding': 'gzip',
|
|
24
|
+
'DNT': '1',
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
export class BlockedError extends Error {
|
|
28
|
+
constructor(source: string) {
|
|
29
|
+
super(source + ' blocked the request (CAPTCHA or rate limit)')
|
|
30
|
+
this.name = 'BlockedError'
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export const detectBlock = (source: string, html: string, markers: string[]): void => {
|
|
35
|
+
for (const marker of markers) {
|
|
36
|
+
if (html.includes(marker)) throw new BlockedError(source)
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export type Strategy<T> = {
|
|
41
|
+
name: string
|
|
42
|
+
run: (userAgent: string) => Promise<T[]>
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// A strategy fails by throwing or by returning zero results; the next one gets a chance
|
|
46
|
+
export const runStrategies = async <T>(strategies: Strategy<T>[]): Promise<T[]> => {
|
|
47
|
+
let lastError: Error = new Error('no strategies provided')
|
|
48
|
+
|
|
49
|
+
for (const strategy of strategies) {
|
|
50
|
+
const { data, error } = await safeAsync(() => strategy.run(nextUserAgent()))
|
|
51
|
+
if (error) {
|
|
52
|
+
lastError = error
|
|
53
|
+
continue
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (data.length > 0) return data
|
|
57
|
+
|
|
58
|
+
lastError = new Error(strategy.name + ' returned no results')
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
throw lastError
|
|
62
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test'
|
|
2
|
+
import { V1_MESSAGE } from './v1-message.ts'
|
|
3
|
+
|
|
4
|
+
describe('V1_MESSAGE', () => {
|
|
5
|
+
it('names the version to pin', () => {
|
|
6
|
+
expect(V1_MESSAGE).toContain('opencode-search@0.0.9')
|
|
7
|
+
})
|
|
8
|
+
|
|
9
|
+
it('offers the upgrade path as well as the pin', () => {
|
|
10
|
+
expect(V1_MESSAGE).toContain('OpenCode 2')
|
|
11
|
+
expect(V1_MESSAGE).toContain('https://opencode.ai/v2/docs')
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
it('says the v1 release is final, so pinning is not a temporary workaround', () => {
|
|
15
|
+
expect(V1_MESSAGE).toContain('will not be updated')
|
|
16
|
+
})
|
|
17
|
+
})
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// Shown when OpenCode 1 loads this build, where the alternative is a module resolution error.
|
|
2
|
+
export const V1_MESSAGE = [
|
|
3
|
+
'opencode-search 1.x requires OpenCode 2 and cannot load under OpenCode 1.',
|
|
4
|
+
'',
|
|
5
|
+
'Either pin the last OpenCode 1 release, which will not be updated:',
|
|
6
|
+
'',
|
|
7
|
+
' { "plugin": ["opencode-search@0.0.9"] }',
|
|
8
|
+
'',
|
|
9
|
+
'or upgrade to OpenCode 2 (https://opencode.ai/v2/docs) and use the "plugins" key.',
|
|
10
|
+
].join('\n')
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
export type StandardDocument = {
|
|
2
|
-
title: string;
|
|
3
|
-
url: string;
|
|
4
|
-
date: string;
|
|
5
|
-
snippet: string;
|
|
6
|
-
};
|
|
7
|
-
export type StandardSearchResult = {
|
|
8
|
-
documents: Array<StandardDocument>;
|
|
9
|
-
totalResults: string;
|
|
10
|
-
};
|
|
11
|
-
type StandardSearchOptions = {
|
|
12
|
-
query: string;
|
|
13
|
-
limit?: number;
|
|
14
|
-
offset?: number;
|
|
15
|
-
};
|
|
16
|
-
export declare const standardSearch: (options: StandardSearchOptions) => Promise<StandardSearchResult>;
|
|
17
|
-
export {};
|
|
18
|
-
//# sourceMappingURL=standard-search.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"standard-search.d.ts","sourceRoot":"","sources":["../../../src/providers/standard-search.ts"],"names":[],"mappings":"AAKA,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,SAAS,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAA;IAClC,YAAY,EAAE,MAAM,CAAA;CACrB,CAAA;AA0BD,KAAK,qBAAqB,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,eAAO,MAAM,cAAc,GAAU,SAAS,qBAAqB,KAAG,OAAO,CAAC,oBAAoB,CA8CjG,CAAA"}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { JSDOM, VirtualConsole } from 'jsdom';
|
|
2
|
-
import * as cache from "../cache.js";
|
|
3
|
-
const SEARCH_URL = 'https://standard-search.octet-stream.net/search';
|
|
4
|
-
const parseDocument = (el) => {
|
|
5
|
-
const titleEl = el.querySelector('.result-title a');
|
|
6
|
-
if (!titleEl || !titleEl.textContent)
|
|
7
|
-
return null;
|
|
8
|
-
const title = titleEl.textContent.trim();
|
|
9
|
-
const href = titleEl.getAttribute('href');
|
|
10
|
-
const urlEl = el.querySelector('.result-url');
|
|
11
|
-
const urlText = urlEl && urlEl.textContent ? urlEl.textContent : '';
|
|
12
|
-
const snippetEl = el.querySelector('.result-snippet');
|
|
13
|
-
const snippet = snippetEl && snippetEl.textContent ? snippetEl.textContent : '';
|
|
14
|
-
const parts = urlText.split(' \u00B7 ');
|
|
15
|
-
const datePart = parts.length > 1 ? parts[1] : '';
|
|
16
|
-
return {
|
|
17
|
-
title,
|
|
18
|
-
url: href ? href.trim() : '',
|
|
19
|
-
date: datePart ? datePart.trim() : '',
|
|
20
|
-
snippet: snippet.trim(),
|
|
21
|
-
};
|
|
22
|
-
};
|
|
23
|
-
export const standardSearch = async (options) => {
|
|
24
|
-
const cacheKey = ['standard', options.query, options.limit ?? 0, options.offset ?? 0].join(':');
|
|
25
|
-
const cached = await cache.get(cacheKey);
|
|
26
|
-
if (cached && cached.documents && cached.documents.length > 0)
|
|
27
|
-
return cached;
|
|
28
|
-
const url = SEARCH_URL + '?q=' + encodeURIComponent(options.query);
|
|
29
|
-
const res = await fetch(url);
|
|
30
|
-
if (!res.ok)
|
|
31
|
-
throw new Error('Standard Search error (' + res.status + '): ' + res.statusText);
|
|
32
|
-
const html = await res.text();
|
|
33
|
-
const virtualConsole = new VirtualConsole();
|
|
34
|
-
virtualConsole.on('jsdomError', () => { });
|
|
35
|
-
const dom = new JSDOM(html, { url, virtualConsole });
|
|
36
|
-
const doc = dom.window.document;
|
|
37
|
-
const countEl = doc.querySelector('.result-count');
|
|
38
|
-
const countText = countEl ? countEl.textContent : '';
|
|
39
|
-
const totalResults = countText ? countText.trim() : '0';
|
|
40
|
-
const elements = doc.querySelectorAll('.result');
|
|
41
|
-
const documents = [];
|
|
42
|
-
for (let i = 0; i < elements.length; i++) {
|
|
43
|
-
const el = elements[i];
|
|
44
|
-
if (!el)
|
|
45
|
-
continue;
|
|
46
|
-
const parsed = parseDocument(el);
|
|
47
|
-
if (parsed)
|
|
48
|
-
documents.push(parsed);
|
|
49
|
-
}
|
|
50
|
-
const start = options.offset ?? 0;
|
|
51
|
-
const end = options.limit ? start + options.limit : documents.length;
|
|
52
|
-
const sliced = documents.slice(start, end);
|
|
53
|
-
const result = {
|
|
54
|
-
documents: sliced,
|
|
55
|
-
totalResults,
|
|
56
|
-
};
|
|
57
|
-
if (sliced.length > 0)
|
|
58
|
-
await cache.set(cacheKey, result);
|
|
59
|
-
return result;
|
|
60
|
-
};
|
|
61
|
-
//# sourceMappingURL=standard-search.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"standard-search.js","sourceRoot":"","sources":["../../../src/providers/standard-search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAA;AAC7C,OAAO,KAAK,KAAK,MAAM,aAAa,CAAA;AAEpC,MAAM,UAAU,GAAG,iDAAiD,CAAA;AAcpE,MAAM,aAAa,GAAG,CAAC,EAAW,EAA2B,EAAE;IAC7D,MAAM,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAA;IACnD,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW;QAAE,OAAO,IAAI,CAAA;IAEjD,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAA;IACxC,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;IAEzC,MAAM,KAAK,GAAG,EAAE,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;IAC7C,MAAM,OAAO,GAAG,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAA;IAEnE,MAAM,SAAS,GAAG,EAAE,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAA;IACrD,MAAM,OAAO,GAAG,SAAS,IAAI,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAA;IAE/E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;IACvC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAEjD,OAAO;QACL,KAAK;QACL,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;QAC5B,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;QACrC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;KACxB,CAAA;AACH,CAAC,CAAA;AAQD,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EAAE,OAA8B,EAAiC,EAAE;IACpG,MAAM,QAAQ,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAE/F,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAuB,QAAQ,CAAC,CAAA;IAC9D,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,MAAM,CAAA;IAE5E,MAAM,GAAG,GAAG,UAAU,GAAG,KAAK,GAAG,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAClE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,GAAG,CAAC,MAAM,GAAG,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,CAAA;IAE7F,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAE7B,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAA;IAC3C,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAEzC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAA;IACpD,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAA;IAE/B,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,eAAe,CAAC,CAAA;IAClD,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAA;IACpD,MAAM,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;IAEvD,MAAM,QAAQ,GAAG,GAAG,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAChD,MAAM,SAAS,GAA4B,EAAE,CAAA;IAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;QACtB,IAAI,CAAC,EAAE;YAAE,SAAQ;QAEjB,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAA;QAChC,IAAI,MAAM;YAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAA;IACjC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAA;IACpE,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAE1C,MAAM,MAAM,GAAyB;QACnC,SAAS,EAAE,MAAM;QACjB,YAAY;KACb,CAAA;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAExD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA"}
|