opencode-search 1.0.0-next.3 → 1.0.0-next.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +7 -7
  2. package/REMOVED.md +3 -3
  3. package/dist/package.json +4 -2
  4. package/dist/src/providers/gnews-search.d.ts +1 -1
  5. package/dist/src/providers/gnews-search.d.ts.map +1 -1
  6. package/dist/src/providers/gnews-search.js +5 -3
  7. package/dist/src/providers/gnews-search.js.map +1 -1
  8. package/dist/src/search.d.ts.map +1 -1
  9. package/dist/src/search.js +35 -25
  10. package/dist/src/search.js.map +1 -1
  11. package/package.json +4 -2
  12. package/search.ts +22 -0
  13. package/src/cache.test.ts +14 -0
  14. package/src/cache.ts +61 -0
  15. package/src/format.test.ts +175 -0
  16. package/src/format.ts +72 -0
  17. package/src/html.ts +39 -0
  18. package/src/normalize-blurb.test.ts +67 -0
  19. package/src/normalize-blurb.ts +22 -0
  20. package/src/opencode/notify.ts +16 -0
  21. package/src/parse-results.test.ts +75 -0
  22. package/src/parse-results.ts +22 -0
  23. package/src/providers/brave-search.ts +39 -0
  24. package/src/providers/bsky-search.test.ts +16 -0
  25. package/src/providers/bsky-search.ts +88 -0
  26. package/src/providers/ddg-search.test.ts +15 -0
  27. package/src/providers/ddg-search.ts +117 -0
  28. package/src/providers/gnews-search.ts +72 -0
  29. package/src/providers/mdn-search.test.ts +17 -0
  30. package/src/providers/mdn-search.ts +83 -0
  31. package/src/providers/standard-search.test.ts +29 -0
  32. package/src/providers/standard-search.ts +94 -0
  33. package/src/providers/wiki-search.test.ts +16 -0
  34. package/src/providers/wiki-search.ts +72 -0
  35. package/src/renderers.test.ts +327 -0
  36. package/src/renderers.ts +51 -0
  37. package/src/safe.test.ts +87 -0
  38. package/src/safe.ts +27 -0
  39. package/src/search.ts +310 -0
  40. package/src/strategies.test.ts +92 -0
  41. package/src/strategies.ts +62 -0
  42. package/src/user-agent.ts +3 -0
  43. package/src/v1-message.test.ts +17 -0
  44. package/src/v1-message.ts +10 -0
@@ -0,0 +1,117 @@
1
+ import { JSDOM, VirtualConsole } from 'jsdom'
2
+ import * as cache from '../cache.ts'
3
+ import { baseHeaders, detectBlock, runStrategies } from '../strategies.ts'
4
+
5
+ // The lite frontend is the fallback when the html frontend is blocked or empty
6
+ const DDG_HTML_URL = 'https://html.duckduckgo.com/html/'
7
+ const DDG_LITE_URL = 'https://lite.duckduckgo.com/lite/'
8
+
9
+ const BLOCK_MARKERS = ['anomaly-modal', 'Please try again']
10
+
11
+ export type DdgResult = {
12
+ title: string
13
+ url: string
14
+ abstract: string
15
+ }
16
+
17
+ const searchParams = (query: string): URLSearchParams => (
18
+ new URLSearchParams({
19
+ q: query,
20
+ b: '',
21
+ kf: '-1',
22
+ kh: '1',
23
+ kl: 'us-en',
24
+ kp: '1',
25
+ k1: '-1',
26
+ })
27
+ )
28
+
29
+ const parseDom = (html: string, url: string): Document => {
30
+ const virtualConsole = new VirtualConsole()
31
+ virtualConsole.on('jsdomError', () => {
32
+ // Silently ignore JSDOM errors
33
+ })
34
+
35
+ return new JSDOM(html, { url, virtualConsole }).window.document
36
+ }
37
+
38
+ const fetchDdg = async (url: string, query: string, userAgent: string): Promise<string> => {
39
+ const response = await fetch(url, {
40
+ method: 'POST',
41
+ headers: {
42
+ ...baseHeaders(userAgent),
43
+ 'Content-Type': 'application/x-www-form-urlencoded',
44
+ },
45
+ body: searchParams(query),
46
+ })
47
+
48
+ if (!response.ok) throw new Error(response.status + ' ' + response.statusText)
49
+
50
+ const html = await response.text()
51
+ detectBlock('DuckDuckGo', html, BLOCK_MARKERS)
52
+
53
+ return html
54
+ }
55
+
56
+ const parseHtmlResults = (html: string): DdgResult[] => {
57
+ const doc = parseDom(html, DDG_HTML_URL)
58
+
59
+ const links = doc.querySelectorAll('.result__a')
60
+ const snippets = doc.querySelectorAll('.result__snippet')
61
+
62
+ const results: DdgResult[] = []
63
+
64
+ for (let i = 0; i < links.length; i++) {
65
+ const link = links[i]
66
+ if (!link) continue
67
+
68
+ const title = link.textContent?.trim() ?? ''
69
+ const href = link.getAttribute('href') ?? ''
70
+ const abstract = snippets[i]?.textContent?.trim() ?? ''
71
+
72
+ if (title && href) results.push({ title, url: href, abstract })
73
+ }
74
+
75
+ return results
76
+ }
77
+
78
+ const parseLiteResults = (html: string): DdgResult[] => {
79
+ const doc = parseDom(html, DDG_LITE_URL)
80
+
81
+ const links = doc.querySelectorAll('a.result-link')
82
+ const snippets = doc.querySelectorAll('td.result-snippet')
83
+
84
+ const results: DdgResult[] = []
85
+
86
+ for (let i = 0; i < links.length; i++) {
87
+ const link = links[i]
88
+ if (!link) continue
89
+
90
+ const title = link.textContent?.trim() ?? ''
91
+ const href = link.getAttribute('href') ?? ''
92
+ const abstract = snippets[i]?.textContent?.trim() ?? ''
93
+
94
+ if (title && href) results.push({ title, url: href, abstract })
95
+ }
96
+
97
+ return results
98
+ }
99
+
100
+ export const ddgSearch = async (query: string): Promise<DdgResult[]> => {
101
+ const cacheKey = 'ddg:' + query
102
+
103
+ const cached = await cache.get<DdgResult[]>(cacheKey)
104
+ if (cached) return cached
105
+
106
+ const results = await runStrategies([{
107
+ name: 'ddg-html',
108
+ run: async (userAgent) => parseHtmlResults(await fetchDdg(DDG_HTML_URL, query, userAgent)),
109
+ }, {
110
+ name: 'ddg-lite',
111
+ run: async (userAgent) => parseLiteResults(await fetchDdg(DDG_LITE_URL, query, userAgent)),
112
+ }])
113
+
114
+ await cache.set(cacheKey, results)
115
+
116
+ return results
117
+ }
@@ -0,0 +1,72 @@
1
+ import * as cache from '../cache.ts'
2
+ import { baseHeaders } from '../strategies.ts'
3
+
4
+ // Not a general web search: the index is news-weighted and reference pages are largely absent.
5
+ const GNEWS_URL = 'https://news.google.com/rss/search'
6
+
7
+ export type GnewsResult = {
8
+ title: string
9
+ url: string
10
+ abstract: string
11
+ source: string
12
+ date: string
13
+ }
14
+
15
+ const ENTITIES: Record<string, string> = {
16
+ '&amp;': '&',
17
+ '&lt;': '<',
18
+ '&gt;': '>',
19
+ '&quot;': '"',
20
+ '&apos;': "'",
21
+ '&#39;': "'",
22
+ }
23
+
24
+ const decodeEntities = (value: string): string => (
25
+ value.replace(/&(?:amp|lt|gt|quot|apos|#39);/g, (match) => ENTITIES[match] ?? match)
26
+ )
27
+
28
+ // Feed titles arrive as 'Headline goes here - Publisher', with the publisher repeated in <source>.
29
+ const stripSource = (title: string, source: string): string => {
30
+ const suffix = ' - ' + source
31
+ return title.endsWith(suffix) ? title.slice(0, -suffix.length) : title
32
+ }
33
+
34
+ // The feed always returns about 100 items, which is ~10k tokens of context for one call.
35
+ const DEFAULT_LIMIT = 20
36
+
37
+ export const gnewsSearch = async (query: string, limit = DEFAULT_LIMIT): Promise<GnewsResult[]> => {
38
+ const cacheKey = 'gnews:' + query
39
+
40
+ const cached = await cache.get<GnewsResult[]>(cacheKey)
41
+ if (cached) return cached.slice(0, limit)
42
+
43
+ const params = new URLSearchParams({ q: query, hl: 'en-US', gl: 'US', ceid: 'US:en' })
44
+ const res = await fetch(GNEWS_URL + '?' + params, { headers: baseHeaders('') })
45
+
46
+ if (!res.ok) throw new Error('Google News returned ' + res.status + ' ' + res.statusText)
47
+
48
+ const xml = await res.text()
49
+ const results: GnewsResult[] = []
50
+
51
+ // Matches one <item> element and captures its body.
52
+ // Not jsdom: it treats <link> as a void element, so every item parses as having no URL.
53
+ for (const match of xml.matchAll(/<item>(.*?)<\/item>/gs)) {
54
+ const item = match[1] ?? ''
55
+
56
+ const tag = (name: string): string => {
57
+ const [, value = ''] = item.match(new RegExp('<' + name + '[^>]*>(.*?)</' + name + '>', 's')) ?? []
58
+ return decodeEntities(value.replace(/<!\[CDATA\[(.*?)\]\]>/gs, '$1').trim())
59
+ }
60
+
61
+ const source = tag('source')
62
+ const title = stripSource(tag('title'), source)
63
+ // A news.google.com redirector that only resolves in a browser, reported as-is.
64
+ const url = tag('link')
65
+
66
+ if (title && url) results.push({ title, url, abstract: '', source, date: tag('pubDate') })
67
+ }
68
+
69
+ await cache.set(cacheKey, results)
70
+
71
+ return results.slice(0, limit)
72
+ }
@@ -0,0 +1,17 @@
1
+ import { describe, expect, it } from 'bun:test'
2
+ import { mdnSearch } from './mdn-search.ts'
3
+
4
+ describe('mdnSearch', () => {
5
+ it('should return MDN documents for a valid query', async () => {
6
+ const results = await mdnSearch({ query: 'fetch' })
7
+
8
+ expect(results.documents.length).toBeGreaterThan(0)
9
+
10
+ const first = results.documents[0]
11
+ if (!first) throw new Error('Expected at least one document')
12
+
13
+ expect(first.title).toBeDefined()
14
+ expect(first.mdn_url).toBeDefined()
15
+ expect(first.summary).toBeDefined()
16
+ })
17
+ })
@@ -0,0 +1,83 @@
1
+ import { z } from 'zod'
2
+ import * as cache from '../cache.ts'
3
+ import { USER_AGENT } from '../user-agent.ts'
4
+
5
+ const MDN_API = 'https://developer.mozilla.org/api/v1/search'
6
+
7
+ const documentSchema = z.object({
8
+ mdn_url: z.string().min(1),
9
+ title: z.string().min(1),
10
+ summary: z.string().default(''),
11
+ slug: z.string().default(''),
12
+ locale: z.string().default('en-US'),
13
+ popularity: z.number().default(0),
14
+ })
15
+
16
+ const responseSchema = z.object({
17
+ documents: z.array(z.unknown()),
18
+ metadata: z.object({
19
+ total: z.object({
20
+ value: z.number(),
21
+ }),
22
+ }).optional(),
23
+ })
24
+
25
+ export type MdnDocument = z.infer<typeof documentSchema>
26
+
27
+ export type MdnSearchResult = {
28
+ documents: Array<MdnDocument>
29
+ total: number
30
+ }
31
+
32
+ type MdnSearchOptions = {
33
+ query: string
34
+ limit?: number
35
+ page?: number
36
+ }
37
+
38
+ export const mdnSearch = async (options: MdnSearchOptions): Promise<MdnSearchResult> => {
39
+ const cacheKey = [
40
+ 'mdn',
41
+ options.query,
42
+ options.limit ?? 10,
43
+ options.page ?? 1,
44
+ ].join(':')
45
+
46
+ const cached = await cache.get<MdnSearchResult>(cacheKey)
47
+ if (cached && cached.documents && cached.documents.length > 0) return cached
48
+
49
+ const params = new URLSearchParams({
50
+ q: options.query,
51
+ locale: 'en-US',
52
+ })
53
+
54
+ if (options.limit) params.set('size', String(options.limit))
55
+
56
+ if (options.page) params.set('page', String(options.page))
57
+
58
+ const url = MDN_API + '?' + params.toString()
59
+ const res = await fetch(url, {
60
+ headers: { 'User-Agent': USER_AGENT },
61
+ })
62
+
63
+ if (!res.ok) {
64
+ const text = await res.text()
65
+ throw new Error('MDN search error (' + res.status + '): ' + text)
66
+ }
67
+
68
+ const json: unknown = await res.json()
69
+ const response = responseSchema.parse(json)
70
+
71
+ const documents = response.documents.flatMap((raw) => {
72
+ const parsed = documentSchema.safeParse(raw)
73
+ return parsed.success ? [parsed.data] : []
74
+ })
75
+
76
+ const total = response.metadata?.total.value ?? documents.length
77
+
78
+ const result: MdnSearchResult = { documents, total }
79
+
80
+ if (documents.length > 0) await cache.set(cacheKey, result)
81
+
82
+ return result
83
+ }
@@ -0,0 +1,29 @@
1
+ import { describe, expect, it } from 'bun:test'
2
+ import { safeAsync } from '../safe.ts'
3
+ import { standardSearch } from './standard-search.ts'
4
+
5
+ const isServiceDown = (error: Error): boolean => (
6
+ error.message.includes('Unable to connect') || error.message.includes('ConnectionRefused')
7
+ )
8
+
9
+ describe('standardSearch', () => {
10
+ it('returns document results', async () => {
11
+ const { data: results, error } = await safeAsync(() => standardSearch({ query: 'atproto' }))
12
+ if (error) {
13
+ // An unreachable appview host is a service outage, not a plugin bug
14
+ if (isServiceDown(error)) {
15
+ console.warn('standard-search appview unreachable, skipping live assertion')
16
+ return
17
+ }
18
+
19
+ throw error
20
+ }
21
+
22
+ expect(results.documents.length > 0).toEqual(true)
23
+
24
+ const first = results.documents[0]
25
+ if (!first) throw new Error('expected at least one document')
26
+ expect(first.title).toBeDefined()
27
+ expect(first.url).toBeDefined()
28
+ })
29
+ })
@@ -0,0 +1,94 @@
1
+ import { JSDOM, VirtualConsole } from 'jsdom'
2
+ import * as cache from '../cache.ts'
3
+
4
+ const SEARCH_URL = 'https://standard-search.octet-stream.net/search'
5
+
6
+ export type StandardDocument = {
7
+ title: string
8
+ url: string
9
+ date: string
10
+ snippet: string
11
+ }
12
+
13
+ export type StandardSearchResult = {
14
+ documents: Array<StandardDocument>
15
+ totalResults: string
16
+ }
17
+
18
+ const parseDocument = (el: Element): StandardDocument | null => {
19
+ const titleEl = el.querySelector('.result-title a')
20
+ if (!titleEl || !titleEl.textContent) return null
21
+
22
+ const title = titleEl.textContent.trim()
23
+ const href = titleEl.getAttribute('href')
24
+
25
+ const urlEl = el.querySelector('.result-url')
26
+ const urlText = urlEl && urlEl.textContent ? urlEl.textContent : ''
27
+
28
+ const snippetEl = el.querySelector('.result-snippet')
29
+ const snippet = snippetEl && snippetEl.textContent ? snippetEl.textContent : ''
30
+
31
+ const parts = urlText.split(' \u00B7 ')
32
+ const datePart = parts.length > 1 ? parts[1] : ''
33
+
34
+ return {
35
+ title,
36
+ url: href ? href.trim() : '',
37
+ date: datePart ? datePart.trim() : '',
38
+ snippet: snippet.trim(),
39
+ }
40
+ }
41
+
42
+ type StandardSearchOptions = {
43
+ query: string
44
+ limit?: number
45
+ offset?: number
46
+ }
47
+
48
+ export const standardSearch = async (options: StandardSearchOptions): Promise<StandardSearchResult> => {
49
+ const cacheKey = ['standard', options.query, options.limit ?? 0, options.offset ?? 0].join(':')
50
+
51
+ const cached = await cache.get<StandardSearchResult>(cacheKey)
52
+ if (cached && cached.documents && cached.documents.length > 0) return cached
53
+
54
+ const url = SEARCH_URL + '?q=' + encodeURIComponent(options.query)
55
+ const res = await fetch(url)
56
+
57
+ if (!res.ok) throw new Error('Standard Search error (' + res.status + '): ' + res.statusText)
58
+
59
+ const html = await res.text()
60
+
61
+ const virtualConsole = new VirtualConsole()
62
+ virtualConsole.on('jsdomError', () => {})
63
+
64
+ const dom = new JSDOM(html, { url, virtualConsole })
65
+ const doc = dom.window.document
66
+
67
+ const countEl = doc.querySelector('.result-count')
68
+ const countText = countEl ? countEl.textContent : ''
69
+ const totalResults = countText ? countText.trim() : '0'
70
+
71
+ const elements = doc.querySelectorAll('.result')
72
+ const documents: Array<StandardDocument> = []
73
+
74
+ for (let i = 0; i < elements.length; i++) {
75
+ const el = elements[i]
76
+ if (!el) continue
77
+
78
+ const parsed = parseDocument(el)
79
+ if (parsed) documents.push(parsed)
80
+ }
81
+
82
+ const start = options.offset ?? 0
83
+ const end = options.limit ? start + options.limit : documents.length
84
+ const sliced = documents.slice(start, end)
85
+
86
+ const result: StandardSearchResult = {
87
+ documents: sliced,
88
+ totalResults,
89
+ }
90
+
91
+ if (sliced.length > 0) await cache.set(cacheKey, result)
92
+
93
+ return result
94
+ }
@@ -0,0 +1,16 @@
1
+ import { describe, expect, it } from 'bun:test'
2
+ import { wikiSearch } from './wiki-search.ts'
3
+
4
+ describe('wikiSearch', () => {
5
+ it('returns page results', async () => {
6
+ const results = await wikiSearch({ query: 'typescript programming' })
7
+
8
+ expect(results.pages.length > 0).toEqual(true)
9
+
10
+ const first = results.pages[0]
11
+ if (!first) throw new Error('expected at least one page')
12
+ expect(first.title).toBeDefined()
13
+ expect(first.key).toBeDefined()
14
+ expect(first.excerpt).toBeDefined()
15
+ })
16
+ })
@@ -0,0 +1,72 @@
1
+ import { z } from 'zod'
2
+ import * as cache from '../cache.ts'
3
+ import { normalizeBlurb } from '../normalize-blurb.ts'
4
+ import { USER_AGENT } from '../user-agent.ts'
5
+
6
+ const WIKI_API = 'https://en.wikipedia.org/w/rest.php/v1/search/page'
7
+
8
+ const thumbnailSchema = z.object({
9
+ url: z.string().min(1),
10
+ width: z.number(),
11
+ height: z.number(),
12
+ })
13
+
14
+ const pageSchema = z.object({
15
+ id: z.number().default(0),
16
+ key: z.string().min(1),
17
+ title: z.string().min(1),
18
+ excerpt: z.string().default('').transform(normalizeBlurb),
19
+ description: z.string().nullable().default(null),
20
+ matched_title: z.string().nullable().default(null),
21
+ thumbnail: thumbnailSchema.nullable().default(null),
22
+ })
23
+
24
+ const responseSchema = z.object({
25
+ pages: z.array(z.unknown()),
26
+ })
27
+
28
+ export type WikiPage = z.infer<typeof pageSchema>
29
+
30
+ export type WikiSearchResult = {
31
+ pages: Array<WikiPage>
32
+ }
33
+
34
+ type WikiSearchOptions = {
35
+ query: string
36
+ limit?: number
37
+ }
38
+
39
+ export const wikiSearch = async (options: WikiSearchOptions): Promise<WikiSearchResult> => {
40
+ const cacheKey = 'wiki:' + options.query + ':' + (options.limit ?? 10)
41
+
42
+ const cached = await cache.get<WikiSearchResult>(cacheKey)
43
+ if (cached && cached.pages && cached.pages.length > 0) return cached
44
+
45
+ const params = new URLSearchParams({ q: options.query })
46
+
47
+ if (options.limit) params.set('limit', String(options.limit))
48
+
49
+ const url = WIKI_API + '?' + params.toString()
50
+ const res = await fetch(url, {
51
+ headers: { 'User-Agent': USER_AGENT },
52
+ })
53
+
54
+ if (!res.ok) {
55
+ const text = await res.text()
56
+ throw new Error('Wikipedia search error (' + res.status + '): ' + text)
57
+ }
58
+
59
+ const json: unknown = await res.json()
60
+ const response = responseSchema.parse(json)
61
+
62
+ const pages = response.pages.flatMap((raw) => {
63
+ const parsed = pageSchema.safeParse(raw)
64
+ return parsed.success ? [parsed.data] : []
65
+ })
66
+
67
+ const result: WikiSearchResult = { pages }
68
+
69
+ if (pages.length > 0) await cache.set(cacheKey, result)
70
+
71
+ return result
72
+ }