opencode-search 1.0.0-next.1 → 1.0.0-next.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -24
- package/REMOVED.md +13 -3
- package/dist/package.json +18 -20
- package/dist/search.js +1 -1
- package/dist/src/providers/gnews-search.d.ts +1 -1
- package/dist/src/providers/gnews-search.d.ts.map +1 -1
- package/dist/src/providers/gnews-search.js +5 -3
- package/dist/src/providers/gnews-search.js.map +1 -1
- package/dist/src/providers/rfc-search.d.ts +9 -0
- package/dist/src/providers/rfc-search.d.ts.map +1 -0
- package/dist/src/providers/rfc-search.js +38 -0
- package/dist/src/providers/rfc-search.js.map +1 -0
- package/dist/src/providers/w3c-search.d.ts +6 -0
- package/dist/src/providers/w3c-search.d.ts.map +1 -0
- package/dist/src/providers/w3c-search.js +42 -0
- package/dist/src/providers/w3c-search.js.map +1 -0
- package/dist/src/providers/whatwg-search.d.ts +8 -0
- package/dist/src/providers/whatwg-search.d.ts.map +1 -0
- package/dist/src/providers/whatwg-search.js +31 -0
- package/dist/src/providers/whatwg-search.js.map +1 -0
- package/dist/src/renderers.d.ts +6 -2
- package/dist/src/renderers.d.ts.map +1 -1
- package/dist/src/renderers.js +14 -8
- package/dist/src/renderers.js.map +1 -1
- package/dist/src/search.d.ts +3 -1
- package/dist/src/search.d.ts.map +1 -1
- package/dist/src/search.js +123 -61
- package/dist/src/search.js.map +1 -1
- package/package.json +18 -20
- package/search.ts +22 -0
- package/src/cache.test.ts +14 -0
- package/src/cache.ts +61 -0
- package/src/format.test.ts +175 -0
- package/src/format.ts +72 -0
- package/src/html.ts +39 -0
- package/src/normalize-blurb.test.ts +67 -0
- package/src/normalize-blurb.ts +22 -0
- package/src/opencode/notify.ts +16 -0
- package/src/parse-results.test.ts +75 -0
- package/src/parse-results.ts +22 -0
- package/src/providers/brave-search.ts +39 -0
- package/src/providers/bsky-search.test.ts +16 -0
- package/src/providers/bsky-search.ts +88 -0
- package/src/providers/ddg-search.test.ts +15 -0
- package/src/providers/ddg-search.ts +117 -0
- package/src/providers/gnews-search.ts +72 -0
- package/src/providers/mdn-search.test.ts +17 -0
- package/src/providers/mdn-search.ts +83 -0
- package/src/providers/rfc-search.test.ts +31 -0
- package/src/providers/rfc-search.ts +61 -0
- package/src/providers/w3c-search.test.ts +22 -0
- package/src/providers/w3c-search.ts +67 -0
- package/src/providers/whatwg-search.test.ts +21 -0
- package/src/providers/whatwg-search.ts +55 -0
- package/src/providers/wiki-search.test.ts +16 -0
- package/src/providers/wiki-search.ts +72 -0
- package/src/renderers.test.ts +263 -0
- package/src/renderers.ts +62 -0
- package/src/safe.test.ts +87 -0
- package/src/safe.ts +27 -0
- package/src/search.ts +369 -0
- package/src/strategies.test.ts +92 -0
- package/src/strategies.ts +62 -0
- package/src/user-agent.ts +3 -0
- package/src/v1-message.test.ts +17 -0
- package/src/v1-message.ts +10 -0
- package/dist/src/providers/standard-search.d.ts +0 -18
- package/dist/src/providers/standard-search.d.ts.map +0 -1
- package/dist/src/providers/standard-search.js +0 -61
- package/dist/src/providers/standard-search.js.map +0 -1
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { JSDOM, VirtualConsole } from 'jsdom'
|
|
2
|
+
import * as cache from '../cache.ts'
|
|
3
|
+
import { baseHeaders, detectBlock, runStrategies } from '../strategies.ts'
|
|
4
|
+
|
|
5
|
+
// The lite frontend is the fallback when the html frontend is blocked or empty
|
|
6
|
+
const DDG_HTML_URL = 'https://html.duckduckgo.com/html/'
|
|
7
|
+
const DDG_LITE_URL = 'https://lite.duckduckgo.com/lite/'
|
|
8
|
+
|
|
9
|
+
const BLOCK_MARKERS = ['anomaly-modal', 'Please try again']
|
|
10
|
+
|
|
11
|
+
export type DdgResult = {
|
|
12
|
+
title: string
|
|
13
|
+
url: string
|
|
14
|
+
abstract: string
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const searchParams = (query: string): URLSearchParams => (
|
|
18
|
+
new URLSearchParams({
|
|
19
|
+
q: query,
|
|
20
|
+
b: '',
|
|
21
|
+
kf: '-1',
|
|
22
|
+
kh: '1',
|
|
23
|
+
kl: 'us-en',
|
|
24
|
+
kp: '1',
|
|
25
|
+
k1: '-1',
|
|
26
|
+
})
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
const parseDom = (html: string, url: string): Document => {
|
|
30
|
+
const virtualConsole = new VirtualConsole()
|
|
31
|
+
virtualConsole.on('jsdomError', () => {
|
|
32
|
+
// Silently ignore JSDOM errors
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
return new JSDOM(html, { url, virtualConsole }).window.document
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const fetchDdg = async (url: string, query: string, userAgent: string): Promise<string> => {
|
|
39
|
+
const response = await fetch(url, {
|
|
40
|
+
method: 'POST',
|
|
41
|
+
headers: {
|
|
42
|
+
...baseHeaders(userAgent),
|
|
43
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
44
|
+
},
|
|
45
|
+
body: searchParams(query),
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
if (!response.ok) throw new Error(response.status + ' ' + response.statusText)
|
|
49
|
+
|
|
50
|
+
const html = await response.text()
|
|
51
|
+
detectBlock('DuckDuckGo', html, BLOCK_MARKERS)
|
|
52
|
+
|
|
53
|
+
return html
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const parseHtmlResults = (html: string): DdgResult[] => {
|
|
57
|
+
const doc = parseDom(html, DDG_HTML_URL)
|
|
58
|
+
|
|
59
|
+
const links = doc.querySelectorAll('.result__a')
|
|
60
|
+
const snippets = doc.querySelectorAll('.result__snippet')
|
|
61
|
+
|
|
62
|
+
const results: DdgResult[] = []
|
|
63
|
+
|
|
64
|
+
for (let i = 0; i < links.length; i++) {
|
|
65
|
+
const link = links[i]
|
|
66
|
+
if (!link) continue
|
|
67
|
+
|
|
68
|
+
const title = link.textContent?.trim() ?? ''
|
|
69
|
+
const href = link.getAttribute('href') ?? ''
|
|
70
|
+
const abstract = snippets[i]?.textContent?.trim() ?? ''
|
|
71
|
+
|
|
72
|
+
if (title && href) results.push({ title, url: href, abstract })
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return results
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const parseLiteResults = (html: string): DdgResult[] => {
|
|
79
|
+
const doc = parseDom(html, DDG_LITE_URL)
|
|
80
|
+
|
|
81
|
+
const links = doc.querySelectorAll('a.result-link')
|
|
82
|
+
const snippets = doc.querySelectorAll('td.result-snippet')
|
|
83
|
+
|
|
84
|
+
const results: DdgResult[] = []
|
|
85
|
+
|
|
86
|
+
for (let i = 0; i < links.length; i++) {
|
|
87
|
+
const link = links[i]
|
|
88
|
+
if (!link) continue
|
|
89
|
+
|
|
90
|
+
const title = link.textContent?.trim() ?? ''
|
|
91
|
+
const href = link.getAttribute('href') ?? ''
|
|
92
|
+
const abstract = snippets[i]?.textContent?.trim() ?? ''
|
|
93
|
+
|
|
94
|
+
if (title && href) results.push({ title, url: href, abstract })
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return results
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export const ddgSearch = async (query: string): Promise<DdgResult[]> => {
|
|
101
|
+
const cacheKey = 'ddg:' + query
|
|
102
|
+
|
|
103
|
+
const cached = await cache.get<DdgResult[]>(cacheKey)
|
|
104
|
+
if (cached) return cached
|
|
105
|
+
|
|
106
|
+
const results = await runStrategies([{
|
|
107
|
+
name: 'ddg-html',
|
|
108
|
+
run: async (userAgent) => parseHtmlResults(await fetchDdg(DDG_HTML_URL, query, userAgent)),
|
|
109
|
+
}, {
|
|
110
|
+
name: 'ddg-lite',
|
|
111
|
+
run: async (userAgent) => parseLiteResults(await fetchDdg(DDG_LITE_URL, query, userAgent)),
|
|
112
|
+
}])
|
|
113
|
+
|
|
114
|
+
await cache.set(cacheKey, results)
|
|
115
|
+
|
|
116
|
+
return results
|
|
117
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import * as cache from '../cache.ts'
|
|
2
|
+
import { baseHeaders } from '../strategies.ts'
|
|
3
|
+
|
|
4
|
+
// Not a general web search: the index is news-weighted and reference pages are largely absent.
|
|
5
|
+
const GNEWS_URL = 'https://news.google.com/rss/search'
|
|
6
|
+
|
|
7
|
+
export type GnewsResult = {
|
|
8
|
+
title: string
|
|
9
|
+
url: string
|
|
10
|
+
abstract: string
|
|
11
|
+
source: string
|
|
12
|
+
date: string
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const ENTITIES: Record<string, string> = {
|
|
16
|
+
'&': '&',
|
|
17
|
+
'<': '<',
|
|
18
|
+
'>': '>',
|
|
19
|
+
'"': '"',
|
|
20
|
+
''': "'",
|
|
21
|
+
''': "'",
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const decodeEntities = (value: string): string => (
|
|
25
|
+
value.replace(/&(?:amp|lt|gt|quot|apos|#39);/g, (match) => ENTITIES[match] ?? match)
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
// Feed titles arrive as 'Headline goes here - Publisher', with the publisher repeated in <source>.
|
|
29
|
+
const stripSource = (title: string, source: string): string => {
|
|
30
|
+
const suffix = ' - ' + source
|
|
31
|
+
return title.endsWith(suffix) ? title.slice(0, -suffix.length) : title
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// The feed always returns about 100 items, which is ~10k tokens of context for one call.
|
|
35
|
+
const DEFAULT_LIMIT = 20
|
|
36
|
+
|
|
37
|
+
export const gnewsSearch = async (query: string, limit = DEFAULT_LIMIT): Promise<GnewsResult[]> => {
|
|
38
|
+
const cacheKey = 'gnews:' + query
|
|
39
|
+
|
|
40
|
+
const cached = await cache.get<GnewsResult[]>(cacheKey)
|
|
41
|
+
if (cached) return cached.slice(0, limit)
|
|
42
|
+
|
|
43
|
+
const params = new URLSearchParams({ q: query, hl: 'en-US', gl: 'US', ceid: 'US:en' })
|
|
44
|
+
const res = await fetch(GNEWS_URL + '?' + params, { headers: baseHeaders('') })
|
|
45
|
+
|
|
46
|
+
if (!res.ok) throw new Error('Google News returned ' + res.status + ' ' + res.statusText)
|
|
47
|
+
|
|
48
|
+
const xml = await res.text()
|
|
49
|
+
const results: GnewsResult[] = []
|
|
50
|
+
|
|
51
|
+
// Matches one <item> element and captures its body.
|
|
52
|
+
// Not jsdom: it treats <link> as a void element, so every item parses as having no URL.
|
|
53
|
+
for (const match of xml.matchAll(/<item>(.*?)<\/item>/gs)) {
|
|
54
|
+
const item = match[1] ?? ''
|
|
55
|
+
|
|
56
|
+
const tag = (name: string): string => {
|
|
57
|
+
const [, value = ''] = item.match(new RegExp('<' + name + '[^>]*>(.*?)</' + name + '>', 's')) ?? []
|
|
58
|
+
return decodeEntities(value.replace(/<!\[CDATA\[(.*?)\]\]>/gs, '$1').trim())
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const source = tag('source')
|
|
62
|
+
const title = stripSource(tag('title'), source)
|
|
63
|
+
// A news.google.com redirector that only resolves in a browser, reported as-is.
|
|
64
|
+
const url = tag('link')
|
|
65
|
+
|
|
66
|
+
if (title && url) results.push({ title, url, abstract: '', source, date: tag('pubDate') })
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
await cache.set(cacheKey, results)
|
|
70
|
+
|
|
71
|
+
return results.slice(0, limit)
|
|
72
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test'
|
|
2
|
+
import { mdnSearch } from './mdn-search.ts'
|
|
3
|
+
|
|
4
|
+
describe('mdnSearch', () => {
|
|
5
|
+
it('should return MDN documents for a valid query', async () => {
|
|
6
|
+
const results = await mdnSearch({ query: 'fetch' })
|
|
7
|
+
|
|
8
|
+
expect(results.documents.length).toBeGreaterThan(0)
|
|
9
|
+
|
|
10
|
+
const first = results.documents[0]
|
|
11
|
+
if (!first) throw new Error('Expected at least one document')
|
|
12
|
+
|
|
13
|
+
expect(first.title).toBeDefined()
|
|
14
|
+
expect(first.mdn_url).toBeDefined()
|
|
15
|
+
expect(first.summary).toBeDefined()
|
|
16
|
+
})
|
|
17
|
+
})
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
import * as cache from '../cache.ts'
|
|
3
|
+
import { USER_AGENT } from '../user-agent.ts'
|
|
4
|
+
|
|
5
|
+
const MDN_API = 'https://developer.mozilla.org/api/v1/search'
|
|
6
|
+
|
|
7
|
+
const documentSchema = z.object({
|
|
8
|
+
mdn_url: z.string().min(1),
|
|
9
|
+
title: z.string().min(1),
|
|
10
|
+
summary: z.string().default(''),
|
|
11
|
+
slug: z.string().default(''),
|
|
12
|
+
locale: z.string().default('en-US'),
|
|
13
|
+
popularity: z.number().default(0),
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
const responseSchema = z.object({
|
|
17
|
+
documents: z.array(z.unknown()),
|
|
18
|
+
metadata: z.object({
|
|
19
|
+
total: z.object({
|
|
20
|
+
value: z.number(),
|
|
21
|
+
}),
|
|
22
|
+
}).optional(),
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
export type MdnDocument = z.infer<typeof documentSchema>
|
|
26
|
+
|
|
27
|
+
export type MdnSearchResult = {
|
|
28
|
+
documents: Array<MdnDocument>
|
|
29
|
+
total: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
type MdnSearchOptions = {
|
|
33
|
+
query: string
|
|
34
|
+
limit?: number
|
|
35
|
+
page?: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export const mdnSearch = async (options: MdnSearchOptions): Promise<MdnSearchResult> => {
|
|
39
|
+
const cacheKey = [
|
|
40
|
+
'mdn',
|
|
41
|
+
options.query,
|
|
42
|
+
options.limit ?? 10,
|
|
43
|
+
options.page ?? 1,
|
|
44
|
+
].join(':')
|
|
45
|
+
|
|
46
|
+
const cached = await cache.get<MdnSearchResult>(cacheKey)
|
|
47
|
+
if (cached && cached.documents && cached.documents.length > 0) return cached
|
|
48
|
+
|
|
49
|
+
const params = new URLSearchParams({
|
|
50
|
+
q: options.query,
|
|
51
|
+
locale: 'en-US',
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
if (options.limit) params.set('size', String(options.limit))
|
|
55
|
+
|
|
56
|
+
if (options.page) params.set('page', String(options.page))
|
|
57
|
+
|
|
58
|
+
const url = MDN_API + '?' + params.toString()
|
|
59
|
+
const res = await fetch(url, {
|
|
60
|
+
headers: { 'User-Agent': USER_AGENT },
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
if (!res.ok) {
|
|
64
|
+
const text = await res.text()
|
|
65
|
+
throw new Error('MDN search error (' + res.status + '): ' + text)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const json: unknown = await res.json()
|
|
69
|
+
const response = responseSchema.parse(json)
|
|
70
|
+
|
|
71
|
+
const documents = response.documents.flatMap((raw) => {
|
|
72
|
+
const parsed = documentSchema.safeParse(raw)
|
|
73
|
+
return parsed.success ? [parsed.data] : []
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
const total = response.metadata?.total.value ?? documents.length
|
|
77
|
+
|
|
78
|
+
const result: MdnSearchResult = { documents, total }
|
|
79
|
+
|
|
80
|
+
if (documents.length > 0) await cache.set(cacheKey, result)
|
|
81
|
+
|
|
82
|
+
return result
|
|
83
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test'
|
|
2
|
+
import { rfcSearch } from './rfc-search.ts'
|
|
3
|
+
|
|
4
|
+
describe('rfcSearch', () => {
|
|
5
|
+
it('returns RFCs matching a title query', async () => {
|
|
6
|
+
const results = await rfcSearch('HTTP Semantics')
|
|
7
|
+
|
|
8
|
+
expect(results.length).toBeGreaterThan(0)
|
|
9
|
+
|
|
10
|
+
const first = results[0]
|
|
11
|
+
if (!first) throw new Error('Expected at least one RFC')
|
|
12
|
+
|
|
13
|
+
expect(first.id).toMatch(/^RFC\d+$/)
|
|
14
|
+
expect(first.url).toStartWith('https://www.rfc-editor.org/rfc/')
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
it('looks up an RFC by number', async () => {
|
|
18
|
+
for (const query of ['RFC9421', 'rfc 9421', '9421']) {
|
|
19
|
+
const results = await rfcSearch(query)
|
|
20
|
+
|
|
21
|
+
expect(results.length).toEqual(1)
|
|
22
|
+
expect(results[0]?.id).toEqual('RFC9421')
|
|
23
|
+
}
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
it('respects the limit', async () => {
|
|
27
|
+
const results = await rfcSearch('HTTP', 3)
|
|
28
|
+
|
|
29
|
+
expect(results.length).toBeLessThanOrEqual(3)
|
|
30
|
+
})
|
|
31
|
+
})
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import * as cache from '../cache.ts'
|
|
2
|
+
|
|
3
|
+
const DATATRACKER_URL = 'https://datatracker.ietf.org/api/v1/doc/document/'
|
|
4
|
+
const DEFAULT_LIMIT = 20
|
|
5
|
+
|
|
6
|
+
export type RfcResult = {
|
|
7
|
+
id: string
|
|
8
|
+
title: string
|
|
9
|
+
url: string
|
|
10
|
+
date: string
|
|
11
|
+
abstract: string
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
type DatatrackerDoc = {
|
|
15
|
+
name?: string
|
|
16
|
+
title?: string
|
|
17
|
+
time?: string
|
|
18
|
+
abstract?: string
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export const rfcSearch = async (query: string, limit = DEFAULT_LIMIT): Promise<RfcResult[]> => {
|
|
22
|
+
const cacheKey = 'rfc:' + query
|
|
23
|
+
|
|
24
|
+
const cached = await cache.get<RfcResult[]>(cacheKey)
|
|
25
|
+
if (cached) return cached.slice(0, limit)
|
|
26
|
+
|
|
27
|
+
// A bare number or an 'RFC 9421' style query is a lookup, not a title search.
|
|
28
|
+
// Matches an optional 'rfc' prefix, optional space, then the digits.
|
|
29
|
+
const [, number] = query.trim().match(/^rfc[\s-]?(\d{1,5})$/i) ?? query.trim().match(/^(\d{1,5})$/) ?? []
|
|
30
|
+
|
|
31
|
+
const params = new URLSearchParams({
|
|
32
|
+
...(number ? { name: 'rfc' + number } : { title__icontains: query }),
|
|
33
|
+
type: 'rfc',
|
|
34
|
+
limit: String(Math.min(limit, 100)),
|
|
35
|
+
format: 'json',
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
const res = await fetch(DATATRACKER_URL + '?' + params)
|
|
39
|
+
|
|
40
|
+
if (!res.ok) throw new Error('IETF Datatracker returned ' + res.status + ' ' + res.statusText)
|
|
41
|
+
|
|
42
|
+
const body = await res.json() as { objects?: DatatrackerDoc[] }
|
|
43
|
+
const results: RfcResult[] = []
|
|
44
|
+
|
|
45
|
+
for (const doc of body.objects ?? []) {
|
|
46
|
+
const id = doc.name ?? ''
|
|
47
|
+
if (!id) continue
|
|
48
|
+
|
|
49
|
+
results.push({
|
|
50
|
+
id: id.toUpperCase(),
|
|
51
|
+
title: doc.title ?? '',
|
|
52
|
+
url: 'https://www.rfc-editor.org/rfc/' + id + '.html',
|
|
53
|
+
date: doc.time ?? '',
|
|
54
|
+
abstract: doc.abstract ?? '',
|
|
55
|
+
})
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
await cache.set(cacheKey, results)
|
|
59
|
+
|
|
60
|
+
return results.slice(0, limit)
|
|
61
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test'
|
|
2
|
+
import { w3cSearch } from './w3c-search.ts'
|
|
3
|
+
|
|
4
|
+
describe('w3cSearch', () => {
|
|
5
|
+
it('filters specifications by title', async () => {
|
|
6
|
+
const results = await w3cSearch('CSS')
|
|
7
|
+
|
|
8
|
+
expect(results.length).toBeGreaterThan(0)
|
|
9
|
+
|
|
10
|
+
const first = results[0]
|
|
11
|
+
if (!first) throw new Error('Expected at least one specification')
|
|
12
|
+
|
|
13
|
+
expect(first.title.toLowerCase()).toContain('css')
|
|
14
|
+
expect(first.url).toStartWith('http')
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
it('respects the limit', async () => {
|
|
18
|
+
const results = await w3cSearch('CSS', 5)
|
|
19
|
+
|
|
20
|
+
expect(results.length).toBeLessThanOrEqual(5)
|
|
21
|
+
})
|
|
22
|
+
})
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import * as cache from '../cache.ts'
|
|
2
|
+
|
|
3
|
+
// The W3C API accepts a `q` parameter and ignores it: 'css' and a nonsense string both return all
|
|
4
|
+
// 1713 specifications.
|
|
5
|
+
// So the whole list is fetched and filtered here, 1000 per page.
|
|
6
|
+
const API_URL = 'https://api.w3.org/specifications'
|
|
7
|
+
const PAGE_SIZE = 1000
|
|
8
|
+
const DEFAULT_LIMIT = 20
|
|
9
|
+
|
|
10
|
+
export type W3cResult = {
|
|
11
|
+
title: string
|
|
12
|
+
url: string
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
type Spec = {
|
|
16
|
+
title?: string
|
|
17
|
+
href?: string
|
|
18
|
+
'shortlink'?: string
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
type Page = {
|
|
22
|
+
pages?: number
|
|
23
|
+
_links?: { specifications?: Spec[] }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const fetchPage = async (page: number): Promise<Page> => {
|
|
27
|
+
const params = new URLSearchParams({ items: String(PAGE_SIZE), page: String(page) })
|
|
28
|
+
const res = await fetch(API_URL + '?' + params)
|
|
29
|
+
|
|
30
|
+
if (!res.ok) throw new Error('W3C API returned ' + res.status + ' ' + res.statusText)
|
|
31
|
+
|
|
32
|
+
return await res.json() as Page
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export const w3cSearch = async (query: string, limit = DEFAULT_LIMIT): Promise<W3cResult[]> => {
|
|
36
|
+
const cacheKey = 'w3c:' + query
|
|
37
|
+
|
|
38
|
+
const cached = await cache.get<W3cResult[]>(cacheKey)
|
|
39
|
+
if (cached) return cached.slice(0, limit)
|
|
40
|
+
|
|
41
|
+
const first = await fetchPage(1)
|
|
42
|
+
const specs = [...first._links?.specifications ?? []]
|
|
43
|
+
|
|
44
|
+
// `pages` counts pages at the requested size, so a second fetch covers the rest at 1000 each.
|
|
45
|
+
const total = first.pages ?? 1
|
|
46
|
+
for (let page = 2; page <= total; page++) {
|
|
47
|
+
const next = await fetchPage(page)
|
|
48
|
+
specs.push(...next._links?.specifications ?? [])
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const needle = query.toLowerCase()
|
|
52
|
+
const results: W3cResult[] = []
|
|
53
|
+
|
|
54
|
+
for (const spec of specs) {
|
|
55
|
+
const title = spec.title ?? ''
|
|
56
|
+
const url = spec.shortlink ?? spec.href ?? ''
|
|
57
|
+
|
|
58
|
+
if (!title || !url) continue
|
|
59
|
+
if (!title.toLowerCase().includes(needle)) continue
|
|
60
|
+
|
|
61
|
+
results.push({ title, url })
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
await cache.set(cacheKey, results)
|
|
65
|
+
|
|
66
|
+
return results.slice(0, limit)
|
|
67
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test'
|
|
2
|
+
import { whatwgSearch } from './whatwg-search.ts'
|
|
3
|
+
|
|
4
|
+
describe('whatwgSearch', () => {
|
|
5
|
+
it('finds a living standard by name', async () => {
|
|
6
|
+
const results = await whatwgSearch('HTML')
|
|
7
|
+
|
|
8
|
+
expect(results.length).toBeGreaterThan(0)
|
|
9
|
+
|
|
10
|
+
const first = results[0]
|
|
11
|
+
if (!first) throw new Error('Expected at least one standard')
|
|
12
|
+
|
|
13
|
+
expect(first.url).toContain('spec.whatwg.org')
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
it('returns nothing for a query that matches no standard', async () => {
|
|
17
|
+
const results = await whatwgSearch('zzzznotastandard')
|
|
18
|
+
|
|
19
|
+
expect(results.length).toEqual(0)
|
|
20
|
+
})
|
|
21
|
+
})
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import * as cache from '../cache.ts'
|
|
2
|
+
|
|
3
|
+
// WHATWG publishes its whole standards index as one small JSON file, and offers no search endpoint.
|
|
4
|
+
const DB_URL = 'https://raw.githubusercontent.com/whatwg/sg/main/db.json'
|
|
5
|
+
const DEFAULT_LIMIT = 20
|
|
6
|
+
|
|
7
|
+
export type WhatwgResult = {
|
|
8
|
+
title: string
|
|
9
|
+
url: string
|
|
10
|
+
abstract: string
|
|
11
|
+
workstream: string
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
type Standard = {
|
|
15
|
+
name?: string
|
|
16
|
+
href?: string
|
|
17
|
+
description?: string
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
type Workstream = {
|
|
21
|
+
name?: string
|
|
22
|
+
standards?: Standard[]
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const whatwgSearch = async (query: string, limit = DEFAULT_LIMIT): Promise<WhatwgResult[]> => {
|
|
26
|
+
const cacheKey = 'whatwg:' + query
|
|
27
|
+
|
|
28
|
+
const cached = await cache.get<WhatwgResult[]>(cacheKey)
|
|
29
|
+
if (cached) return cached.slice(0, limit)
|
|
30
|
+
|
|
31
|
+
const res = await fetch(DB_URL)
|
|
32
|
+
|
|
33
|
+
if (!res.ok) throw new Error('WHATWG index returned ' + res.status + ' ' + res.statusText)
|
|
34
|
+
|
|
35
|
+
const body = await res.json() as { workstreams?: Workstream[] }
|
|
36
|
+
const needle = query.toLowerCase()
|
|
37
|
+
const results: WhatwgResult[] = []
|
|
38
|
+
|
|
39
|
+
for (const workstream of body.workstreams ?? []) {
|
|
40
|
+
for (const standard of workstream.standards ?? []) {
|
|
41
|
+
const title = standard.name ?? ''
|
|
42
|
+
const url = standard.href ?? ''
|
|
43
|
+
const abstract = standard.description ?? ''
|
|
44
|
+
|
|
45
|
+
if (!title || !url) continue
|
|
46
|
+
if (!(title + ' ' + abstract).toLowerCase().includes(needle)) continue
|
|
47
|
+
|
|
48
|
+
results.push({ title, url, abstract, workstream: workstream.name ?? '' })
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
await cache.set(cacheKey, results)
|
|
53
|
+
|
|
54
|
+
return results.slice(0, limit)
|
|
55
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test'
|
|
2
|
+
import { wikiSearch } from './wiki-search.ts'
|
|
3
|
+
|
|
4
|
+
describe('wikiSearch', () => {
|
|
5
|
+
it('returns page results', async () => {
|
|
6
|
+
const results = await wikiSearch({ query: 'typescript programming' })
|
|
7
|
+
|
|
8
|
+
expect(results.pages.length > 0).toEqual(true)
|
|
9
|
+
|
|
10
|
+
const first = results.pages[0]
|
|
11
|
+
if (!first) throw new Error('expected at least one page')
|
|
12
|
+
expect(first.title).toBeDefined()
|
|
13
|
+
expect(first.key).toBeDefined()
|
|
14
|
+
expect(first.excerpt).toBeDefined()
|
|
15
|
+
})
|
|
16
|
+
})
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
import * as cache from '../cache.ts'
|
|
3
|
+
import { normalizeBlurb } from '../normalize-blurb.ts'
|
|
4
|
+
import { USER_AGENT } from '../user-agent.ts'
|
|
5
|
+
|
|
6
|
+
const WIKI_API = 'https://en.wikipedia.org/w/rest.php/v1/search/page'
|
|
7
|
+
|
|
8
|
+
const thumbnailSchema = z.object({
|
|
9
|
+
url: z.string().min(1),
|
|
10
|
+
width: z.number(),
|
|
11
|
+
height: z.number(),
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
const pageSchema = z.object({
|
|
15
|
+
id: z.number().default(0),
|
|
16
|
+
key: z.string().min(1),
|
|
17
|
+
title: z.string().min(1),
|
|
18
|
+
excerpt: z.string().default('').transform(normalizeBlurb),
|
|
19
|
+
description: z.string().nullable().default(null),
|
|
20
|
+
matched_title: z.string().nullable().default(null),
|
|
21
|
+
thumbnail: thumbnailSchema.nullable().default(null),
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
const responseSchema = z.object({
|
|
25
|
+
pages: z.array(z.unknown()),
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
export type WikiPage = z.infer<typeof pageSchema>
|
|
29
|
+
|
|
30
|
+
export type WikiSearchResult = {
|
|
31
|
+
pages: Array<WikiPage>
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type WikiSearchOptions = {
|
|
35
|
+
query: string
|
|
36
|
+
limit?: number
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export const wikiSearch = async (options: WikiSearchOptions): Promise<WikiSearchResult> => {
|
|
40
|
+
const cacheKey = 'wiki:' + options.query + ':' + (options.limit ?? 10)
|
|
41
|
+
|
|
42
|
+
const cached = await cache.get<WikiSearchResult>(cacheKey)
|
|
43
|
+
if (cached && cached.pages && cached.pages.length > 0) return cached
|
|
44
|
+
|
|
45
|
+
const params = new URLSearchParams({ q: options.query })
|
|
46
|
+
|
|
47
|
+
if (options.limit) params.set('limit', String(options.limit))
|
|
48
|
+
|
|
49
|
+
const url = WIKI_API + '?' + params.toString()
|
|
50
|
+
const res = await fetch(url, {
|
|
51
|
+
headers: { 'User-Agent': USER_AGENT },
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
if (!res.ok) {
|
|
55
|
+
const text = await res.text()
|
|
56
|
+
throw new Error('Wikipedia search error (' + res.status + '): ' + text)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const json: unknown = await res.json()
|
|
60
|
+
const response = responseSchema.parse(json)
|
|
61
|
+
|
|
62
|
+
const pages = response.pages.flatMap((raw) => {
|
|
63
|
+
const parsed = pageSchema.safeParse(raw)
|
|
64
|
+
return parsed.success ? [parsed.data] : []
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
const result: WikiSearchResult = { pages }
|
|
68
|
+
|
|
69
|
+
if (pages.length > 0) await cache.set(cacheKey, result)
|
|
70
|
+
|
|
71
|
+
return result
|
|
72
|
+
}
|