@askalf/deepdive 0.13.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -1
- package/dist/agent.d.ts +3 -1
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +48 -13
- package/dist/agent.js.map +1 -1
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js.map +1 -1
- package/dist/cache.d.ts.map +1 -1
- package/dist/cache.js +22 -2
- package/dist/cache.js.map +1 -1
- package/dist/citations.d.ts +1 -0
- package/dist/citations.d.ts.map +1 -1
- package/dist/citations.js +4 -1
- package/dist/citations.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +428 -19
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +3 -0
- package/dist/completion.d.ts.map +1 -0
- package/dist/completion.js +110 -0
- package/dist/completion.js.map +1 -0
- package/dist/confidence.d.ts +17 -0
- package/dist/confidence.d.ts.map +1 -0
- package/dist/confidence.js +41 -0
- package/dist/confidence.js.map +1 -0
- package/dist/config-file.d.ts +13 -0
- package/dist/config-file.d.ts.map +1 -0
- package/dist/config-file.js +150 -0
- package/dist/config-file.js.map +1 -0
- package/dist/config.d.ts +11 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +7 -0
- package/dist/config.js.map +1 -1
- package/dist/dates.d.ts +9 -0
- package/dist/dates.d.ts.map +1 -0
- package/dist/dates.js +218 -0
- package/dist/dates.js.map +1 -0
- package/dist/diff.d.ts +46 -0
- package/dist/diff.d.ts.map +1 -0
- package/dist/diff.js +227 -0
- package/dist/diff.js.map +1 -0
- package/dist/html-export.d.ts +6 -0
- package/dist/html-export.d.ts.map +1 -0
- package/dist/html-export.js +146 -0
- package/dist/html-export.js.map +1 -0
- package/dist/index.d.ts +11 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -1
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +7 -0
- package/dist/markdown.d.ts.map +1 -0
- package/dist/markdown.js +232 -0
- package/dist/markdown.js.map +1 -0
- package/dist/open.d.ts +6 -0
- package/dist/open.d.ts.map +1 -0
- package/dist/open.js +25 -0
- package/dist/open.js.map +1 -0
- package/dist/profiles.d.ts +5 -0
- package/dist/profiles.d.ts.map +1 -0
- package/dist/profiles.js +38 -0
- package/dist/profiles.js.map +1 -0
- package/dist/robots.d.ts +1 -0
- package/dist/robots.d.ts.map +1 -1
- package/dist/robots.js +4 -7
- package/dist/robots.js.map +1 -1
- package/dist/search/arxiv.d.ts +7 -0
- package/dist/search/arxiv.d.ts.map +1 -0
- package/dist/search/arxiv.js +81 -0
- package/dist/search/arxiv.js.map +1 -0
- package/dist/search/brave.d.ts +1 -1
- package/dist/search/brave.d.ts.map +1 -1
- package/dist/search/brave.js +2 -1
- package/dist/search/brave.js.map +1 -1
- package/dist/search/duckduckgo.d.ts +1 -1
- package/dist/search/duckduckgo.d.ts.map +1 -1
- package/dist/search/duckduckgo.js +2 -1
- package/dist/search/duckduckgo.js.map +1 -1
- package/dist/search/github.d.ts +16 -0
- package/dist/search/github.d.ts.map +1 -0
- package/dist/search/github.js +59 -0
- package/dist/search/github.js.map +1 -0
- package/dist/search/hackernews.d.ts +16 -0
- package/dist/search/hackernews.d.ts.map +1 -0
- package/dist/search/hackernews.js +47 -0
- package/dist/search/hackernews.js.map +1 -0
- package/dist/search/pubmed.d.ts +7 -0
- package/dist/search/pubmed.d.ts.map +1 -0
- package/dist/search/pubmed.js +75 -0
- package/dist/search/pubmed.js.map +1 -0
- package/dist/search/searxng.d.ts +1 -1
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +2 -1
- package/dist/search/searxng.js.map +1 -1
- package/dist/search/stackexchange.d.ts +17 -0
- package/dist/search/stackexchange.d.ts.map +1 -0
- package/dist/search/stackexchange.js +70 -0
- package/dist/search/stackexchange.js.map +1 -0
- package/dist/search/wikipedia.d.ts +16 -0
- package/dist/search/wikipedia.d.ts.map +1 -0
- package/dist/search/wikipedia.js +56 -0
- package/dist/search/wikipedia.js.map +1 -0
- package/dist/search.d.ts +1 -0
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +43 -0
- package/dist/search.js.map +1 -1
- package/dist/sessions.d.ts +15 -0
- package/dist/sessions.d.ts.map +1 -1
- package/dist/sessions.js +67 -0
- package/dist/sessions.js.map +1 -1
- package/dist/synthesize.d.ts +6 -1
- package/dist/synthesize.d.ts.map +1 -1
- package/dist/synthesize.js +14 -5
- package/dist/synthesize.js.map +1 -1
- package/package.json +1 -1
package/dist/robots.js
CHANGED
|
@@ -107,18 +107,15 @@ export function parseRobotsTxt(text) {
|
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
109
|
return {
|
|
110
|
-
|
|
110
|
+
// Full rules in file order, each carrying its user-agent. isPathAllowed
|
|
111
|
+
// reads these directly (longest-match per RFC 9309).
|
|
112
|
+
rules: grouped,
|
|
111
113
|
crawlDelaySec: crawlDelay,
|
|
112
|
-
// We stash the grouping by keeping a hidden field. But since we want a
|
|
113
|
-
// clean exported type, bake agent-matching in: we'll re-do the parse at
|
|
114
|
-
// check time. Simpler: re-parse cheaply or store a bigger structure.
|
|
115
|
-
// Actually let's just store the grouped form and compute at check time:
|
|
116
|
-
...{ _grouped: grouped },
|
|
117
114
|
};
|
|
118
115
|
}
|
|
119
116
|
// Exported for unit tests.
|
|
120
117
|
export function isPathAllowed(parsed, path, userAgent) {
|
|
121
|
-
const grouped = parsed.
|
|
118
|
+
const grouped = parsed.rules;
|
|
122
119
|
if (grouped.length === 0)
|
|
123
120
|
return true;
|
|
124
121
|
const ua = userAgent.toLowerCase();
|
package/dist/robots.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,yEAAyE;AACzE,sDAAsD;AACtD,EAAE;AACF,0EAA0E;AAC1E,sEAAsE;AACtE,gEAAgE;
|
|
1
|
+
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,yEAAyE;AACzE,sDAAsD;AACtD,EAAE;AACF,0EAA0E;AAC1E,sEAAsE;AACtE,gEAAgE;AA8BhE,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAC;AAEjD,MAAM,UAAU,iBAAiB;IAC/B,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrD,OAAO;QACL,GAAG,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;QAClC,GAAG,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,IAAqB;IAErB,IAAI,MAAc,CAAC;IACnB,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,OAAO,CAAC;QACtE,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,MAAM,GACV,MAAM,KAAK,SAAS;QAClB,CAAC,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC;QACnC,CAAC,CAAC,MAAM,CAAC;IACb,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM,KAAK,SAAS;QAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEvE,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC,CAAC,4BAA4B;IACnE,OAAO,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAc,EACd,IAAqB;IAErB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM;QACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC,CAAC,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,MAAM,aAAa,EAAE;YAClD,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;YACzC,MAAM;SACP,CAAC,CAAC;QACH,uEAAuE;QACvE,uEAAuE;QACvE,uEAAuE;QACvE,wEAAwE;QACxE,kCAAkC;QAClC,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YACtB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAMlC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,aAAa,GAAa,EAAE,CAAC;IACjC,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,UAA8B,CAAC;IAEnC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,KAAK,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,gBAAgB,EAAE,CAAC;gBACrB,YAAY;gBACZ,aAAa,GAAG,EAAE,CAAC;gBACnB,gBAAgB,GAAG,KAAK,CAAC;YAC3B,CAAC;YACD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACjD,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACrD,gBAAgB,GAAG,IAAI,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,KAAK,EAAE,KAAK,KAAK,OAAO;oBACxB,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,UAAU,GAAG,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,OAAO;QACL,wEAAwE;QACxE,qDAAqD;QACrD,KAAK,EAAE,OAAO;QACd,aAAa,EAAE,UAAU;KAC1B,CAAC;AACJ,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,aAAa,CAC3B,MAAoB,EACpB,IAAY,EACZ,SAAiB;IAEjB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;IAC7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,EAAE,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAEnC,mEAAmE;IACnE,IAAI,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC;IACjF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,+DAA+D;IAC/D,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,mEAAmE;YACnE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAChB,OAAO,GAAG,CAAC,CAAC;oBACZ,SAAS,GAAG,IAAI,CAAC,CAAC,mCAAmC;gBACvD,CAAC;YACH,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;YAAE,SAAS;QAC5C,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/E,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC3B,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,OAAe,EAAE,IAAY;IAChD,yEAAyE;IACzE,0EAA0E;IAC1E,mEAAmE;IACnE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,0EAA0E;IAC1E,2EAA2E;IAC3E,4EAA4E;IAC5E,oEAAoE;IACpE,sEAAsE;IACtE,4EAA4E;IAC5E,0DAA0D;IAC1D,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,SAAS,GAAG,IAAI,CAAC;QACjB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC7C,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,GAAG,CAAC,CAAC;YACV,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACxB,kEAAkE;YAClE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;YACd,CAAC,GAAG,EAAE,KAAK,CAAC;QACd,CAAC;aAAM,CAAC;YACN,OAAO,KAAK,CAAC;QACf,CAAC;QACD,4EAA4E;QAC5E,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IACD,yEAAyE;IACzE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACvC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9C,CAAC;AAED,SAAS,YAAY,CAAC,CAAS;IAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
|
+
export declare class ArxivSearch implements SearchAdapter {
|
|
3
|
+
readonly name = "arxiv";
|
|
4
|
+
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
5
|
+
}
|
|
6
|
+
export declare function parseArxivAtom(xml: string, limit: number): SearchResult[];
|
|
7
|
+
//# sourceMappingURL=arxiv.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arxiv.d.ts","sourceRoot":"","sources":["../../src/search/arxiv.ts"],"names":[],"mappings":"AAWA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAG1F,qBAAa,WAAY,YAAW,aAAa;IAC/C,QAAQ,CAAC,IAAI,WAAW;IAElB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAc1F;AAKD,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAoBzE"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// arXiv search adapter. No API key. Uses the arXiv Atom API
|
|
2
|
+
// (export.arxiv.org/api/query), which returns an Atom feed of matching
|
|
3
|
+
// papers. A fit for research-paper / preprint questions where the planner's
|
|
4
|
+
// sub-queries name a method, model, or result.
|
|
5
|
+
//
|
|
6
|
+
// The response is XML; we parse <entry> blocks with a hand-rolled regex (the
|
|
7
|
+
// same "fix the parser if it breaks, don't reach for a dep" stance the DDG
|
|
8
|
+
// adapter takes). The abstract page URL (<id>) is the kept source; deepdive's
|
|
9
|
+
// fetch step pulls the abstract page, and the PDF path handles the linked PDF
|
|
10
|
+
// if the planner picks it.
|
|
11
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
12
|
+
import { decodeHtmlEntities } from "./duckduckgo.js";
|
|
13
|
+
export class ArxivSearch {
|
|
14
|
+
name = "arxiv";
|
|
15
|
+
async search(query, limit, signal) {
|
|
16
|
+
const url = new URL("https://export.arxiv.org/api/query");
|
|
17
|
+
url.searchParams.set("search_query", `all:${query}`);
|
|
18
|
+
url.searchParams.set("start", "0");
|
|
19
|
+
url.searchParams.set("max_results", String(Math.min(Math.max(limit, 1), 50)));
|
|
20
|
+
url.searchParams.set("sortBy", "relevance");
|
|
21
|
+
const res = await fetch(url, {
|
|
22
|
+
headers: { "user-agent": "deepdive (+https://github.com/askalf/deepdive)" },
|
|
23
|
+
signal: searchTimeoutSignal(signal),
|
|
24
|
+
});
|
|
25
|
+
if (!res.ok)
|
|
26
|
+
throw new Error(`arxiv ${res.status} ${res.statusText}`);
|
|
27
|
+
const xml = await res.text();
|
|
28
|
+
return parseArxivAtom(xml, limit);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
// Exported for unit tests. Pure Atom-feed parser. Pulls title / abstract URL /
|
|
32
|
+
// summary from each <entry>; collapses the whitespace arXiv wraps titles and
|
|
33
|
+
// abstracts with.
|
|
34
|
+
export function parseArxivAtom(xml, limit) {
|
|
35
|
+
const results = [];
|
|
36
|
+
const entryRe = /<entry\b[^>]*>([\s\S]*?)<\/entry>/g;
|
|
37
|
+
let m;
|
|
38
|
+
let rank = 0;
|
|
39
|
+
while ((m = entryRe.exec(xml)) !== null) {
|
|
40
|
+
const entry = m[1];
|
|
41
|
+
const id = tag(entry, "id");
|
|
42
|
+
const url = normalizeAbsUrl(id);
|
|
43
|
+
if (!url)
|
|
44
|
+
continue;
|
|
45
|
+
rank++;
|
|
46
|
+
results.push({
|
|
47
|
+
url,
|
|
48
|
+
title: clean(tag(entry, "title")),
|
|
49
|
+
snippet: clean(tag(entry, "summary")),
|
|
50
|
+
rank,
|
|
51
|
+
});
|
|
52
|
+
if (results.length >= limit)
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
return results;
|
|
56
|
+
}
|
|
57
|
+
function tag(xml, name) {
|
|
58
|
+
const m = new RegExp(`<${name}\\b[^>]*>([\\s\\S]*?)<\\/${name}>`).exec(xml);
|
|
59
|
+
return m ? m[1] : "";
|
|
60
|
+
}
|
|
61
|
+
function clean(s) {
|
|
62
|
+
return decodeHtmlEntities(s).replace(/\s+/g, " ").trim();
|
|
63
|
+
}
|
|
64
|
+
// arXiv <id> is the abstract URL (http://arxiv.org/abs/...). Force https and
|
|
65
|
+
// reject anything that isn't an arxiv.org abstract link.
|
|
66
|
+
function normalizeAbsUrl(id) {
|
|
67
|
+
const raw = id.trim();
|
|
68
|
+
if (!raw)
|
|
69
|
+
return null;
|
|
70
|
+
try {
|
|
71
|
+
const u = new URL(raw);
|
|
72
|
+
if (!/(^|\.)arxiv\.org$/.test(u.hostname))
|
|
73
|
+
return null;
|
|
74
|
+
u.protocol = "https:";
|
|
75
|
+
return u.toString();
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=arxiv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arxiv.js","sourceRoot":"","sources":["../../src/search/arxiv.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,uEAAuE;AACvE,4EAA4E;AAC5E,+CAA+C;AAC/C,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,8EAA8E;AAC9E,8EAA8E;AAC9E,2BAA2B;AAE3B,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,OAAO,WAAW;IACb,IAAI,GAAG,OAAO,CAAC;IAExB,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAC1D,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,KAAK,EAAE,CAAC,CAAC;QACrD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACnC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,gDAAgD,EAAE;YAC3E,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACtE,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7B,OAAO,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;CACF;AAED,+EAA+E;AAC/E,6EAA6E;AAC7E,kBAAkB;AAClB,MAAM,UAAU,cAAc,CAAC,GAAW,EAAE,KAAa;IACvD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,oCAAoC,CAAC;IACrD,IAAI,CAAyB,CAAC;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC5B,MAAM,GAAG,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,EAAE,CAAC;QACP,OAAO,CAAC,IAAI,CAAC;YACX,GAAG;YACH,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACjC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YACrC,IAAI;SACL,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACrC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,GAAG,CAAC,GAAW,EAAE,IAAY;IACpC,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,IAAI,4BAA4B,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5E,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACvB,CAAC;AAED,SAAS,KAAK,CAAC,CAAS;IACtB,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3D,CAAC;AAED,6EAA6E;AAC7E,yDAAyD;AACzD,SAAS,eAAe,CAAC,EAAU;IACjC,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QACvD,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACtB,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/dist/search/brave.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brave.d.ts","sourceRoot":"","sources":["../../src/search/brave.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"brave.d.ts","sourceRoot":"","sources":["../../src/search/brave.ts"],"names":[],"mappings":"AAEA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1F,qBAAa,WAAY,YAAW,aAAa;IAEnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IADhC,QAAQ,CAAC,IAAI,WAAW;gBACK,GAAG,EAAE,MAAM;IAElC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAuB1F"}
|
package/dist/search/brave.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// Brave Search API adapter. Requires DEEPDIVE_BRAVE_KEY.
|
|
2
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
2
3
|
export class BraveSearch {
|
|
3
4
|
key;
|
|
4
5
|
name = "brave";
|
|
@@ -14,7 +15,7 @@ export class BraveSearch {
|
|
|
14
15
|
accept: "application/json",
|
|
15
16
|
"x-subscription-token": this.key,
|
|
16
17
|
},
|
|
17
|
-
signal,
|
|
18
|
+
signal: searchTimeoutSignal(signal),
|
|
18
19
|
});
|
|
19
20
|
if (!res.ok)
|
|
20
21
|
throw new Error(`brave ${res.status} ${res.statusText}`);
|
package/dist/search/brave.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brave.js","sourceRoot":"","sources":["../../src/search/brave.ts"],"names":[],"mappings":"AAAA,yDAAyD;
|
|
1
|
+
{"version":3,"file":"brave.js","sourceRoot":"","sources":["../../src/search/brave.ts"],"names":[],"mappings":"AAAA,yDAAyD;AAEzD,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAE1F,MAAM,OAAO,WAAW;IAEO;IADpB,IAAI,GAAG,OAAO,CAAC;IACxB,YAA6B,GAAW;QAAX,QAAG,GAAH,GAAG,CAAQ;IAAG,CAAC;IAE5C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,gDAAgD,CAAC,CAAC;QACtE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,sBAAsB,EAAE,IAAI,CAAC,GAAG;aACjC;YACD,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAE7B,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1B,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;YACpB,OAAO,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE;YAC5B,IAAI,EAAE,CAAC,GAAG,CAAC;SACZ,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
2
|
export declare class DuckDuckGoSearch implements SearchAdapter {
|
|
3
3
|
readonly name = "duckduckgo";
|
|
4
4
|
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duckduckgo.d.ts","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"duckduckgo.d.ts","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAOA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAI1F,qBAAa,gBAAiB,YAAW,aAAa;IACpD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAiB1F;AAGD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAuB/E;AA+BD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBpD;AAWD,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAE3C"}
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// redirect tracking, and returns { url, title, snippet }.
|
|
5
5
|
//
|
|
6
6
|
// If DDG changes their HTML layout, this breaks. Regenerate parser accordingly.
|
|
7
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
7
8
|
const ENDPOINT = "https://html.duckduckgo.com/html/";
|
|
8
9
|
export class DuckDuckGoSearch {
|
|
9
10
|
name = "duckduckgo";
|
|
@@ -17,7 +18,7 @@ export class DuckDuckGoSearch {
|
|
|
17
18
|
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
18
19
|
},
|
|
19
20
|
body: form.toString(),
|
|
20
|
-
signal,
|
|
21
|
+
signal: searchTimeoutSignal(signal),
|
|
21
22
|
});
|
|
22
23
|
if (!res.ok)
|
|
23
24
|
throw new Error(`duckduckgo ${res.status} ${res.statusText}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duckduckgo.js","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAC7E,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;
|
|
1
|
+
{"version":3,"file":"duckduckgo.js","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAC7E,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;AAEhF,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAE1F,MAAM,QAAQ,GAAG,mCAAmC,CAAC;AAErD,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,YAAY,CAAC;IAE7B,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,mCAAmC;gBACnD,YAAY,EACV,+DAA+D;oBAC/D,oDAAoD;aACvD;YACD,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE;YACrB,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF;AAED,2BAA2B;AAC3B,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAa;IAC7D,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,oEAAoE;IACpE,6DAA6D;IAC7D,MAAM,OAAO,GACX,8JAA8J,CAAC;IAEjK,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACzC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;YAAE,SAAS;QACnC,IAAI,EAAE,CAAC;QACP,OAAO,CAAC,IAAI,CAAC;YACX,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE;YACrD,OAAO,EAAE,SAAS,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE;YACzD,IAAI;SACL,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACrC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,2EAA2E;IAC3E,wCAAwC;IACxC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,QAAQ,GAAG,IAAI,CAAC;IAClD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,gBAAgB,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACxF,IAAI,KAAK,IAAI,CAAC,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,IAAI;gBAAE,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,CAAS;IAC/B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,2EAA2E;AAC3E,4DAA4D;AAC5D,MAAM,UAAU,kBAAkB,CAAC,CAAS;IAC1C,OAAO,CAAC,CAAC,OAAO,CAAC,0CAA0C,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC3E,MAAM,KAAK,GAA2B;YACpC,GAAG,EAAE,GAAG;YACR,EAAE,EAAE,GAAG;YACP,EAAE,EAAE,GAAG;YACP,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG;SACV,CAAC;QACF,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,GAAG,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrE,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAS;IACjC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;AACvD,CAAC;AAED,6EAA6E;AAC7E,wEAAwE;AACxE,iEAAiE;AACjE,2EAA2E;AAC3E,0CAA0C;AAC1C,MAAM,UAAU,SAAS,CAAC,CAAS;IACjC,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
|
+
interface GitHubRepoItem {
|
|
3
|
+
html_url?: string;
|
|
4
|
+
full_name?: string;
|
|
5
|
+
description?: string | null;
|
|
6
|
+
stargazers_count?: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class GitHubSearch implements SearchAdapter {
|
|
9
|
+
private readonly token?;
|
|
10
|
+
readonly name = "github";
|
|
11
|
+
constructor(token?: string | undefined);
|
|
12
|
+
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
13
|
+
}
|
|
14
|
+
export declare function mapGitHubRepos(items: GitHubRepoItem[], limit: number): SearchResult[];
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=github.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/search/github.ts"],"names":[],"mappings":"AAMA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1F,UAAU,cAAc;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,YAAa,YAAW,aAAa;IAEpC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;IADnC,QAAQ,CAAC,IAAI,YAAY;gBACI,KAAK,CAAC,EAAE,MAAM,YAAA;IAErC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAiB1F;AAID,wBAAgB,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAUrF"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// GitHub repository search adapter. Works without a key (60 search
|
|
2
|
+
// requests/hour unauthenticated); set DEEPDIVE_GITHUB_TOKEN for the
|
|
3
|
+
// authenticated rate limit (30 req/min). A fit for "what library / project
|
|
4
|
+
// does X" sub-queries — the kept source is the repo's GitHub page, whose
|
|
5
|
+
// rendered README deepdive's fetch step then extracts.
|
|
6
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
7
|
+
export class GitHubSearch {
|
|
8
|
+
token;
|
|
9
|
+
name = "github";
|
|
10
|
+
constructor(token) {
|
|
11
|
+
this.token = token;
|
|
12
|
+
}
|
|
13
|
+
async search(query, limit, signal) {
|
|
14
|
+
const url = new URL("https://api.github.com/search/repositories");
|
|
15
|
+
url.searchParams.set("q", query);
|
|
16
|
+
url.searchParams.set("per_page", String(Math.min(Math.max(limit, 1), 100)));
|
|
17
|
+
url.searchParams.set("sort", "best-match");
|
|
18
|
+
const headers = {
|
|
19
|
+
accept: "application/vnd.github+json",
|
|
20
|
+
"x-github-api-version": "2022-11-28",
|
|
21
|
+
// GitHub rejects requests with no User-Agent.
|
|
22
|
+
"user-agent": "deepdive (+https://github.com/askalf/deepdive)",
|
|
23
|
+
};
|
|
24
|
+
if (this.token)
|
|
25
|
+
headers.authorization = `Bearer ${this.token}`;
|
|
26
|
+
const res = await fetch(url, { headers, signal: searchTimeoutSignal(signal) });
|
|
27
|
+
if (!res.ok)
|
|
28
|
+
throw new Error(`github ${res.status} ${res.statusText}`);
|
|
29
|
+
const json = (await res.json());
|
|
30
|
+
return mapGitHubRepos(json.items ?? [], limit);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
// Exported for unit tests. Pure mapper. Snippet is the repo description with a
|
|
34
|
+
// star count appended when known, so the planner/critic can weigh popularity.
|
|
35
|
+
export function mapGitHubRepos(items, limit) {
|
|
36
|
+
return items
|
|
37
|
+
.filter((r) => typeof r.html_url === "string" && r.html_url.length > 0)
|
|
38
|
+
.slice(0, limit)
|
|
39
|
+
.map((r, i) => ({
|
|
40
|
+
url: r.html_url,
|
|
41
|
+
title: r.full_name ?? r.html_url ?? "",
|
|
42
|
+
snippet: buildSnippet(r),
|
|
43
|
+
rank: i + 1,
|
|
44
|
+
}));
|
|
45
|
+
}
|
|
46
|
+
function buildSnippet(r) {
|
|
47
|
+
const desc = (r.description ?? "").trim();
|
|
48
|
+
if (typeof r.stargazers_count === "number" && r.stargazers_count > 0) {
|
|
49
|
+
const stars = `★ ${formatStars(r.stargazers_count)}`;
|
|
50
|
+
return desc ? `${desc} (${stars})` : stars;
|
|
51
|
+
}
|
|
52
|
+
return desc;
|
|
53
|
+
}
|
|
54
|
+
function formatStars(n) {
|
|
55
|
+
if (n >= 1000)
|
|
56
|
+
return `${(n / 1000).toFixed(n >= 10000 ? 0 : 1)}k`;
|
|
57
|
+
return String(n);
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=github.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/search/github.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,oEAAoE;AACpE,2EAA2E;AAC3E,yEAAyE;AACzE,uDAAuD;AAEvD,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAS1F,MAAM,OAAO,YAAY;IAEM;IADpB,IAAI,GAAG,QAAQ,CAAC;IACzB,YAA6B,KAAc;QAAd,UAAK,GAAL,KAAK,CAAS;IAAG,CAAC;IAE/C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAClE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC3C,MAAM,OAAO,GAA2B;YACtC,MAAM,EAAE,6BAA6B;YACrC,sBAAsB,EAAE,YAAY;YACpC,8CAA8C;YAC9C,YAAY,EAAE,gDAAgD;SAC/D,CAAC;QACF,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,aAAa,GAAG,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/E,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACvE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAiC,CAAC;QAChE,OAAO,cAAc,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;CACF;AAED,+EAA+E;AAC/E,8EAA8E;AAC9E,MAAM,UAAU,cAAc,CAAC,KAAuB,EAAE,KAAa;IACnE,OAAO,KAAK;SACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;SACtE,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACd,GAAG,EAAE,CAAC,CAAC,QAAkB;QACzB,KAAK,EAAE,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,QAAQ,IAAI,EAAE;QACtC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACxB,IAAI,EAAE,CAAC,GAAG,CAAC;KACZ,CAAC,CAAC,CAAC;AACR,CAAC;AAED,SAAS,YAAY,CAAC,CAAiB;IACrC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,IAAI,OAAO,CAAC,CAAC,gBAAgB,KAAK,QAAQ,IAAI,CAAC,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACrE,MAAM,KAAK,GAAG,KAAK,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACnE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
|
+
interface HNHit {
|
|
3
|
+
objectID?: string;
|
|
4
|
+
title?: string | null;
|
|
5
|
+
url?: string | null;
|
|
6
|
+
points?: number | null;
|
|
7
|
+
num_comments?: number | null;
|
|
8
|
+
author?: string | null;
|
|
9
|
+
}
|
|
10
|
+
export declare class HackerNewsSearch implements SearchAdapter {
|
|
11
|
+
readonly name = "hackernews";
|
|
12
|
+
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
13
|
+
}
|
|
14
|
+
export declare function mapHNHits(hits: HNHit[], limit: number): SearchResult[];
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=hackernews.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hackernews.d.ts","sourceRoot":"","sources":["../../src/search/hackernews.ts"],"names":[],"mappings":"AAMA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1F,UAAU,KAAK;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,qBAAa,gBAAiB,YAAW,aAAa;IACpD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAa1F;AAID,wBAAgB,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAUtE"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Hacker News search adapter. No API key. Uses the Algolia-hosted HN Search
|
|
2
|
+
// API (hn.algolia.com). A fit for "what does the community think of X",
|
|
3
|
+
// release discussions, and primary-source threads. Defaults to story results;
|
|
4
|
+
// the kept source is the story's target URL (or the HN thread itself for
|
|
5
|
+
// Ask/Show HN posts with no external link).
|
|
6
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
7
|
+
export class HackerNewsSearch {
|
|
8
|
+
name = "hackernews";
|
|
9
|
+
async search(query, limit, signal) {
|
|
10
|
+
const url = new URL("https://hn.algolia.com/api/v1/search");
|
|
11
|
+
url.searchParams.set("query", query);
|
|
12
|
+
url.searchParams.set("tags", "story");
|
|
13
|
+
url.searchParams.set("hitsPerPage", String(Math.min(Math.max(limit, 1), 50)));
|
|
14
|
+
const res = await fetch(url, {
|
|
15
|
+
headers: { accept: "application/json", "user-agent": "deepdive (+https://github.com/askalf/deepdive)" },
|
|
16
|
+
signal: searchTimeoutSignal(signal),
|
|
17
|
+
});
|
|
18
|
+
if (!res.ok)
|
|
19
|
+
throw new Error(`hackernews ${res.status} ${res.statusText}`);
|
|
20
|
+
const json = (await res.json());
|
|
21
|
+
return mapHNHits(json.hits ?? [], limit);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// Exported for unit tests. Pure mapper. Ask/Show HN posts have a null `url`;
|
|
25
|
+
// fall back to the HN thread page so there's always something to fetch.
|
|
26
|
+
export function mapHNHits(hits, limit) {
|
|
27
|
+
return hits
|
|
28
|
+
.filter((h) => (h.title ?? "").length > 0 && (h.url || h.objectID))
|
|
29
|
+
.slice(0, limit)
|
|
30
|
+
.map((h, i) => ({
|
|
31
|
+
url: h.url || `https://news.ycombinator.com/item?id=${h.objectID}`,
|
|
32
|
+
title: h.title,
|
|
33
|
+
snippet: hnSnippet(h),
|
|
34
|
+
rank: i + 1,
|
|
35
|
+
}));
|
|
36
|
+
}
|
|
37
|
+
function hnSnippet(h) {
|
|
38
|
+
const parts = [];
|
|
39
|
+
if (typeof h.points === "number")
|
|
40
|
+
parts.push(`${h.points} points`);
|
|
41
|
+
if (typeof h.num_comments === "number")
|
|
42
|
+
parts.push(`${h.num_comments} comments`);
|
|
43
|
+
if (h.author)
|
|
44
|
+
parts.push(`by ${h.author}`);
|
|
45
|
+
return parts.join(" · ");
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=hackernews.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hackernews.js","sourceRoot":"","sources":["../../src/search/hackernews.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,wEAAwE;AACxE,8EAA8E;AAC9E,yEAAyE;AACzE,4CAA4C;AAE5C,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAW1F,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,YAAY,CAAC;IAE7B,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,sCAAsC,CAAC,CAAC;QAC5D,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACrC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,YAAY,EAAE,gDAAgD,EAAE;YACvG,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAuB,CAAC;QACtD,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,6EAA6E;AAC7E,wEAAwE;AACxE,MAAM,UAAU,SAAS,CAAC,IAAa,EAAE,KAAa;IACpD,OAAO,IAAI;SACR,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC;SAClE,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACd,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,wCAAwC,CAAC,CAAC,QAAQ,EAAE;QAClE,KAAK,EAAE,CAAC,CAAC,KAAe;QACxB,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;QACrB,IAAI,EAAE,CAAC,GAAG,CAAC;KACZ,CAAC,CAAC,CAAC;AACR,CAAC;AAED,SAAS,SAAS,CAAC,CAAQ;IACzB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC;IACnE,IAAI,OAAO,CAAC,CAAC,YAAY,KAAK,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,YAAY,WAAW,CAAC,CAAC;IACjF,IAAI,CAAC,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
|
+
export declare class PubMedSearch implements SearchAdapter {
|
|
3
|
+
readonly name = "pubmed";
|
|
4
|
+
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
5
|
+
}
|
|
6
|
+
export declare function mapPubMedSummary(result: Record<string, unknown>, ids: string[], limit: number): SearchResult[];
|
|
7
|
+
//# sourceMappingURL=pubmed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pubmed.d.ts","sourceRoot":"","sources":["../../src/search/pubmed.ts"],"names":[],"mappings":"AAMA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAY1F,qBAAa,YAAa,YAAW,aAAa;IAChD,QAAQ,CAAC,IAAI,YAAY;IAEnB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CA+B1F;AAKD,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,GAAG,EAAE,MAAM,EAAE,EACb,KAAK,EAAE,MAAM,GACZ,YAAY,EAAE,CAchB"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// PubMed search adapter. No API key (NCBI E-utilities allow ~3 req/s
|
|
2
|
+
// unauthenticated). Two-step: esearch returns matching PMIDs, esummary
|
|
3
|
+
// returns their metadata. A fit for biomedical / life-sciences questions
|
|
4
|
+
// where peer-reviewed literature is the authoritative source. The kept source
|
|
5
|
+
// is the PubMed abstract page.
|
|
6
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
7
|
+
const BASE = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils";
|
|
8
|
+
export class PubMedSearch {
|
|
9
|
+
name = "pubmed";
|
|
10
|
+
async search(query, limit, signal) {
|
|
11
|
+
const n = Math.min(Math.max(limit, 1), 50);
|
|
12
|
+
const sig = searchTimeoutSignal(signal);
|
|
13
|
+
const esearch = new URL(`${BASE}/esearch.fcgi`);
|
|
14
|
+
esearch.searchParams.set("db", "pubmed");
|
|
15
|
+
esearch.searchParams.set("term", query);
|
|
16
|
+
esearch.searchParams.set("retmax", String(n));
|
|
17
|
+
esearch.searchParams.set("retmode", "json");
|
|
18
|
+
esearch.searchParams.set("sort", "relevance");
|
|
19
|
+
const sres = await fetch(esearch, {
|
|
20
|
+
headers: { accept: "application/json", "user-agent": "deepdive (+https://github.com/askalf/deepdive)" },
|
|
21
|
+
signal: sig,
|
|
22
|
+
});
|
|
23
|
+
if (!sres.ok)
|
|
24
|
+
throw new Error(`pubmed esearch ${sres.status} ${sres.statusText}`);
|
|
25
|
+
const sjson = (await sres.json());
|
|
26
|
+
const ids = sjson.esearchresult?.idlist ?? [];
|
|
27
|
+
if (ids.length === 0)
|
|
28
|
+
return [];
|
|
29
|
+
const esummary = new URL(`${BASE}/esummary.fcgi`);
|
|
30
|
+
esummary.searchParams.set("db", "pubmed");
|
|
31
|
+
esummary.searchParams.set("id", ids.join(","));
|
|
32
|
+
esummary.searchParams.set("retmode", "json");
|
|
33
|
+
const ures = await fetch(esummary, {
|
|
34
|
+
headers: { accept: "application/json", "user-agent": "deepdive (+https://github.com/askalf/deepdive)" },
|
|
35
|
+
signal: sig,
|
|
36
|
+
});
|
|
37
|
+
if (!ures.ok)
|
|
38
|
+
throw new Error(`pubmed esummary ${ures.status} ${ures.statusText}`);
|
|
39
|
+
const ujson = (await ures.json());
|
|
40
|
+
return mapPubMedSummary(ujson.result ?? {}, ids, limit);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
// Exported for unit tests. Pure mapper over the esummary `result` map, in the
|
|
44
|
+
// PMID order esearch returned. Builds the abstract URL and a journal/date/
|
|
45
|
+
// author snippet.
|
|
46
|
+
export function mapPubMedSummary(result, ids, limit) {
|
|
47
|
+
const out = [];
|
|
48
|
+
for (const id of ids) {
|
|
49
|
+
if (out.length >= limit)
|
|
50
|
+
break;
|
|
51
|
+
const doc = result[id];
|
|
52
|
+
if (!doc || typeof doc.title !== "string" || doc.title.length === 0)
|
|
53
|
+
continue;
|
|
54
|
+
out.push({
|
|
55
|
+
url: `https://pubmed.ncbi.nlm.nih.gov/${id}/`,
|
|
56
|
+
title: doc.title.replace(/\.$/, ""),
|
|
57
|
+
snippet: pubmedSnippet(doc),
|
|
58
|
+
rank: out.length + 1,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
function pubmedSnippet(doc) {
|
|
64
|
+
const parts = [];
|
|
65
|
+
const authors = (doc.authors ?? []).map((a) => a.name).filter(Boolean);
|
|
66
|
+
if (authors.length > 0) {
|
|
67
|
+
parts.push(authors.length > 3 ? `${authors.slice(0, 3).join(", ")}, et al.` : authors.join(", "));
|
|
68
|
+
}
|
|
69
|
+
if (doc.source)
|
|
70
|
+
parts.push(doc.source);
|
|
71
|
+
if (doc.pubdate)
|
|
72
|
+
parts.push(doc.pubdate);
|
|
73
|
+
return parts.join(" · ");
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=pubmed.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pubmed.js","sourceRoot":"","sources":["../../src/search/pubmed.ts"],"names":[],"mappings":"AAAA,qEAAqE;AACrE,uEAAuE;AACvE,yEAAyE;AACzE,8EAA8E;AAC9E,+BAA+B;AAE/B,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAE1F,MAAM,IAAI,GAAG,+CAA+C,CAAC;AAU7D,MAAM,OAAO,YAAY;IACd,IAAI,GAAG,QAAQ,CAAC;IAEzB,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAExC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,eAAe,CAAC,CAAC;QAChD,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC5C,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAChC,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,YAAY,EAAE,gDAAgD,EAAE;YACvG,MAAM,EAAE,GAAG;SACZ,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA8C,CAAC;QAC/E,MAAM,GAAG,GAAG,KAAK,CAAC,aAAa,EAAE,MAAM,IAAI,EAAE,CAAC;QAC9C,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEhC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,gBAAgB,CAAC,CAAC;QAClD,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1C,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/C,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YACjC,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,YAAY,EAAE,gDAAgD,EAAE;YACvG,MAAM,EAAE,GAAG;SACZ,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACnF,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAyC,CAAC;QAC1E,OAAO,gBAAgB,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC;CACF;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,kBAAkB;AAClB,MAAM,UAAU,gBAAgB,CAC9B,MAA+B,EAC/B,GAAa,EACb,KAAa;IAEb,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,GAAG,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;QAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAA4B,CAAC;QAClD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9E,GAAG,CAAC,IAAI,CAAC;YACP,GAAG,EAAE,mCAAmC,EAAE,GAAG;YAC7C,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;YACnC,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC;YAC3B,IAAI,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC;SACrB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,GAAgB;IACrC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACvE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpG,CAAC;IACD,IAAI,GAAG,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,GAAG,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC"}
|
package/dist/search/searxng.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"searxng.d.ts","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"searxng.d.ts","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAGA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAG1F,qBAAa,aAAc,YAAW,aAAa;IAErC,OAAO,CAAC,QAAQ,CAAC,OAAO;IADpC,QAAQ,CAAC,IAAI,aAAa;gBACG,OAAO,EAAE,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAoB1F"}
|
package/dist/search/searxng.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// SearXNG adapter. Points at an existing SearXNG instance (self-hosted or
|
|
2
2
|
// public). Requires DEEPDIVE_SEARXNG_URL. Uses the JSON output format.
|
|
3
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
3
4
|
import { trimTrailingSlashes } from "../url-util.js";
|
|
4
5
|
export class SearXNGSearch {
|
|
5
6
|
baseUrl;
|
|
@@ -13,7 +14,7 @@ export class SearXNGSearch {
|
|
|
13
14
|
url.searchParams.set("format", "json");
|
|
14
15
|
const res = await fetch(url, {
|
|
15
16
|
headers: { accept: "application/json" },
|
|
16
|
-
signal,
|
|
17
|
+
signal: searchTimeoutSignal(signal),
|
|
17
18
|
});
|
|
18
19
|
if (!res.ok)
|
|
19
20
|
throw new Error(`searxng ${res.status} ${res.statusText}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"searxng.js","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;
|
|
1
|
+
{"version":3,"file":"searxng.js","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;AAEvE,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAC1F,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,OAAO,aAAa;IAEK;IADpB,IAAI,GAAG,SAAS,CAAC;IAC1B,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEhD,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;QACnE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;YACvC,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAE7B,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;YACxB,IAAI,EAAE,CAAC,GAAG,CAAC;SACZ,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
|
+
interface SEItem {
|
|
3
|
+
title?: string;
|
|
4
|
+
link?: string;
|
|
5
|
+
score?: number;
|
|
6
|
+
answer_count?: number;
|
|
7
|
+
is_answered?: boolean;
|
|
8
|
+
}
|
|
9
|
+
export declare class StackExchangeSearch implements SearchAdapter {
|
|
10
|
+
private readonly site;
|
|
11
|
+
readonly name = "stackexchange";
|
|
12
|
+
constructor(site?: string);
|
|
13
|
+
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
14
|
+
}
|
|
15
|
+
export declare function mapStackExchangeItems(items: SEItem[], limit: number): SearchResult[];
|
|
16
|
+
export {};
|
|
17
|
+
//# sourceMappingURL=stackexchange.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stackexchange.d.ts","sourceRoot":"","sources":["../../src/search/stackexchange.ts"],"names":[],"mappings":"AAMA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1F,UAAU,MAAM;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,qBAAa,mBAAoB,YAAW,aAAa;IAE3C,OAAO,CAAC,QAAQ,CAAC,IAAI;IADjC,QAAQ,CAAC,IAAI,mBAAmB;gBACH,IAAI,GAAE,MAAwB;IAErD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAgB1F;AAID,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAUpF"}
|