@askalf/deepdive 0.13.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -1
- package/dist/agent.d.ts +2 -1
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +36 -13
- package/dist/agent.js.map +1 -1
- package/dist/browser.d.ts +1 -0
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +23 -5
- package/dist/browser.js.map +1 -1
- package/dist/cache.d.ts.map +1 -1
- package/dist/cache.js +22 -2
- package/dist/cache.js.map +1 -1
- package/dist/citations.d.ts +1 -0
- package/dist/citations.d.ts.map +1 -1
- package/dist/citations.js +4 -1
- package/dist/citations.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +313 -17
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +3 -0
- package/dist/completion.d.ts.map +1 -0
- package/dist/completion.js +110 -0
- package/dist/completion.js.map +1 -0
- package/dist/confidence.d.ts +17 -0
- package/dist/confidence.d.ts.map +1 -0
- package/dist/confidence.js +41 -0
- package/dist/confidence.js.map +1 -0
- package/dist/config-file.d.ts +13 -0
- package/dist/config-file.d.ts.map +1 -0
- package/dist/config-file.js +149 -0
- package/dist/config-file.js.map +1 -0
- package/dist/config.d.ts +9 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +3 -0
- package/dist/config.js.map +1 -1
- package/dist/dates.d.ts +8 -0
- package/dist/dates.d.ts.map +1 -0
- package/dist/dates.js +195 -0
- package/dist/dates.js.map +1 -0
- package/dist/diff.d.ts +46 -0
- package/dist/diff.d.ts.map +1 -0
- package/dist/diff.js +227 -0
- package/dist/diff.js.map +1 -0
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +12 -6
- package/dist/doctor.js.map +1 -1
- package/dist/html-export.d.ts +6 -0
- package/dist/html-export.d.ts.map +1 -0
- package/dist/html-export.js +146 -0
- package/dist/html-export.js.map +1 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +7 -0
- package/dist/markdown.d.ts.map +1 -0
- package/dist/markdown.js +232 -0
- package/dist/markdown.js.map +1 -0
- package/dist/profiles.d.ts +5 -0
- package/dist/profiles.d.ts.map +1 -0
- package/dist/profiles.js +38 -0
- package/dist/profiles.js.map +1 -0
- package/dist/robots.d.ts +1 -0
- package/dist/robots.d.ts.map +1 -1
- package/dist/robots.js +4 -7
- package/dist/robots.js.map +1 -1
- package/dist/search/arxiv.d.ts +7 -0
- package/dist/search/arxiv.d.ts.map +1 -0
- package/dist/search/arxiv.js +81 -0
- package/dist/search/arxiv.js.map +1 -0
- package/dist/search/brave.d.ts +1 -1
- package/dist/search/brave.d.ts.map +1 -1
- package/dist/search/brave.js +2 -1
- package/dist/search/brave.js.map +1 -1
- package/dist/search/duckduckgo.d.ts +1 -1
- package/dist/search/duckduckgo.d.ts.map +1 -1
- package/dist/search/duckduckgo.js +2 -1
- package/dist/search/duckduckgo.js.map +1 -1
- package/dist/search/github.d.ts +16 -0
- package/dist/search/github.d.ts.map +1 -0
- package/dist/search/github.js +59 -0
- package/dist/search/github.js.map +1 -0
- package/dist/search/searxng.d.ts +1 -1
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +2 -1
- package/dist/search/searxng.js.map +1 -1
- package/dist/search/wikipedia.d.ts +16 -0
- package/dist/search/wikipedia.d.ts.map +1 -0
- package/dist/search/wikipedia.js +56 -0
- package/dist/search/wikipedia.js.map +1 -0
- package/dist/search.d.ts +1 -0
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +26 -0
- package/dist/search.js.map +1 -1
- package/dist/sessions.d.ts +15 -0
- package/dist/sessions.d.ts.map +1 -1
- package/dist/sessions.js +67 -0
- package/dist/sessions.js.map +1 -1
- package/dist/synthesize.d.ts +6 -1
- package/dist/synthesize.d.ts.map +1 -1
- package/dist/synthesize.js +14 -5
- package/dist/synthesize.js.map +1 -1
- package/package.json +1 -1
package/dist/robots.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,yEAAyE;AACzE,sDAAsD;AACtD,EAAE;AACF,0EAA0E;AAC1E,sEAAsE;AACtE,gEAAgE;
|
|
1
|
+
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,yEAAyE;AACzE,sDAAsD;AACtD,EAAE;AACF,0EAA0E;AAC1E,sEAAsE;AACtE,gEAAgE;AA8BhE,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAC;AAEjD,MAAM,UAAU,iBAAiB;IAC/B,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrD,OAAO;QACL,GAAG,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;QAClC,GAAG,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,IAAqB;IAErB,IAAI,MAAc,CAAC;IACnB,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,OAAO,CAAC;QACtE,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,MAAM,GACV,MAAM,KAAK,SAAS;QAClB,CAAC,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC;QACnC,CAAC,CAAC,MAAM,CAAC;IACb,IAAI,IAAI,CAAC,KAAK,IAAI,MAAM,KAAK,SAAS;QAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEvE,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC,CAAC,4BAA4B;IACnE,OAAO,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAc,EACd,IAAqB;IAErB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM;QACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC,CAAC,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,MAAM,aAAa,EAAE;YAClD,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;YACzC,MAAM;SACP,CAAC,CAAC;QACH,uEAAuE;QACvE,uEAAuE;QACvE,uEAAuE;QACvE,wEAAwE;QACxE,kCAAkC;QAClC,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YACtB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACvB,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAMlC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,aAAa,GAAa,EAAE,CAAC;IACjC,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,UAA8B,CAAC;IAEnC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,KAAK,GAAG,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,gBAAgB,EAAE,CAAC;gBACrB,YAAY;gBACZ,aAAa,GAAG,EAAE,CAAC;gBACnB,gBAAgB,GAAG,KAAK,CAAC;YAC3B,CAAC;YACD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACjD,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACrD,gBAAgB,GAAG,IAAI,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,KAAK,EAAE,KAAK,KAAK,OAAO;oBACxB,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,UAAU,GAAG,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,OAAO;QACL,wEAAwE;QACxE,qDAAqD;QACrD,KAAK,EAAE,OAAO;QACd,aAAa,EAAE,UAAU;KAC1B,CAAC;AACJ,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,aAAa,CAC3B,MAAoB,EACpB,IAAY,EACZ,SAAiB;IAEjB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;IAC7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,EAAE,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAEnC,mEAAmE;IACnE,IAAI,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC;IACjF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,+DAA+D;IAC/D,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,mEAAmE;YACnE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAChB,OAAO,GAAG,CAAC,CAAC;oBACZ,SAAS,GAAG,IAAI,CAAC,CAAC,mCAAmC;gBACvD,CAAC;YACH,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;YAAE,SAAS;QAC5C,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/E,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC3B,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,OAAe,EAAE,IAAY;IAChD,yEAAyE;IACzE,0EAA0E;IAC1E,mEAAmE;IACnE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,0EAA0E;IAC1E,2EAA2E;IAC3E,4EAA4E;IAC5E,oEAAoE;IACpE,sEAAsE;IACtE,4EAA4E;IAC5E,0DAA0D;IAC1D,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,SAAS,GAAG,IAAI,CAAC;QACjB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC7C,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,GAAG,CAAC,CAAC;YACV,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACxB,kEAAkE;YAClE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;YACd,CAAC,GAAG,EAAE,KAAK,CAAC;QACd,CAAC;aAAM,CAAC;YACN,OAAO,KAAK,CAAC;QACf,CAAC;QACD,4EAA4E;QAC5E,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IACD,yEAAyE;IACzE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACvC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9C,CAAC;AAED,SAAS,YAAY,CAAC,CAAS;IAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
|
+
export declare class ArxivSearch implements SearchAdapter {
|
|
3
|
+
readonly name = "arxiv";
|
|
4
|
+
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
5
|
+
}
|
|
6
|
+
export declare function parseArxivAtom(xml: string, limit: number): SearchResult[];
|
|
7
|
+
//# sourceMappingURL=arxiv.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arxiv.d.ts","sourceRoot":"","sources":["../../src/search/arxiv.ts"],"names":[],"mappings":"AAWA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAG1F,qBAAa,WAAY,YAAW,aAAa;IAC/C,QAAQ,CAAC,IAAI,WAAW;IAElB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAc1F;AAKD,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAoBzE"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// arXiv search adapter. No API key. Uses the arXiv Atom API
|
|
2
|
+
// (export.arxiv.org/api/query), which returns an Atom feed of matching
|
|
3
|
+
// papers. A fit for research-paper / preprint questions where the planner's
|
|
4
|
+
// sub-queries name a method, model, or result.
|
|
5
|
+
//
|
|
6
|
+
// The response is XML; we parse <entry> blocks with a hand-rolled regex (the
|
|
7
|
+
// same "fix the parser if it breaks, don't reach for a dep" stance the DDG
|
|
8
|
+
// adapter takes). The abstract page URL (<id>) is the kept source; deepdive's
|
|
9
|
+
// fetch step pulls the abstract page, and the PDF path handles the linked PDF
|
|
10
|
+
// if the planner picks it.
|
|
11
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
12
|
+
import { decodeHtmlEntities } from "./duckduckgo.js";
|
|
13
|
+
export class ArxivSearch {
|
|
14
|
+
name = "arxiv";
|
|
15
|
+
async search(query, limit, signal) {
|
|
16
|
+
const url = new URL("https://export.arxiv.org/api/query");
|
|
17
|
+
url.searchParams.set("search_query", `all:${query}`);
|
|
18
|
+
url.searchParams.set("start", "0");
|
|
19
|
+
url.searchParams.set("max_results", String(Math.min(Math.max(limit, 1), 50)));
|
|
20
|
+
url.searchParams.set("sortBy", "relevance");
|
|
21
|
+
const res = await fetch(url, {
|
|
22
|
+
headers: { "user-agent": "deepdive (+https://github.com/askalf/deepdive)" },
|
|
23
|
+
signal: searchTimeoutSignal(signal),
|
|
24
|
+
});
|
|
25
|
+
if (!res.ok)
|
|
26
|
+
throw new Error(`arxiv ${res.status} ${res.statusText}`);
|
|
27
|
+
const xml = await res.text();
|
|
28
|
+
return parseArxivAtom(xml, limit);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
// Exported for unit tests. Pure Atom-feed parser. Pulls title / abstract URL /
|
|
32
|
+
// summary from each <entry>; collapses the whitespace arXiv wraps titles and
|
|
33
|
+
// abstracts with.
|
|
34
|
+
export function parseArxivAtom(xml, limit) {
|
|
35
|
+
const results = [];
|
|
36
|
+
const entryRe = /<entry\b[^>]*>([\s\S]*?)<\/entry>/g;
|
|
37
|
+
let m;
|
|
38
|
+
let rank = 0;
|
|
39
|
+
while ((m = entryRe.exec(xml)) !== null) {
|
|
40
|
+
const entry = m[1];
|
|
41
|
+
const id = tag(entry, "id");
|
|
42
|
+
const url = normalizeAbsUrl(id);
|
|
43
|
+
if (!url)
|
|
44
|
+
continue;
|
|
45
|
+
rank++;
|
|
46
|
+
results.push({
|
|
47
|
+
url,
|
|
48
|
+
title: clean(tag(entry, "title")),
|
|
49
|
+
snippet: clean(tag(entry, "summary")),
|
|
50
|
+
rank,
|
|
51
|
+
});
|
|
52
|
+
if (results.length >= limit)
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
return results;
|
|
56
|
+
}
|
|
57
|
+
function tag(xml, name) {
|
|
58
|
+
const m = new RegExp(`<${name}\\b[^>]*>([\\s\\S]*?)<\\/${name}>`).exec(xml);
|
|
59
|
+
return m ? m[1] : "";
|
|
60
|
+
}
|
|
61
|
+
function clean(s) {
|
|
62
|
+
return decodeHtmlEntities(s).replace(/\s+/g, " ").trim();
|
|
63
|
+
}
|
|
64
|
+
// arXiv <id> is the abstract URL (http://arxiv.org/abs/...). Force https and
|
|
65
|
+
// reject anything that isn't an arxiv.org abstract link.
|
|
66
|
+
function normalizeAbsUrl(id) {
|
|
67
|
+
const raw = id.trim();
|
|
68
|
+
if (!raw)
|
|
69
|
+
return null;
|
|
70
|
+
try {
|
|
71
|
+
const u = new URL(raw);
|
|
72
|
+
if (!/(^|\.)arxiv\.org$/.test(u.hostname))
|
|
73
|
+
return null;
|
|
74
|
+
u.protocol = "https:";
|
|
75
|
+
return u.toString();
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=arxiv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arxiv.js","sourceRoot":"","sources":["../../src/search/arxiv.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,uEAAuE;AACvE,4EAA4E;AAC5E,+CAA+C;AAC/C,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,8EAA8E;AAC9E,8EAA8E;AAC9E,2BAA2B;AAE3B,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,OAAO,WAAW;IACb,IAAI,GAAG,OAAO,CAAC;IAExB,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAC1D,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,KAAK,EAAE,CAAC,CAAC;QACrD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACnC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,gDAAgD,EAAE;YAC3E,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACtE,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7B,OAAO,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;CACF;AAED,+EAA+E;AAC/E,6EAA6E;AAC7E,kBAAkB;AAClB,MAAM,UAAU,cAAc,CAAC,GAAW,EAAE,KAAa;IACvD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,oCAAoC,CAAC;IACrD,IAAI,CAAyB,CAAC;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC5B,MAAM,GAAG,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,EAAE,CAAC;QACP,OAAO,CAAC,IAAI,CAAC;YACX,GAAG;YACH,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACjC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YACrC,IAAI;SACL,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACrC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,GAAG,CAAC,GAAW,EAAE,IAAY;IACpC,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,IAAI,4BAA4B,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5E,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACvB,CAAC;AAED,SAAS,KAAK,CAAC,CAAS;IACtB,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3D,CAAC;AAED,6EAA6E;AAC7E,yDAAyD;AACzD,SAAS,eAAe,CAAC,EAAU;IACjC,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QACvD,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACtB,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/dist/search/brave.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brave.d.ts","sourceRoot":"","sources":["../../src/search/brave.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"brave.d.ts","sourceRoot":"","sources":["../../src/search/brave.ts"],"names":[],"mappings":"AAEA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1F,qBAAa,WAAY,YAAW,aAAa;IAEnC,OAAO,CAAC,QAAQ,CAAC,GAAG;IADhC,QAAQ,CAAC,IAAI,WAAW;gBACK,GAAG,EAAE,MAAM;IAElC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAuB1F"}
|
package/dist/search/brave.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// Brave Search API adapter. Requires DEEPDIVE_BRAVE_KEY.
|
|
2
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
2
3
|
export class BraveSearch {
|
|
3
4
|
key;
|
|
4
5
|
name = "brave";
|
|
@@ -14,7 +15,7 @@ export class BraveSearch {
|
|
|
14
15
|
accept: "application/json",
|
|
15
16
|
"x-subscription-token": this.key,
|
|
16
17
|
},
|
|
17
|
-
signal,
|
|
18
|
+
signal: searchTimeoutSignal(signal),
|
|
18
19
|
});
|
|
19
20
|
if (!res.ok)
|
|
20
21
|
throw new Error(`brave ${res.status} ${res.statusText}`);
|
package/dist/search/brave.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"brave.js","sourceRoot":"","sources":["../../src/search/brave.ts"],"names":[],"mappings":"AAAA,yDAAyD;
|
|
1
|
+
{"version":3,"file":"brave.js","sourceRoot":"","sources":["../../src/search/brave.ts"],"names":[],"mappings":"AAAA,yDAAyD;AAEzD,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAE1F,MAAM,OAAO,WAAW;IAEO;IADpB,IAAI,GAAG,OAAO,CAAC;IACxB,YAA6B,GAAW;QAAX,QAAG,GAAH,GAAG,CAAQ;IAAG,CAAC;IAE5C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,gDAAgD,CAAC,CAAC;QACtE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,sBAAsB,EAAE,IAAI,CAAC,GAAG;aACjC;YACD,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAE7B,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1B,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;YACpB,OAAO,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE;YAC5B,IAAI,EAAE,CAAC,GAAG,CAAC;SACZ,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
2
|
export declare class DuckDuckGoSearch implements SearchAdapter {
|
|
3
3
|
readonly name = "duckduckgo";
|
|
4
4
|
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duckduckgo.d.ts","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"duckduckgo.d.ts","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAOA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAI1F,qBAAa,gBAAiB,YAAW,aAAa;IACpD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAiB1F;AAGD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAuB/E;AA+BD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBpD;AAWD,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAE3C"}
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// redirect tracking, and returns { url, title, snippet }.
|
|
5
5
|
//
|
|
6
6
|
// If DDG changes their HTML layout, this breaks. Regenerate parser accordingly.
|
|
7
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
7
8
|
const ENDPOINT = "https://html.duckduckgo.com/html/";
|
|
8
9
|
export class DuckDuckGoSearch {
|
|
9
10
|
name = "duckduckgo";
|
|
@@ -17,7 +18,7 @@ export class DuckDuckGoSearch {
|
|
|
17
18
|
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
18
19
|
},
|
|
19
20
|
body: form.toString(),
|
|
20
|
-
signal,
|
|
21
|
+
signal: searchTimeoutSignal(signal),
|
|
21
22
|
});
|
|
22
23
|
if (!res.ok)
|
|
23
24
|
throw new Error(`duckduckgo ${res.status} ${res.statusText}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duckduckgo.js","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAC7E,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;
|
|
1
|
+
{"version":3,"file":"duckduckgo.js","sourceRoot":"","sources":["../../src/search/duckduckgo.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAC7E,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;AAEhF,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAE1F,MAAM,QAAQ,GAAG,mCAAmC,CAAC;AAErD,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,YAAY,CAAC;IAE7B,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,mCAAmC;gBACnD,YAAY,EACV,+DAA+D;oBAC/D,oDAAoD;aACvD;YACD,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE;YACrB,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF;AAED,2BAA2B;AAC3B,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAa;IAC7D,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,oEAAoE;IACpE,6DAA6D;IAC7D,MAAM,OAAO,GACX,8JAA8J,CAAC;IAEjK,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACzC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;YAAE,SAAS;QACnC,IAAI,EAAE,CAAC;QACP,OAAO,CAAC,IAAI,CAAC;YACX,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE;YACrD,OAAO,EAAE,SAAS,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE;YACzD,IAAI;SACL,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACrC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,2EAA2E;IAC3E,wCAAwC;IACxC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,IAAI,GAAG,QAAQ,GAAG,IAAI,CAAC;IAClD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,gBAAgB,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACxF,IAAI,KAAK,IAAI,CAAC,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,IAAI;gBAAE,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,CAAS;IAC/B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,2EAA2E;AAC3E,4DAA4D;AAC5D,MAAM,UAAU,kBAAkB,CAAC,CAAS;IAC1C,OAAO,CAAC,CAAC,OAAO,CAAC,0CAA0C,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC3E,MAAM,KAAK,GAA2B;YACpC,GAAG,EAAE,GAAG;YACR,EAAE,EAAE,GAAG;YACP,EAAE,EAAE,GAAG;YACP,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG;SACV,CAAC;QACF,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,GAAG,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrE,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAS;IACjC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;AACvD,CAAC;AAED,6EAA6E;AAC7E,wEAAwE;AACxE,iEAAiE;AACjE,2EAA2E;AAC3E,0CAA0C;AAC1C,MAAM,UAAU,SAAS,CAAC,CAAS;IACjC,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
|
+
interface GitHubRepoItem {
|
|
3
|
+
html_url?: string;
|
|
4
|
+
full_name?: string;
|
|
5
|
+
description?: string | null;
|
|
6
|
+
stargazers_count?: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class GitHubSearch implements SearchAdapter {
|
|
9
|
+
private readonly token?;
|
|
10
|
+
readonly name = "github";
|
|
11
|
+
constructor(token?: string | undefined);
|
|
12
|
+
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
13
|
+
}
|
|
14
|
+
export declare function mapGitHubRepos(items: GitHubRepoItem[], limit: number): SearchResult[];
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=github.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/search/github.ts"],"names":[],"mappings":"AAMA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1F,UAAU,cAAc;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,YAAa,YAAW,aAAa;IAEpC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;IADnC,QAAQ,CAAC,IAAI,YAAY;gBACI,KAAK,CAAC,EAAE,MAAM,YAAA;IAErC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAiB1F;AAID,wBAAgB,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAUrF"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// GitHub repository search adapter. Works without a key (60 search
|
|
2
|
+
// requests/hour unauthenticated); set DEEPDIVE_GITHUB_TOKEN for the
|
|
3
|
+
// authenticated rate limit (30 req/min). A fit for "what library / project
|
|
4
|
+
// does X" sub-queries — the kept source is the repo's GitHub page, whose
|
|
5
|
+
// rendered README deepdive's fetch step then extracts.
|
|
6
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
7
|
+
export class GitHubSearch {
|
|
8
|
+
token;
|
|
9
|
+
name = "github";
|
|
10
|
+
constructor(token) {
|
|
11
|
+
this.token = token;
|
|
12
|
+
}
|
|
13
|
+
async search(query, limit, signal) {
|
|
14
|
+
const url = new URL("https://api.github.com/search/repositories");
|
|
15
|
+
url.searchParams.set("q", query);
|
|
16
|
+
url.searchParams.set("per_page", String(Math.min(Math.max(limit, 1), 100)));
|
|
17
|
+
url.searchParams.set("sort", "best-match");
|
|
18
|
+
const headers = {
|
|
19
|
+
accept: "application/vnd.github+json",
|
|
20
|
+
"x-github-api-version": "2022-11-28",
|
|
21
|
+
// GitHub rejects requests with no User-Agent.
|
|
22
|
+
"user-agent": "deepdive (+https://github.com/askalf/deepdive)",
|
|
23
|
+
};
|
|
24
|
+
if (this.token)
|
|
25
|
+
headers.authorization = `Bearer ${this.token}`;
|
|
26
|
+
const res = await fetch(url, { headers, signal: searchTimeoutSignal(signal) });
|
|
27
|
+
if (!res.ok)
|
|
28
|
+
throw new Error(`github ${res.status} ${res.statusText}`);
|
|
29
|
+
const json = (await res.json());
|
|
30
|
+
return mapGitHubRepos(json.items ?? [], limit);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
// Exported for unit tests. Pure mapper. Snippet is the repo description with a
|
|
34
|
+
// star count appended when known, so the planner/critic can weigh popularity.
|
|
35
|
+
export function mapGitHubRepos(items, limit) {
|
|
36
|
+
return items
|
|
37
|
+
.filter((r) => typeof r.html_url === "string" && r.html_url.length > 0)
|
|
38
|
+
.slice(0, limit)
|
|
39
|
+
.map((r, i) => ({
|
|
40
|
+
url: r.html_url,
|
|
41
|
+
title: r.full_name ?? r.html_url ?? "",
|
|
42
|
+
snippet: buildSnippet(r),
|
|
43
|
+
rank: i + 1,
|
|
44
|
+
}));
|
|
45
|
+
}
|
|
46
|
+
function buildSnippet(r) {
|
|
47
|
+
const desc = (r.description ?? "").trim();
|
|
48
|
+
if (typeof r.stargazers_count === "number" && r.stargazers_count > 0) {
|
|
49
|
+
const stars = `★ ${formatStars(r.stargazers_count)}`;
|
|
50
|
+
return desc ? `${desc} (${stars})` : stars;
|
|
51
|
+
}
|
|
52
|
+
return desc;
|
|
53
|
+
}
|
|
54
|
+
function formatStars(n) {
|
|
55
|
+
if (n >= 1000)
|
|
56
|
+
return `${(n / 1000).toFixed(n >= 10000 ? 0 : 1)}k`;
|
|
57
|
+
return String(n);
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=github.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/search/github.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,oEAAoE;AACpE,2EAA2E;AAC3E,yEAAyE;AACzE,uDAAuD;AAEvD,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAS1F,MAAM,OAAO,YAAY;IAEM;IADpB,IAAI,GAAG,QAAQ,CAAC;IACzB,YAA6B,KAAc;QAAd,UAAK,GAAL,KAAK,CAAS;IAAG,CAAC;IAE/C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAClE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC3C,MAAM,OAAO,GAA2B;YACtC,MAAM,EAAE,6BAA6B;YACrC,sBAAsB,EAAE,YAAY;YACpC,8CAA8C;YAC9C,YAAY,EAAE,gDAAgD;SAC/D,CAAC;QACF,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,aAAa,GAAG,UAAU,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/E,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACvE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAiC,CAAC;QAChE,OAAO,cAAc,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;CACF;AAED,+EAA+E;AAC/E,8EAA8E;AAC9E,MAAM,UAAU,cAAc,CAAC,KAAuB,EAAE,KAAa;IACnE,OAAO,KAAK;SACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;SACtE,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACd,GAAG,EAAE,CAAC,CAAC,QAAkB;QACzB,KAAK,EAAE,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,QAAQ,IAAI,EAAE;QACtC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACxB,IAAI,EAAE,CAAC,GAAG,CAAC;KACZ,CAAC,CAAC,CAAC;AACR,CAAC;AAED,SAAS,YAAY,CAAC,CAAiB;IACrC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,IAAI,OAAO,CAAC,CAAC,gBAAgB,KAAK,QAAQ,IAAI,CAAC,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACrE,MAAM,KAAK,GAAG,KAAK,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACnE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC"}
|
package/dist/search/searxng.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"searxng.d.ts","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"searxng.d.ts","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAGA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAG1F,qBAAa,aAAc,YAAW,aAAa;IAErC,OAAO,CAAC,QAAQ,CAAC,OAAO;IADpC,QAAQ,CAAC,IAAI,aAAa;gBACG,OAAO,EAAE,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAoB1F"}
|
package/dist/search/searxng.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// SearXNG adapter. Points at an existing SearXNG instance (self-hosted or
|
|
2
2
|
// public). Requires DEEPDIVE_SEARXNG_URL. Uses the JSON output format.
|
|
3
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
3
4
|
import { trimTrailingSlashes } from "../url-util.js";
|
|
4
5
|
export class SearXNGSearch {
|
|
5
6
|
baseUrl;
|
|
@@ -13,7 +14,7 @@ export class SearXNGSearch {
|
|
|
13
14
|
url.searchParams.set("format", "json");
|
|
14
15
|
const res = await fetch(url, {
|
|
15
16
|
headers: { accept: "application/json" },
|
|
16
|
-
signal,
|
|
17
|
+
signal: searchTimeoutSignal(signal),
|
|
17
18
|
});
|
|
18
19
|
if (!res.ok)
|
|
19
20
|
throw new Error(`searxng ${res.status} ${res.statusText}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"searxng.js","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;
|
|
1
|
+
{"version":3,"file":"searxng.js","sourceRoot":"","sources":["../../src/search/searxng.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,uEAAuE;AAEvE,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAC1F,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,OAAO,aAAa;IAEK;IADpB,IAAI,GAAG,SAAS,CAAC;IAC1B,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEhD,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;QACnE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;YACvC,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAE7B,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;YACxB,IAAI,EAAE,CAAC,GAAG,CAAC;SACZ,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type SearchAdapter, type SearchResult } from "../search.js";
|
|
2
|
+
interface WikiSearchItem {
|
|
3
|
+
title?: string;
|
|
4
|
+
snippet?: string;
|
|
5
|
+
pageid?: number;
|
|
6
|
+
}
|
|
7
|
+
export declare class WikipediaSearch implements SearchAdapter {
|
|
8
|
+
private readonly lang;
|
|
9
|
+
readonly name = "wikipedia";
|
|
10
|
+
constructor(lang?: string);
|
|
11
|
+
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
12
|
+
}
|
|
13
|
+
export declare function mapWikipediaResults(items: WikiSearchItem[], lang: string, limit: number): SearchResult[];
|
|
14
|
+
export declare function wikipediaArticleUrl(lang: string, title: string): string;
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=wikipedia.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wikipedia.d.ts","sourceRoot":"","sources":["../../src/search/wikipedia.ts"],"names":[],"mappings":"AAUA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAG1F,UAAU,cAAc;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,eAAgB,YAAW,aAAa;IAEvC,OAAO,CAAC,QAAQ,CAAC,IAAI;IADjC,QAAQ,CAAC,IAAI,eAAe;gBACC,IAAI,GAAE,MAAa;IAE1C,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAmB1F;AAID,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,cAAc,EAAE,EACvB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GACZ,YAAY,EAAE,CAOhB;AAKD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAGvE"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// Wikipedia search adapter. No API key. Uses the MediaWiki search API
|
|
2
|
+
// (action=query&list=search) against the language edition of your choice
|
|
3
|
+
// (DEEPDIVE_WIKIPEDIA_LANG, default "en"). Good for definitional / factual
|
|
4
|
+
// sub-queries where an encyclopedia is the authoritative first stop.
|
|
5
|
+
//
|
|
6
|
+
// The API returns HTML-ish snippets with <span class="searchmatch"> markers;
|
|
7
|
+
// we strip tags and decode entities so the snippet is clean text. Full page
|
|
8
|
+
// content is left to deepdive's normal fetch step (the article URL), keeping
|
|
9
|
+
// the downstream extract/synthesis path identical across adapters.
|
|
10
|
+
import { searchTimeoutSignal } from "../search.js";
|
|
11
|
+
import { stripTags, decodeHtmlEntities } from "./duckduckgo.js";
|
|
12
|
+
export class WikipediaSearch {
|
|
13
|
+
lang;
|
|
14
|
+
name = "wikipedia";
|
|
15
|
+
constructor(lang = "en") {
|
|
16
|
+
this.lang = lang;
|
|
17
|
+
}
|
|
18
|
+
async search(query, limit, signal) {
|
|
19
|
+
const url = new URL(`https://${this.lang}.wikipedia.org/w/api.php`);
|
|
20
|
+
url.searchParams.set("action", "query");
|
|
21
|
+
url.searchParams.set("list", "search");
|
|
22
|
+
url.searchParams.set("srsearch", query);
|
|
23
|
+
url.searchParams.set("srlimit", String(Math.min(Math.max(limit, 1), 50)));
|
|
24
|
+
url.searchParams.set("format", "json");
|
|
25
|
+
url.searchParams.set("origin", "*");
|
|
26
|
+
const res = await fetch(url, {
|
|
27
|
+
headers: {
|
|
28
|
+
accept: "application/json",
|
|
29
|
+
"user-agent": "deepdive (+https://github.com/askalf/deepdive)",
|
|
30
|
+
},
|
|
31
|
+
signal: searchTimeoutSignal(signal),
|
|
32
|
+
});
|
|
33
|
+
if (!res.ok)
|
|
34
|
+
throw new Error(`wikipedia ${res.status} ${res.statusText}`);
|
|
35
|
+
const json = (await res.json());
|
|
36
|
+
return mapWikipediaResults(json.query?.search ?? [], this.lang, limit);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
// Exported for unit tests. Pure, deterministic, no I/O. Builds the canonical
|
|
40
|
+
// article URL from the page title and cleans the search-match snippet.
|
|
41
|
+
export function mapWikipediaResults(items, lang, limit) {
|
|
42
|
+
return items.slice(0, limit).map((r, i) => ({
|
|
43
|
+
url: wikipediaArticleUrl(lang, r.title ?? ""),
|
|
44
|
+
title: r.title ?? "",
|
|
45
|
+
snippet: stripTags(decodeHtmlEntities(r.snippet ?? "")).replace(/\s+/g, " ").trim(),
|
|
46
|
+
rank: i + 1,
|
|
47
|
+
}));
|
|
48
|
+
}
|
|
49
|
+
// Exported for unit tests. Title → canonical /wiki/<Title_With_Underscores>.
|
|
50
|
+
// encodeURI (not encodeURIComponent) so the slashes/colons MediaWiki allows in
|
|
51
|
+
// titles survive, while spaces become underscores per Wikipedia convention.
|
|
52
|
+
export function wikipediaArticleUrl(lang, title) {
|
|
53
|
+
const slug = encodeURI(title.replace(/ /g, "_")).replace(/\?/g, "%3F").replace(/#/g, "%23");
|
|
54
|
+
return `https://${lang}.wikipedia.org/wiki/${slug}`;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=wikipedia.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wikipedia.js","sourceRoot":"","sources":["../../src/search/wikipedia.ts"],"names":[],"mappings":"AAAA,sEAAsE;AACtE,yEAAyE;AACzE,2EAA2E;AAC3E,qEAAqE;AACrE,EAAE;AACF,6EAA6E;AAC7E,4EAA4E;AAC5E,6EAA6E;AAC7E,mEAAmE;AAEnE,OAAO,EAAE,mBAAmB,EAAyC,MAAM,cAAc,CAAC;AAC1F,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAQhE,MAAM,OAAO,eAAe;IAEG;IADpB,IAAI,GAAG,WAAW,CAAC;IAC5B,YAA6B,OAAe,IAAI;QAAnB,SAAI,GAAJ,IAAI,CAAe;IAAG,CAAC;IAEpD,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,KAAa,EAAE,MAAoB;QAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,WAAW,IAAI,CAAC,IAAI,0BAA0B,CAAC,CAAC;QACpE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACxC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACxC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACvC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,YAAY,EAAE,gDAAgD;aAC/D;YACD,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1E,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA8C,CAAC;QAC7E,OAAO,mBAAmB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzE,CAAC;CACF;AAED,6EAA6E;AAC7E,uEAAuE;AACvE,MAAM,UAAU,mBAAmB,CACjC,KAAuB,EACvB,IAAY,EACZ,KAAa;IAEb,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1C,GAAG,EAAE,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;QACpB,OAAO,EAAE,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE;QACnF,IAAI,EAAE,CAAC,GAAG,CAAC;KACZ,CAAC,CAAC,CAAC;AACN,CAAC;AAED,6EAA6E;AAC7E,+EAA+E;AAC/E,4EAA4E;AAC5E,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAa;IAC7D,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC5F,OAAO,WAAW,IAAI,uBAAuB,IAAI,EAAE,CAAC;AACtD,CAAC"}
|
package/dist/search.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ export interface SearchAdapter {
|
|
|
8
8
|
readonly name: string;
|
|
9
9
|
search(query: string, limit: number, signal?: AbortSignal): Promise<SearchResult[]>;
|
|
10
10
|
}
|
|
11
|
+
export declare function searchTimeoutSignal(signal?: AbortSignal): AbortSignal;
|
|
11
12
|
export declare function resolveSearchAdapter(name: string, env: Record<string, string | undefined>): Promise<SearchAdapter>;
|
|
12
13
|
export declare function dedupeByUrl(results: SearchResult[]): SearchResult[];
|
|
13
14
|
//# sourceMappingURL=search.d.ts.map
|
package/dist/search.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CACrF;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GACtC,OAAO,CAAC,aAAa,CAAC,
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CACrF;AAOD,wBAAgB,mBAAmB,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW,CAIrE;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GACtC,OAAO,CAAC,aAAa,CAAC,CAkExB;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAUnE"}
|
package/dist/search.js
CHANGED
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
// Search adapter interface. A "search" returns candidate URLs with metadata.
|
|
2
2
|
// Adapters live under src/search/*. Default is DuckDuckGo HTML (no API key).
|
|
3
3
|
import { dedupeKey } from "./url-util.js";
|
|
4
|
+
// Per-request search timeout. Adapters previously passed only the caller's
|
|
5
|
+
// abort signal (wired to SIGINT/SIGTERM), so a hung search endpoint blocked the
|
|
6
|
+
// whole run with no escape. Combine the caller's signal with a hard timeout,
|
|
7
|
+
// mirroring the per-request timeout robots.ts already applies. Override the
|
|
8
|
+
// 15s default via DEEPDIVE_SEARCH_TIMEOUT_MS.
|
|
9
|
+
export function searchTimeoutSignal(signal) {
|
|
10
|
+
const ms = Number(process.env["DEEPDIVE_SEARCH_TIMEOUT_MS"]) || 15_000;
|
|
11
|
+
const timeout = AbortSignal.timeout(ms);
|
|
12
|
+
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
13
|
+
}
|
|
4
14
|
export async function resolveSearchAdapter(name, env) {
|
|
5
15
|
switch (name) {
|
|
6
16
|
case "duckduckgo":
|
|
@@ -36,6 +46,22 @@ export async function resolveSearchAdapter(name, env) {
|
|
|
36
46
|
throw new Error("exa adapter requires DEEPDIVE_EXA_KEY");
|
|
37
47
|
return new ExaSearch(key);
|
|
38
48
|
}
|
|
49
|
+
case "wikipedia":
|
|
50
|
+
case "wiki": {
|
|
51
|
+
const { WikipediaSearch } = await import("./search/wikipedia.js");
|
|
52
|
+
const lang = (env.DEEPDIVE_WIKIPEDIA_LANG ?? "en").trim() || "en";
|
|
53
|
+
return new WikipediaSearch(lang);
|
|
54
|
+
}
|
|
55
|
+
case "arxiv": {
|
|
56
|
+
const { ArxivSearch } = await import("./search/arxiv.js");
|
|
57
|
+
return new ArxivSearch();
|
|
58
|
+
}
|
|
59
|
+
case "github": {
|
|
60
|
+
// Token is optional — unauthenticated search works at a lower rate
|
|
61
|
+
// limit. DEEPDIVE_GITHUB_TOKEN raises it.
|
|
62
|
+
const { GitHubSearch } = await import("./search/github.js");
|
|
63
|
+
return new GitHubSearch(env.DEEPDIVE_GITHUB_TOKEN);
|
|
64
|
+
}
|
|
39
65
|
case "auto": {
|
|
40
66
|
// DDG primary, Brave fallback. Brave is optional — if no key is set,
|
|
41
67
|
// `auto` degrades to DDG-only (the pre-auto default behavior) rather
|
package/dist/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAc1C,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,GAAuC;IAEvC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,YAAY,CAAC;QAClB,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACpE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;YAC9D,MAAM,GAAG,GAAG,GAAG,CAAC,oBAAoB,CAAC;YACrC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC3E,OAAO,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,GAAG,CAAC,kBAAkB,CAAC;YACnC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YACvE,OAAO,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;QAC9B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,GAAG,CAAC,mBAAmB,CAAC;YACpC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACzE,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC;YACjC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACnE,OAAO,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,qEAAqE;YACrE,qEAAqE;YACrE,gEAAgE;YAChE,WAAW;YACX,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YACxD,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,GAAG,CAAC,kBAAkB,CAAC;YACxC,IAAI,SAAS,GAAG,IAAI,CAAC;YACrB,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAC1D,SAAS,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAuB;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,6EAA6E;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAc1C,2EAA2E;AAC3E,gFAAgF;AAChF,6EAA6E;AAC7E,4EAA4E;AAC5E,8CAA8C;AAC9C,MAAM,UAAU,mBAAmB,CAAC,MAAoB;IACtD,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC,IAAI,MAAM,CAAC;IACvE,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACxC,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAC/D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAY,EACZ,GAAuC;IAEvC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,YAAY,CAAC;QAClB,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACpE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;YAC9D,MAAM,GAAG,GAAG,GAAG,CAAC,oBAAoB,CAAC;YACrC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC3E,OAAO,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,GAAG,CAAC,kBAAkB,CAAC;YACnC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YACvE,OAAO,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;QAC9B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,GAAG,CAAC,mBAAmB,CAAC;YACpC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACzE,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,CAAC;YACjC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACnE,OAAO,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,KAAK,WAAW,CAAC;QACjB,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YAClE,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;YAClE,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC1D,OAAO,IAAI,WAAW,EAAE,CAAC;QAC3B,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,mEAAmE;YACnE,0CAA0C;YAC1C,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAC5D,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACrD,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,qEAAqE;YACrE,qEAAqE;YACrE,gEAAgE;YAChE,WAAW;YACX,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YACxD,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,GAAG,CAAC,kBAAkB,CAAC;YACxC,IAAI,SAAS,GAAG,IAAI,CAAC;YACrB,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBAC1D,SAAS,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAuB;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/dist/sessions.d.ts
CHANGED
|
@@ -42,4 +42,19 @@ export declare function listSessions(opts: SessionStorageOptions): Promise<{
|
|
|
42
42
|
export declare function resolveSessionId(prefix: string, opts: SessionStorageOptions): Promise<string>;
|
|
43
43
|
export declare function renderSessionsList(metas: SessionMeta[]): string;
|
|
44
44
|
export declare function humanDuration(ms: number): string;
|
|
45
|
+
export declare function deleteSession(id: string, opts: SessionStorageOptions): Promise<void>;
|
|
46
|
+
export interface PruneCriteria {
|
|
47
|
+
olderThanMs?: number;
|
|
48
|
+
keep?: number;
|
|
49
|
+
now?: number;
|
|
50
|
+
}
|
|
51
|
+
export declare function selectSessionsToPrune(metas: SessionMeta[], crit: PruneCriteria): SessionMeta[];
|
|
52
|
+
export declare function pruneSessions(opts: SessionStorageOptions, crit: PruneCriteria & {
|
|
53
|
+
dryRun?: boolean;
|
|
54
|
+
}): Promise<{
|
|
55
|
+
removed: SessionMeta[];
|
|
56
|
+
remaining: number;
|
|
57
|
+
bad: string[];
|
|
58
|
+
}>;
|
|
59
|
+
export declare function parseDuration(s: string | undefined): number | undefined;
|
|
45
60
|
//# sourceMappingURL=sessions.d.ts.map
|
package/dist/sessions.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sessions.d.ts","sourceRoot":"","sources":["../src/sessions.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,MAAM,WAAW,aAAa;IAE5B,MAAM,EAAE,CAAC,CAAC;IACV,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,IAAI,EAAE,YAAY,CAAC;IACnB,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAKxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;CACb;AAGD,wBAAgB,kBAAkB,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,CAEnF;AAMD,wBAAgB,iBAAiB,CAAC,GAAG,GAAE,IAAiB,GAAG,MAAM,CAShE;AAgBD,wBAAsB,WAAW,CAC/B,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,MAAM,CAAC,CAOjB;AAGD,wBAAsB,WAAW,CAC/B,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAUxB;AAID,wBAAsB,YAAY,CAChC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC;IAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAiCrD;AAYD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,MAAM,CAAC,CAuBjB;AAID,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,CAc/D;AAGD,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAUhD"}
|
|
1
|
+
{"version":3,"file":"sessions.d.ts","sourceRoot":"","sources":["../src/sessions.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAChD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,MAAM,WAAW,aAAa;IAE5B,MAAM,EAAE,CAAC,CAAC;IACV,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,IAAI,EAAE,YAAY,CAAC;IACnB,GAAG,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAKxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;CACb;AAGD,wBAAgB,kBAAkB,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,CAEnF;AAMD,wBAAgB,iBAAiB,CAAC,GAAG,GAAE,IAAiB,GAAG,MAAM,CAShE;AAgBD,wBAAsB,WAAW,CAC/B,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,MAAM,CAAC,CAOjB;AAGD,wBAAsB,WAAW,CAC/B,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAUxB;AAID,wBAAsB,YAAY,CAChC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC;IAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAiCrD;AAYD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,MAAM,CAAC,CAuBjB;AAID,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,CAc/D;AAGD,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAUhD;AAMD,wBAAsB,aAAa,CACjC,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,MAAM,WAAW,aAAa;IAE5B,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAMD,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,WAAW,EAAE,EACpB,IAAI,EAAE,aAAa,GAClB,WAAW,EAAE,CAaf;AAKD,wBAAsB,aAAa,CACjC,IAAI,EAAE,qBAAqB,EAC3B,IAAI,EAAE,aAAa,GAAG;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GACzC,OAAO,CAAC;IAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAavE;AAMD,wBAAgB,aAAa,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAgBvE"}
|
package/dist/sessions.js
CHANGED
|
@@ -162,4 +162,71 @@ export function humanDuration(ms) {
|
|
|
162
162
|
const d = Math.floor(h / 24);
|
|
163
163
|
return `${d}d ago`;
|
|
164
164
|
}
|
|
165
|
+
// ── Lifecycle: delete + prune ────────────────────────────────────────────────
|
|
166
|
+
// Deletes a single session file by exact id. Resolve a prefix with
|
|
167
|
+
// resolveSessionId first. Throws ENOENT if the file is already gone.
|
|
168
|
+
export async function deleteSession(id, opts) {
|
|
169
|
+
await fs.unlink(join(opts.dir, `${id}.json`));
|
|
170
|
+
}
|
|
171
|
+
// Exported for unit tests. Pure selection: given metadata (newest-first, as
|
|
172
|
+
// listSessions returns) and the criteria, returns the sessions that should be
|
|
173
|
+
// removed. With neither criterion set, returns [] — never prunes everything by
|
|
174
|
+
// default; the CLI requires at least one of --older-than / --keep.
|
|
175
|
+
export function selectSessionsToPrune(metas, crit) {
|
|
176
|
+
if (crit.olderThanMs === undefined && crit.keep === undefined)
|
|
177
|
+
return [];
|
|
178
|
+
const now = crit.now ?? Date.now();
|
|
179
|
+
const keep = crit.keep ?? 0;
|
|
180
|
+
const out = [];
|
|
181
|
+
metas.forEach((m, idx) => {
|
|
182
|
+
if (idx < keep)
|
|
183
|
+
return; // protected: among the newest `keep`
|
|
184
|
+
if (crit.olderThanMs !== undefined && now - m.createdAt < crit.olderThanMs) {
|
|
185
|
+
return; // too new to prune
|
|
186
|
+
}
|
|
187
|
+
out.push(m);
|
|
188
|
+
});
|
|
189
|
+
return out;
|
|
190
|
+
}
|
|
191
|
+
// Lists, selects, and (unless dryRun) deletes matching sessions. Returns the
|
|
192
|
+
// metadata of removed sessions, the count still remaining, and any unparsable
|
|
193
|
+
// files encountered while listing.
|
|
194
|
+
export async function pruneSessions(opts, crit) {
|
|
195
|
+
const { sessions, bad } = await listSessions(opts);
|
|
196
|
+
const toRemove = selectSessionsToPrune(sessions, crit);
|
|
197
|
+
if (!crit.dryRun) {
|
|
198
|
+
for (const m of toRemove) {
|
|
199
|
+
await deleteSession(m.id, opts).catch(() => undefined);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return {
|
|
203
|
+
removed: toRemove,
|
|
204
|
+
remaining: sessions.length - toRemove.length,
|
|
205
|
+
bad,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
// Exported for unit tests. Parses a human duration like "30d", "12h", "90m",
|
|
209
|
+
// "45s", "2w" into milliseconds. A bare integer is interpreted as days
|
|
210
|
+
// (the common "prune things older than N days" case). Returns undefined for
|
|
211
|
+
// unparsable input.
|
|
212
|
+
export function parseDuration(s) {
|
|
213
|
+
if (!s)
|
|
214
|
+
return undefined;
|
|
215
|
+
const t = s.trim().toLowerCase();
|
|
216
|
+
const m = /^(\d+)\s*(w|d|h|m|s)?$/.exec(t);
|
|
217
|
+
if (!m)
|
|
218
|
+
return undefined;
|
|
219
|
+
const n = Number(m[1]);
|
|
220
|
+
if (!Number.isFinite(n))
|
|
221
|
+
return undefined;
|
|
222
|
+
const unit = m[2] ?? "d";
|
|
223
|
+
const mult = {
|
|
224
|
+
s: 1000,
|
|
225
|
+
m: 60_000,
|
|
226
|
+
h: 3_600_000,
|
|
227
|
+
d: 86_400_000,
|
|
228
|
+
w: 604_800_000,
|
|
229
|
+
};
|
|
230
|
+
return n * mult[unit];
|
|
231
|
+
}
|
|
165
232
|
//# sourceMappingURL=sessions.js.map
|