@lloyal-labs/rig 2.0.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +107 -0
- package/LICENSE-FAQ.md +256 -0
- package/README.md +78 -72
- package/dist/bundle.d.ts +211 -0
- package/dist/bundle.d.ts.map +1 -0
- package/dist/bundle.js +296 -0
- package/dist/bundle.js.map +1 -0
- package/dist/cancellable-fetch.d.ts +98 -0
- package/dist/cancellable-fetch.d.ts.map +1 -0
- package/dist/cancellable-fetch.js +133 -0
- package/dist/cancellable-fetch.js.map +1 -0
- package/dist/config-store.d.ts +30 -0
- package/dist/config-store.d.ts.map +1 -0
- package/dist/config-store.js +45 -0
- package/dist/config-store.js.map +1 -0
- package/dist/define-app.d.ts +98 -0
- package/dist/define-app.d.ts.map +1 -0
- package/dist/define-app.js +232 -0
- package/dist/define-app.js.map +1 -0
- package/dist/grant-store.d.ts +31 -0
- package/dist/grant-store.d.ts.map +1 -0
- package/dist/grant-store.js +49 -0
- package/dist/grant-store.js.map +1 -0
- package/dist/index.d.ts +13 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +37 -11
- package/dist/index.js.map +1 -1
- package/dist/node.d.ts +3 -2
- package/dist/node.d.ts.map +1 -1
- package/dist/node.js +3 -2
- package/dist/node.js.map +1 -1
- package/dist/protocol.d.ts +155 -0
- package/dist/protocol.d.ts.map +1 -0
- package/dist/protocol.js +184 -0
- package/dist/protocol.js.map +1 -0
- package/dist/registry.d.ts +87 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/registry.js +245 -0
- package/dist/registry.js.map +1 -0
- package/dist/reranker.d.ts +25 -7
- package/dist/reranker.d.ts.map +1 -1
- package/dist/reranker.js +103 -63
- package/dist/reranker.js.map +1 -1
- package/dist/resources/types.d.ts +10 -37
- package/dist/resources/types.d.ts.map +1 -1
- package/dist/resources/types.js +12 -0
- package/dist/resources/types.js.map +1 -1
- package/dist/spine-render.d.ts +97 -0
- package/dist/spine-render.d.ts.map +1 -0
- package/dist/spine-render.js +121 -0
- package/dist/spine-render.js.map +1 -0
- package/dist/tools/delegate.js +1 -1
- package/dist/tools/index.d.ts +26 -22
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +24 -28
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/keyless-search.d.ts +67 -0
- package/dist/tools/keyless-search.d.ts.map +1 -0
- package/dist/tools/keyless-search.js +401 -0
- package/dist/tools/keyless-search.js.map +1 -0
- package/dist/tools/plan.d.ts +31 -4
- package/dist/tools/plan.d.ts.map +1 -1
- package/dist/tools/plan.js +46 -11
- package/dist/tools/plan.js.map +1 -1
- package/dist/tools/report.d.ts +4 -3
- package/dist/tools/report.d.ts.map +1 -1
- package/dist/tools/report.js +4 -3
- package/dist/tools/report.js.map +1 -1
- package/dist/tools/types.d.ts +12 -56
- package/dist/tools/types.d.ts.map +1 -1
- package/dist/tools/types.js +17 -0
- package/dist/tools/types.js.map +1 -1
- package/dist/tools/web-search.d.ts +9 -25
- package/dist/tools/web-search.d.ts.map +1 -1
- package/dist/tools/web-search.js +11 -119
- package/dist/tools/web-search.js.map +1 -1
- package/package.json +9 -6
- package/dist/sources/corpus.d.ts +0 -80
- package/dist/sources/corpus.d.ts.map +0 -1
- package/dist/sources/corpus.js +0 -100
- package/dist/sources/corpus.js.map +0 -1
- package/dist/sources/index.d.ts +0 -12
- package/dist/sources/index.d.ts.map +0 -1
- package/dist/sources/index.js +0 -14
- package/dist/sources/index.js.map +0 -1
- package/dist/sources/web.d.ts +0 -67
- package/dist/sources/web.d.ts.map +0 -1
- package/dist/sources/web.js +0 -104
- package/dist/sources/web.js.map +0 -1
- package/dist/tools/fetch-page.d.ts +0 -48
- package/dist/tools/fetch-page.d.ts.map +0 -1
- package/dist/tools/fetch-page.js +0 -309
- package/dist/tools/fetch-page.js.map +0 -1
- package/dist/tools/grep.d.ts +0 -35
- package/dist/tools/grep.d.ts.map +0 -1
- package/dist/tools/grep.js +0 -84
- package/dist/tools/grep.js.map +0 -1
- package/dist/tools/read-file.d.ts +0 -74
- package/dist/tools/read-file.d.ts.map +0 -1
- package/dist/tools/read-file.js +0 -192
- package/dist/tools/read-file.js.map +0 -1
- package/dist/tools/search.d.ts +0 -34
- package/dist/tools/search.d.ts.map +0 -1
- package/dist/tools/search.js +0 -101
- package/dist/tools/search.js.map +0 -1
package/dist/sources/web.d.ts
DELETED
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
import type { Operation } from "effection";
|
|
2
|
-
import { Source } from "@lloyal-labs/lloyal-agents";
|
|
3
|
-
import type { Tool } from "@lloyal-labs/lloyal-agents";
|
|
4
|
-
import type { Chunk } from "../resources/types";
|
|
5
|
-
import type { SearchProvider } from "../tools/types";
|
|
6
|
-
import type { Reranker } from "../tools/types";
|
|
7
|
-
export { chunkFetchedPages, type FetchedPage } from "./chunking";
|
|
8
|
-
/**
|
|
9
|
-
* Configuration for {@link WebSource}.
|
|
10
|
-
*
|
|
11
|
-
* @category Rig
|
|
12
|
-
*/
|
|
13
|
-
export interface WebSourceOpts {
|
|
14
|
-
/** Max search results returned to agents. @default 8 */
|
|
15
|
-
topN?: number;
|
|
16
|
-
/** FetchPageTool configuration */
|
|
17
|
-
fetch?: {
|
|
18
|
-
/** Max chars for full-content fallback (no reranker). @default 6000 */
|
|
19
|
-
maxChars?: number;
|
|
20
|
-
/** Top-K reranked chunks returned. @default 5 */
|
|
21
|
-
topK?: number;
|
|
22
|
-
/** Fetch timeout in ms. @default 10000 */
|
|
23
|
-
timeout?: number;
|
|
24
|
-
/** Reranker token budget for chunk selection. @default 2048 */
|
|
25
|
-
tokenBudget?: number;
|
|
26
|
-
};
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Web-backed data source
|
|
30
|
-
*
|
|
31
|
-
* Provides two tools: {@link WebSearchTool} (search the web) and
|
|
32
|
-
* {@link FetchPageTool} (fetch and extract page content with optional
|
|
33
|
-
* reranking). Fetched content is buffered for post-use reranking via
|
|
34
|
-
* {@link getChunks}.
|
|
35
|
-
*
|
|
36
|
-
* No orchestration, no prompts, no node:fs. Works on Node and React
|
|
37
|
-
* Native identically. Use {@link spawnAgents} from your harness to
|
|
38
|
-
* orchestrate agents with these tools.
|
|
39
|
-
*
|
|
40
|
-
* @category Rig
|
|
41
|
-
*/
|
|
42
|
-
export declare class WebSource extends Source<{
|
|
43
|
-
reranker: Reranker;
|
|
44
|
-
}, Chunk> {
|
|
45
|
-
private _buffer;
|
|
46
|
-
private _fetchPage;
|
|
47
|
-
private _webSearch;
|
|
48
|
-
/** @inheritDoc */
|
|
49
|
-
readonly name = "web";
|
|
50
|
-
/**
|
|
51
|
-
* @param provider - Search backend (e.g. {@link TavilyProvider}) for web_search calls
|
|
52
|
-
* @param opts - Configuration for search and fetch tools
|
|
53
|
-
*/
|
|
54
|
-
constructor(provider: SearchProvider, opts?: WebSourceOpts);
|
|
55
|
-
/** @inheritDoc */
|
|
56
|
-
get tools(): Tool[];
|
|
57
|
-
/**
|
|
58
|
-
* Wire reranker to FetchPageTool for chunk scoring.
|
|
59
|
-
* @inheritDoc
|
|
60
|
-
*/
|
|
61
|
-
bind(ctx: {
|
|
62
|
-
reranker: Reranker;
|
|
63
|
-
}): Operation<void>;
|
|
64
|
-
/** @inheritDoc */
|
|
65
|
-
getChunks(): Chunk[];
|
|
66
|
-
}
|
|
67
|
-
//# sourceMappingURL=web.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/sources/web.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAS,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EAAE,IAAI,EAAe,MAAM,4BAA4B,CAAC;AACpE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAO/C,OAAO,EAAE,iBAAiB,EAAE,KAAK,WAAW,EAAE,MAAM,YAAY,CAAC;AA4DjE;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,KAAK,CAAC,EAAE;QACN,uEAAuE;QACvE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,iDAAiD;QACjD,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,0CAA0C;QAC1C,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,+DAA+D;QAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,SAAU,SAAQ,MAAM,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAA;CAAE,EAAE,KAAK,CAAC;IAClE,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,UAAU,CAAgB;IAElC,kBAAkB;IAClB,QAAQ,CAAC,IAAI,SAAS;IAEtB;;;OAGG;gBACS,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,aAAa;IAM1D,kBAAkB;IAClB,IAAI,KAAK,IAAI,IAAI,EAAE,CAElB;IAED;;;OAGG;IACF,IAAI,CAAC,GAAG,EAAE;QAAE,QAAQ,EAAE,QAAQ,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC;IAgBnD,kBAAkB;IAClB,SAAS,IAAI,KAAK,EAAE;CAGrB"}
|
package/dist/sources/web.js
DELETED
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.WebSource = exports.chunkFetchedPages = void 0;
|
|
4
|
-
const lloyal_agents_1 = require("@lloyal-labs/lloyal-agents");
|
|
5
|
-
const web_search_1 = require("../tools/web-search");
|
|
6
|
-
const fetch_page_1 = require("../tools/fetch-page");
|
|
7
|
-
const chunking_1 = require("./chunking");
|
|
8
|
-
// Re-export for backwards compatibility
|
|
9
|
-
var chunking_2 = require("./chunking");
|
|
10
|
-
Object.defineProperty(exports, "chunkFetchedPages", { enumerable: true, get: function () { return chunking_2.chunkFetchedPages; } });
|
|
11
|
-
// ── BufferingFetchPage ───────────────────────────────────────
|
|
12
|
-
/**
|
|
13
|
-
* Thin wrapper over {@link FetchPageTool} that buffers fetched content
|
|
14
|
-
* for post-use reranking via {@link WebSource.getChunks}.
|
|
15
|
-
*
|
|
16
|
-
* @category Rig
|
|
17
|
-
*/
|
|
18
|
-
class BufferingFetchPage extends fetch_page_1.FetchPageTool {
|
|
19
|
-
_buffer;
|
|
20
|
-
_urlCache = new Map();
|
|
21
|
-
constructor(buffer, opts) {
|
|
22
|
-
super(opts);
|
|
23
|
-
this._buffer = buffer;
|
|
24
|
-
}
|
|
25
|
-
clearCache() {
|
|
26
|
-
this._urlCache.clear();
|
|
27
|
-
}
|
|
28
|
-
*execute(args, context) {
|
|
29
|
-
const cached = this._urlCache.get(args.url);
|
|
30
|
-
if (cached)
|
|
31
|
-
return cached;
|
|
32
|
-
const result = yield* super.execute(args, context);
|
|
33
|
-
this._urlCache.set(args.url, result);
|
|
34
|
-
const r = result;
|
|
35
|
-
const hasContent = typeof r?.content === "string" &&
|
|
36
|
-
r.content !== "[Could not extract article content]";
|
|
37
|
-
if (hasContent) {
|
|
38
|
-
this._buffer.push({
|
|
39
|
-
url: r.url || args.url,
|
|
40
|
-
title: r.title || "",
|
|
41
|
-
text: r.content,
|
|
42
|
-
});
|
|
43
|
-
}
|
|
44
|
-
return result;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Web-backed data source
|
|
49
|
-
*
|
|
50
|
-
* Provides two tools: {@link WebSearchTool} (search the web) and
|
|
51
|
-
* {@link FetchPageTool} (fetch and extract page content with optional
|
|
52
|
-
* reranking). Fetched content is buffered for post-use reranking via
|
|
53
|
-
* {@link getChunks}.
|
|
54
|
-
*
|
|
55
|
-
* No orchestration, no prompts, no node:fs. Works on Node and React
|
|
56
|
-
* Native identically. Use {@link spawnAgents} from your harness to
|
|
57
|
-
* orchestrate agents with these tools.
|
|
58
|
-
*
|
|
59
|
-
* @category Rig
|
|
60
|
-
*/
|
|
61
|
-
class WebSource extends lloyal_agents_1.Source {
|
|
62
|
-
_buffer = [];
|
|
63
|
-
_fetchPage;
|
|
64
|
-
_webSearch;
|
|
65
|
-
/** @inheritDoc */
|
|
66
|
-
name = "web";
|
|
67
|
-
/**
|
|
68
|
-
* @param provider - Search backend (e.g. {@link TavilyProvider}) for web_search calls
|
|
69
|
-
* @param opts - Configuration for search and fetch tools
|
|
70
|
-
*/
|
|
71
|
-
constructor(provider, opts) {
|
|
72
|
-
super();
|
|
73
|
-
this._fetchPage = new BufferingFetchPage(this._buffer, opts?.fetch);
|
|
74
|
-
this._webSearch = new web_search_1.WebSearchTool(provider, opts?.topN);
|
|
75
|
-
}
|
|
76
|
-
/** @inheritDoc */
|
|
77
|
-
get tools() {
|
|
78
|
-
return [this._webSearch, this._fetchPage];
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Wire reranker to FetchPageTool for chunk scoring.
|
|
82
|
-
* @inheritDoc
|
|
83
|
-
*/
|
|
84
|
-
*bind(ctx) {
|
|
85
|
-
this._buffer.length = 0;
|
|
86
|
-
this._fetchPage.clearCache();
|
|
87
|
-
this._reranker = ctx.reranker;
|
|
88
|
-
this._fetchPage.setReranker(ctx.reranker);
|
|
89
|
-
const tw = yield* lloyal_agents_1.Trace.expect();
|
|
90
|
-
tw.write({
|
|
91
|
-
traceId: tw.nextId(),
|
|
92
|
-
parentTraceId: null,
|
|
93
|
-
ts: performance.now(),
|
|
94
|
-
type: "source:bind",
|
|
95
|
-
sourceName: this.name,
|
|
96
|
-
});
|
|
97
|
-
}
|
|
98
|
-
/** @inheritDoc */
|
|
99
|
-
getChunks() {
|
|
100
|
-
return (0, chunking_1.chunkFetchedPages)(this._buffer);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
exports.WebSource = WebSource;
|
|
104
|
-
//# sourceMappingURL=web.js.map
|
package/dist/sources/web.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"web.js","sourceRoot":"","sources":["../../src/sources/web.ts"],"names":[],"mappings":";;;AACA,8DAA2D;AAK3D,oDAAoD;AACpD,oDAAoD;AACpD,yCAA+C;AAG/C,wCAAwC;AACxC,uCAAiE;AAAxD,6GAAA,iBAAiB,OAAA;AAE1B,gEAAgE;AAEhE;;;;;GAKG;AACH,MAAM,kBAAmB,SAAQ,0BAAa;IACpC,OAAO,CAAgB;IACvB,SAAS,GAAG,IAAI,GAAG,EAAmB,CAAC;IAE/C,YACE,MAAqB,EACrB,IAKC;QAED,KAAK,CAAC,IAAI,CAAC,CAAC;QACZ,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,UAAU;QACR,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,CAAC,OAAO,CACN,IAAqC,EACrC,OAAqB;QAErB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAEnD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,CAAC,GAAG,MAAiC,CAAC;QAC5C,MAAM,UAAU,GACd,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ;YAC9B,CAAC,CAAC,OAAO,KAAK,qCAAqC,CAAC;QACtD,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAChB,GAAG,EAAG,CAAC,CAAC,GAAc,IAAI,IAAI,CAAC,GAAG;gBAClC,KAAK,EAAG,CAAC,CAAC,KAAgB,IAAI,EAAE;gBAChC,IAAI,EAAE,CAAC,CAAC,OAAiB;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAyBD;;;;;;;;;;;;;GAaG;AACH,MAAa,SAAU,SAAQ,sBAAqC;IAC1D,OAAO,GAAkB,EAAE,CAAC;IAC5B,UAAU,CAAqB;IAC/B,UAAU,CAAgB;IAElC,kBAAkB;IACT,IAAI,GAAG,KAAK,CAAC;IAEtB;;;OAGG;IACH,YAAY,QAAwB,EAAE,IAAoB;QACxD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,UAAU,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACpE,IAAI,CAAC,UAAU,GAAG,IAAI,0BAAa,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,kBAAkB;IAClB,IAAI,KAAK;QACP,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACH,CAAC,IAAI,CAAC,GAA2B;QAC/B,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE1C,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,qBAAK,CAAC,MAAM,EAAE,CAAC;QACjC,EAAE,CAAC,KAAK,CAAC;YACP,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE;YACpB,aAAa,EAAE,IAAI;YACnB,EAAE,EAAE,WAAW,CAAC,GAAG,EAAE;YACrB,IAAI,EAAE,aAAa;YACnB,UAAU,EAAE,IAAI,CAAC,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,SAAS;QACP,OAAO,IAAA,4BAAiB,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;CACF;AA/CD,8BA+CC"}
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
import type { Operation } from "effection";
|
|
2
|
-
import { Tool } from "@lloyal-labs/lloyal-agents";
|
|
3
|
-
import type { JsonSchema, ToolContext } from "@lloyal-labs/lloyal-agents";
|
|
4
|
-
import type { Reranker } from "./types";
|
|
5
|
-
/**
|
|
6
|
-
* Fetch a web page and extract readable article content.
|
|
7
|
-
*
|
|
8
|
-
* Uses the Fetch API with a 10-second timeout, then extracts the
|
|
9
|
-
* article body via linkedom + Readability.
|
|
10
|
-
*
|
|
11
|
-
* When a reranker is set (via {@link setReranker}) and the agent provides
|
|
12
|
-
* a `query` argument, the article HTML is structurally chunked on heading
|
|
13
|
-
* boundaries (same pattern as corpus `parseMarkdown`) and scored against
|
|
14
|
-
* the query. Only the top-K most relevant verbatim chunks are returned —
|
|
15
|
-
* reducing KV pressure without lossy summarization. The reranker runs on
|
|
16
|
-
* its own `llama_context`, consuming zero inference KV.
|
|
17
|
-
*
|
|
18
|
-
* Without a reranker or query, returns the full content truncated to
|
|
19
|
-
* `maxChars` (default 6000). Fully backward compatible.
|
|
20
|
-
*
|
|
21
|
-
* @category Rig
|
|
22
|
-
*/
|
|
23
|
-
export declare class FetchPageTool extends Tool<{
|
|
24
|
-
url: string;
|
|
25
|
-
query?: string;
|
|
26
|
-
}> {
|
|
27
|
-
readonly name = "fetch_page";
|
|
28
|
-
readonly description = "Fetch a web page and extract its article content. Returns readable text with title and excerpt. Pass a query to get only the most relevant sections.";
|
|
29
|
-
readonly parameters: JsonSchema;
|
|
30
|
-
private _maxChars;
|
|
31
|
-
private _reranker;
|
|
32
|
-
private _topK;
|
|
33
|
-
private _timeout;
|
|
34
|
-
private _tokenBudget;
|
|
35
|
-
constructor(opts?: {
|
|
36
|
-
maxChars?: number;
|
|
37
|
-
topK?: number;
|
|
38
|
-
timeout?: number;
|
|
39
|
-
tokenBudget?: number;
|
|
40
|
-
});
|
|
41
|
-
/** Inject reranker for chunk scoring. Call from Source.bind(). */
|
|
42
|
-
setReranker(reranker: Reranker): void;
|
|
43
|
-
execute(args: {
|
|
44
|
-
url: string;
|
|
45
|
-
query?: string;
|
|
46
|
-
}, context?: ToolContext): Operation<unknown>;
|
|
47
|
-
}
|
|
48
|
-
//# sourceMappingURL=fetch-page.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-page.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-page.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAS,MAAM,4BAA4B,CAAC;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAG1E,OAAO,KAAK,EAAE,QAAQ,EAAe,MAAM,SAAS,CAAC;AA+CrD;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAc,SAAQ,IAAI,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;IACtE,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,QAAQ,CAAC,WAAW,0JACqI;IACzJ,QAAQ,CAAC,UAAU,EAAE,UAAU,CAW7B;IAEF,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,YAAY,CAAS;gBAEjB,IAAI,CAAC,EAAE;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAQD,kEAAkE;IAClE,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAIpC,OAAO,CACN,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EACrC,OAAO,CAAC,EAAE,WAAW,GACpB,SAAS,CAAC,OAAO,CAAC;CAwPtB"}
|
package/dist/tools/fetch-page.js
DELETED
|
@@ -1,309 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.FetchPageTool = void 0;
|
|
4
|
-
const effection_1 = require("effection");
|
|
5
|
-
const lloyal_agents_1 = require("@lloyal-labs/lloyal-agents");
|
|
6
|
-
const chunking_1 = require("../sources/chunking");
|
|
7
|
-
/** Select top-K scored chunks within a token budget. */
|
|
8
|
-
function selectTopChunks(scored, chunks, topK, tokenBudget) {
|
|
9
|
-
const selected = [];
|
|
10
|
-
let tokenTotal = 0;
|
|
11
|
-
for (const sc of scored.slice(0, topK)) {
|
|
12
|
-
const chunk = chunks.find((c) => c.resource === sc.file && c.startLine === sc.startLine);
|
|
13
|
-
if (!chunk?.text)
|
|
14
|
-
continue;
|
|
15
|
-
const chunkTokens = chunk.tokens.length || Math.ceil(chunk.text.length / 4);
|
|
16
|
-
if (tokenTotal + chunkTokens > tokenBudget) {
|
|
17
|
-
// First chunk exceeds budget — truncate on paragraph boundary
|
|
18
|
-
if (selected.length === 0) {
|
|
19
|
-
const charLimit = tokenBudget * 4;
|
|
20
|
-
let truncated = chunk.text.slice(0, charLimit);
|
|
21
|
-
const lastBreak = Math.max(truncated.lastIndexOf("\n\n"), truncated.lastIndexOf(". "));
|
|
22
|
-
if (lastBreak > charLimit * 0.4)
|
|
23
|
-
truncated = truncated.slice(0, lastBreak + 1);
|
|
24
|
-
selected.push({
|
|
25
|
-
text: truncated + "\n\n[truncated]",
|
|
26
|
-
heading: sc.heading,
|
|
27
|
-
score: sc.score,
|
|
28
|
-
});
|
|
29
|
-
}
|
|
30
|
-
break;
|
|
31
|
-
}
|
|
32
|
-
selected.push({ text: chunk.text, heading: sc.heading, score: sc.score });
|
|
33
|
-
tokenTotal += chunkTokens;
|
|
34
|
-
}
|
|
35
|
-
return selected;
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* Fetch a web page and extract readable article content.
|
|
39
|
-
*
|
|
40
|
-
* Uses the Fetch API with a 10-second timeout, then extracts the
|
|
41
|
-
* article body via linkedom + Readability.
|
|
42
|
-
*
|
|
43
|
-
* When a reranker is set (via {@link setReranker}) and the agent provides
|
|
44
|
-
* a `query` argument, the article HTML is structurally chunked on heading
|
|
45
|
-
* boundaries (same pattern as corpus `parseMarkdown`) and scored against
|
|
46
|
-
* the query. Only the top-K most relevant verbatim chunks are returned —
|
|
47
|
-
* reducing KV pressure without lossy summarization. The reranker runs on
|
|
48
|
-
* its own `llama_context`, consuming zero inference KV.
|
|
49
|
-
*
|
|
50
|
-
* Without a reranker or query, returns the full content truncated to
|
|
51
|
-
* `maxChars` (default 6000). Fully backward compatible.
|
|
52
|
-
*
|
|
53
|
-
* @category Rig
|
|
54
|
-
*/
|
|
55
|
-
class FetchPageTool extends lloyal_agents_1.Tool {
|
|
56
|
-
name = "fetch_page";
|
|
57
|
-
description = "Fetch a web page and extract its article content. Returns readable text with title and excerpt. Pass a query to get only the most relevant sections.";
|
|
58
|
-
parameters = {
|
|
59
|
-
type: "object",
|
|
60
|
-
properties: {
|
|
61
|
-
url: { type: "string", description: "URL to fetch" },
|
|
62
|
-
query: {
|
|
63
|
-
type: "string",
|
|
64
|
-
description: "What to look for in this page (optional — improves relevance of returned content)",
|
|
65
|
-
},
|
|
66
|
-
},
|
|
67
|
-
required: ["url"],
|
|
68
|
-
};
|
|
69
|
-
_maxChars;
|
|
70
|
-
_reranker = null;
|
|
71
|
-
_topK;
|
|
72
|
-
_timeout;
|
|
73
|
-
_tokenBudget;
|
|
74
|
-
constructor(opts) {
|
|
75
|
-
super();
|
|
76
|
-
this._maxChars = opts?.maxChars ?? 6000;
|
|
77
|
-
this._topK = opts?.topK ?? 5;
|
|
78
|
-
this._timeout = opts?.timeout ?? 10_000;
|
|
79
|
-
this._tokenBudget = opts?.tokenBudget ?? 2048;
|
|
80
|
-
}
|
|
81
|
-
/** Inject reranker for chunk scoring. Call from Source.bind(). */
|
|
82
|
-
setReranker(reranker) {
|
|
83
|
-
this._reranker = reranker;
|
|
84
|
-
}
|
|
85
|
-
*execute(args, context) {
|
|
86
|
-
const url = args.url?.trim();
|
|
87
|
-
if (!url)
|
|
88
|
-
return { error: "url must not be empty" };
|
|
89
|
-
// Cross-agent dedup: another worker in this pool already fetched this URL
|
|
90
|
-
if (context?.peerHistory?.some(h => {
|
|
91
|
-
if (h.name !== 'fetch_page')
|
|
92
|
-
return false;
|
|
93
|
-
try {
|
|
94
|
-
const prev = JSON.parse(h.args).url;
|
|
95
|
-
return prev === url;
|
|
96
|
-
}
|
|
97
|
-
catch {
|
|
98
|
-
return false;
|
|
99
|
-
}
|
|
100
|
-
})) {
|
|
101
|
-
return { error: 'Resource unavailable. Try a different URL.' };
|
|
102
|
-
}
|
|
103
|
-
// Early reject PDF URLs
|
|
104
|
-
const lowerUrl = url.toLowerCase();
|
|
105
|
-
if (lowerUrl.endsWith(".pdf") ||
|
|
106
|
-
lowerUrl.includes(".pdf?") ||
|
|
107
|
-
lowerUrl.includes(".pdf#")) {
|
|
108
|
-
return {
|
|
109
|
-
error: "PDF documents cannot be extracted. Try searching for an HTML version of this content.",
|
|
110
|
-
url,
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
const maxChars = this._maxChars;
|
|
114
|
-
const reranker = this._reranker;
|
|
115
|
-
const topK = this._topK;
|
|
116
|
-
const timeout = this._timeout;
|
|
117
|
-
const tokenBudget = this._tokenBudget;
|
|
118
|
-
// Step 1: Fetch + readability (async)
|
|
119
|
-
const fetched = yield* (0, effection_1.call)(async () => {
|
|
120
|
-
const controller = new AbortController();
|
|
121
|
-
const timer = setTimeout(() => controller.abort(), timeout);
|
|
122
|
-
let res;
|
|
123
|
-
try {
|
|
124
|
-
res = await fetch(url, {
|
|
125
|
-
headers: {
|
|
126
|
-
"User-Agent": "Mozilla/5.0 (compatible; lloyal-agents/1.0)",
|
|
127
|
-
},
|
|
128
|
-
signal: controller.signal,
|
|
129
|
-
});
|
|
130
|
-
}
|
|
131
|
-
catch (err) {
|
|
132
|
-
return {
|
|
133
|
-
error: `Fetch failed: ${err.message}`,
|
|
134
|
-
url,
|
|
135
|
-
};
|
|
136
|
-
}
|
|
137
|
-
finally {
|
|
138
|
-
clearTimeout(timer);
|
|
139
|
-
}
|
|
140
|
-
if (!res.ok)
|
|
141
|
-
return { error: `HTTP ${res.status} ${res.statusText}`, url };
|
|
142
|
-
const contentType = res.headers.get("content-type") || "";
|
|
143
|
-
if (contentType.includes("application/pdf")) {
|
|
144
|
-
return {
|
|
145
|
-
error: "PDF documents cannot be extracted. Try searching for an HTML version of this content.",
|
|
146
|
-
url,
|
|
147
|
-
};
|
|
148
|
-
}
|
|
149
|
-
const html = await res.text();
|
|
150
|
-
const { parseHTML } = await Promise.resolve().then(() => require("linkedom"));
|
|
151
|
-
const { document } = parseHTML(html);
|
|
152
|
-
if (!document || !document.documentElement) {
|
|
153
|
-
return { url, content: "[Could not parse HTML]" };
|
|
154
|
-
}
|
|
155
|
-
const { Readability } = await Promise.resolve().then(() => require("@mozilla/readability"));
|
|
156
|
-
const article = new Readability(document).parse();
|
|
157
|
-
if (!article)
|
|
158
|
-
return { url, content: "[Could not extract article content]" };
|
|
159
|
-
return {
|
|
160
|
-
url,
|
|
161
|
-
title: article.title ?? "",
|
|
162
|
-
content: article.textContent ?? "",
|
|
163
|
-
articleHtml: article.content ?? "",
|
|
164
|
-
excerpt: article.excerpt ?? "",
|
|
165
|
-
};
|
|
166
|
-
});
|
|
167
|
-
// Early return on error or no article
|
|
168
|
-
if ("error" in fetched)
|
|
169
|
-
return fetched;
|
|
170
|
-
if (!fetched.articleHtml) {
|
|
171
|
-
let content = fetched.content;
|
|
172
|
-
if (content.length > maxChars)
|
|
173
|
-
content = content.slice(0, maxChars) + "\n\n[truncated]";
|
|
174
|
-
return {
|
|
175
|
-
url: fetched.url,
|
|
176
|
-
title: fetched.title,
|
|
177
|
-
content,
|
|
178
|
-
excerpt: fetched.excerpt,
|
|
179
|
-
};
|
|
180
|
-
}
|
|
181
|
-
// Step 2: Reranker path — chunk HTML structurally, score, return top-K
|
|
182
|
-
if (reranker && args.query) {
|
|
183
|
-
const chunks = yield* (0, effection_1.call)(() => (0, chunking_1.chunkHtml)(fetched.articleHtml, url, fetched.title));
|
|
184
|
-
// Write chunks to trace for replay sufficiency
|
|
185
|
-
let tw;
|
|
186
|
-
try {
|
|
187
|
-
tw = yield* lloyal_agents_1.Trace.expect();
|
|
188
|
-
}
|
|
189
|
-
catch {
|
|
190
|
-
/* no trace context */
|
|
191
|
-
}
|
|
192
|
-
const rerankT0 = performance.now();
|
|
193
|
-
if (tw) {
|
|
194
|
-
tw.write({
|
|
195
|
-
traceId: tw.nextId(),
|
|
196
|
-
parentTraceId: null,
|
|
197
|
-
ts: rerankT0,
|
|
198
|
-
type: "rerank:start",
|
|
199
|
-
query: args.query,
|
|
200
|
-
chunkCount: chunks.length,
|
|
201
|
-
tool: "fetch_page",
|
|
202
|
-
url,
|
|
203
|
-
chunks: chunks.map((c) => ({
|
|
204
|
-
heading: c.heading,
|
|
205
|
-
textLength: c.text.length,
|
|
206
|
-
startLine: c.startLine,
|
|
207
|
-
})),
|
|
208
|
-
});
|
|
209
|
-
}
|
|
210
|
-
if (chunks.length > 0) {
|
|
211
|
-
yield* (0, effection_1.call)(() => reranker.tokenizeChunks(chunks));
|
|
212
|
-
// Score chunks against agent's local query
|
|
213
|
-
let scored = [];
|
|
214
|
-
yield* (0, effection_1.call)(async () => {
|
|
215
|
-
for await (const batch of reranker.score(args.query, chunks)) {
|
|
216
|
-
if (context?.onProgress)
|
|
217
|
-
context.onProgress({ filled: batch.filled, total: batch.total });
|
|
218
|
-
scored = batch.results;
|
|
219
|
-
}
|
|
220
|
-
});
|
|
221
|
-
// Explore mode (default): agent-local scoring only. The agent chose
|
|
222
|
-
// this page — content scored against what it asked for. Filtering
|
|
223
|
-
// against original query removes bridging content that produces
|
|
224
|
-
// hypothesis greps. alsoOnPage provides discovery signals instead.
|
|
225
|
-
//
|
|
226
|
-
// Exploit mode (!explore): dual scoring via scoreRelevanceBatch.
|
|
227
|
-
if (!context?.explore && context?.scorer && scored.length > 0) {
|
|
228
|
-
const chunkTexts = scored.map((sc) => {
|
|
229
|
-
const chunk = chunks.find((c) => c.resource === sc.file && c.startLine === sc.startLine);
|
|
230
|
-
return chunk?.text ?? "";
|
|
231
|
-
});
|
|
232
|
-
const combinedScores = yield* (0, effection_1.call)(() => context.scorer.scoreRelevanceBatch(chunkTexts, args.query));
|
|
233
|
-
const reordered = scored
|
|
234
|
-
.map((sc, i) => ({
|
|
235
|
-
...sc,
|
|
236
|
-
score: combinedScores[i],
|
|
237
|
-
_toolQueryScore: sc.score,
|
|
238
|
-
}))
|
|
239
|
-
.sort((a, b) => b.score - a.score);
|
|
240
|
-
scored = reordered;
|
|
241
|
-
if (tw) {
|
|
242
|
-
tw.write({
|
|
243
|
-
traceId: tw.nextId(),
|
|
244
|
-
parentTraceId: null,
|
|
245
|
-
ts: performance.now(),
|
|
246
|
-
type: "entailment:content:exploit",
|
|
247
|
-
tool: "fetch_page",
|
|
248
|
-
pressure: {
|
|
249
|
-
percentAvailable: context.pressurePercentAvailable ?? -1,
|
|
250
|
-
remaining: -1,
|
|
251
|
-
nCtx: -1,
|
|
252
|
-
},
|
|
253
|
-
chunks: reordered.slice(0, 5).map((sc) => ({
|
|
254
|
-
heading: sc.heading,
|
|
255
|
-
toolQueryScore: sc._toolQueryScore,
|
|
256
|
-
combinedScore: sc.score,
|
|
257
|
-
})),
|
|
258
|
-
});
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
// Select top-K within token budget (tokens populated by tokenizeChunks)
|
|
262
|
-
const topChunks = selectTopChunks(scored, chunks, topK, tokenBudget);
|
|
263
|
-
if (tw) {
|
|
264
|
-
tw.write({
|
|
265
|
-
traceId: tw.nextId(),
|
|
266
|
-
parentTraceId: null,
|
|
267
|
-
ts: performance.now(),
|
|
268
|
-
type: "rerank:end",
|
|
269
|
-
topResults: topChunks.map((c) => ({
|
|
270
|
-
file: url,
|
|
271
|
-
heading: c.heading,
|
|
272
|
-
score: c.score,
|
|
273
|
-
textPreview: c.text.slice(0, 200),
|
|
274
|
-
})),
|
|
275
|
-
selectedPassageCount: topChunks.length,
|
|
276
|
-
totalChars: topChunks.reduce((sum, c) => sum + c.text.length, 0),
|
|
277
|
-
durationMs: performance.now() - rerankT0,
|
|
278
|
-
tool: "fetch_page",
|
|
279
|
-
url,
|
|
280
|
-
});
|
|
281
|
-
}
|
|
282
|
-
if (topChunks.length > 0) {
|
|
283
|
-
// Discovery signal: headings of chunks that didn't make the cut.
|
|
284
|
-
// Lightweight (~50 tokens) but gives the agent topics to explore.
|
|
285
|
-
const selectedHeadings = new Set(topChunks.map((c) => c.heading));
|
|
286
|
-
const alsoOnPage = scored
|
|
287
|
-
.filter((sc) => !selectedHeadings.has(sc.heading))
|
|
288
|
-
.map((sc) => sc.heading)
|
|
289
|
-
.filter((h, i, arr) => arr.indexOf(h) === i);
|
|
290
|
-
return {
|
|
291
|
-
url,
|
|
292
|
-
title: fetched.title,
|
|
293
|
-
content: topChunks.map((c) => c.text).join("\n\n---\n\n"),
|
|
294
|
-
chunks: topChunks.length,
|
|
295
|
-
...(alsoOnPage.length > 0 ? { alsoOnPage } : {}),
|
|
296
|
-
};
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
// Fallback: return full content, truncated
|
|
301
|
-
let content = fetched.content;
|
|
302
|
-
if (content.length > maxChars) {
|
|
303
|
-
content = content.slice(0, maxChars) + "\n\n[truncated]";
|
|
304
|
-
}
|
|
305
|
-
return { url, title: fetched.title, content, excerpt: fetched.excerpt };
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
exports.FetchPageTool = FetchPageTool;
|
|
309
|
-
//# sourceMappingURL=fetch-page.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-page.js","sourceRoot":"","sources":["../../src/tools/fetch-page.ts"],"names":[],"mappings":";;;AAAA,yCAAiC;AAEjC,8DAAyD;AAEzD,kDAAgD;AAIhD,wDAAwD;AACxD,SAAS,eAAe,CACtB,MAAqB,EACrB,MAAe,EACf,IAAY,EACZ,WAAmB;IAEnB,MAAM,QAAQ,GAA4D,EAAE,CAAC;IAC7E,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CACvB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAC9D,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,IAAI;YAAE,SAAS;QAE3B,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE5E,IAAI,UAAU,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC;YAC3C,8DAA8D;YAC9D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,SAAS,GAAG,WAAW,GAAG,CAAC,CAAC;gBAClC,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;gBAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,EAC7B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAC5B,CAAC;gBACF,IAAI,SAAS,GAAG,SAAS,GAAG,GAAG;oBAC7B,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;gBAChD,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,SAAS,GAAG,iBAAiB;oBACnC,OAAO,EAAE,EAAE,CAAC,OAAO;oBACnB,KAAK,EAAE,EAAE,CAAC,KAAK;iBAChB,CAAC,CAAC;YACL,CAAC;YACD,MAAM;QACR,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;QAC1E,UAAU,IAAI,WAAW,CAAC;IAC5B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,aAAc,SAAQ,oBAAqC;IAC7D,IAAI,GAAG,YAAY,CAAC;IACpB,WAAW,GAClB,sJAAsJ,CAAC;IAChJ,UAAU,GAAe;QAChC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;YACpD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,mFAAmF;aACtF;SACF;QACD,QAAQ,EAAE,CAAC,KAAK,CAAC;KAClB,CAAC;IAEM,SAAS,CAAS;IAClB,SAAS,GAAoB,IAAI,CAAC;IAClC,KAAK,CAAS;IACd,QAAQ,CAAS;IACjB,YAAY,CAAS;IAE7B,YAAY,IAKX;QACC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,SAAS,GAAG,IAAI,EAAE,QAAQ,IAAI,IAAI,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,IAAI,EAAE,OAAO,IAAI,MAAM,CAAC;QACxC,IAAI,CAAC,YAAY,GAAG,IAAI,EAAE,WAAW,IAAI,IAAI,CAAC;IAChD,CAAC;IAED,kEAAkE;IAClE,WAAW,CAAC,QAAkB;QAC5B,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,CAAC,OAAO,CACN,IAAqC,EACrC,OAAqB;QAErB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC;QAEpD,0EAA0E;QAC1E,IAAI,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE;YACjC,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY;gBAAE,OAAO,KAAK,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,IAAI,GAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAsB,CAAC,GAAG,CAAC;gBAC1D,OAAO,IAAI,KAAK,GAAG,CAAC;YACtB,CAAC;YAAC,MAAM,CAAC;gBAAC,OAAO,KAAK,CAAC;YAAC,CAAC;QAC3B,CAAC,CAAC,EAAE,CAAC;YACH,OAAO,EAAE,KAAK,EAAE,4CAA4C,EAAE,CAAC;QACjE,CAAC;QAED,wBAAwB;QACxB,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACnC,IACE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YACzB,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC1B,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAC1B,CAAC;YACD,OAAO;gBACL,KAAK,EACH,uFAAuF;gBACzF,GAAG;aACJ,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC;QAEtC,sCAAsC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,KAAK,IAAI,EAAE;YACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;YAC5D,IAAI,GAAa,CAAC;YAClB,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBACrB,OAAO,EAAE;wBACP,YAAY,EAAE,6CAA6C;qBAC5D;oBACD,MAAM,EAAE,UAAU,CAAC,MAAM;iBAC1B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,KAAK,EAAE,iBAAkB,GAAa,CAAC,OAAO,EAAE;oBAChD,GAAG;iBACK,CAAC;YACb,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,EAAE;gBACT,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,EAAW,CAAC;YAEzE,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC1D,IAAI,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC5C,OAAO;oBACL,KAAK,EACH,uFAAuF;oBACzF,GAAG;iBACK,CAAC;YACb,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAE9B,MAAM,EAAE,SAAS,EAAE,GAAG,2CAAa,UAAU,EAAC,CAAC;YAC/C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAErC,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC;gBAC3C,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,wBAAwB,EAAW,CAAC;YAC7D,CAAC;YAED,MAAM,EAAE,WAAW,EAAE,GAAG,2CAAa,sBAAsB,EAAC,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;YAElD,IAAI,CAAC,OAAO;gBACV,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,qCAAqC,EAAW,CAAC;YAE1E,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE;gBAC1B,OAAO,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;gBAClC,WAAW,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;gBAClC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;aACtB,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,sCAAsC;QACtC,IAAI,OAAO,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;YAC9B,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ;gBAC3B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC;YAC3D,OAAO;gBACL,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC;QACJ,CAAC;QAED,uEAAuE;QACvE,IAAI,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CAC9B,IAAA,oBAAS,EAAC,OAAO,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CACnD,CAAC;YAEF,+CAA+C;YAC/C,IAAI,EAAE,CAAC;YACP,IAAI,CAAC;gBACH,EAAE,GAAG,KAAK,CAAC,CAAC,qBAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;YACD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,EAAE,EAAE,CAAC;gBACP,EAAE,CAAC,KAAK,CAAC;oBACP,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE;oBACpB,aAAa,EAAE,IAAI;oBACnB,EAAE,EAAE,QAAQ;oBACZ,IAAI,EAAE,cAAc;oBACpB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,IAAI,EAAE,YAAY;oBAClB,GAAG;oBACH,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBACzB,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;wBACzB,SAAS,EAAE,CAAC,CAAC,SAAS;qBACvB,CAAC,CAAC;iBACJ,CAAC,CAAC;YACL,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;gBAEnD,2CAA2C;gBAC3C,IAAI,MAAM,GAAkB,EAAE,CAAC;gBAC/B,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,KAAK,IAAI,EAAE;oBACrB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAM,EAAE,MAAM,CAAC,EAAE,CAAC;wBAC9D,IAAI,OAAO,EAAE,UAAU;4BACrB,OAAO,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;wBACnE,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;oBACzB,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,oEAAoE;gBACpE,kEAAkE;gBAClE,gEAAgE;gBAChE,mEAAmE;gBACnE,EAAE;gBACF,iEAAiE;gBACjE,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAE9D,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;wBACnC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CACvB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,SAAS,KAAK,EAAE,CAAC,SAAS,CAC9D,CAAC;wBACF,OAAO,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC;oBAC3B,CAAC,CAAC,CAAC;oBACH,MAAM,cAAc,GAAa,KAAK,CAAC,CAAC,IAAA,gBAAI,EAAC,GAAG,EAAE,CAChD,OAAO,CAAC,MAAO,CAAC,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,KAAM,CAAC,CAC7D,CAAC;oBACF,MAAM,SAAS,GAAyB,MAAM;yBAC3C,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;wBACf,GAAG,EAAE;wBACL,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC;wBACxB,eAAe,EAAE,EAAE,CAAC,KAAK;qBAC1B,CAAC,CAAC;yBACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;oBACrC,MAAM,GAAG,SAAS,CAAC;oBAEnB,IAAI,EAAE,EAAE,CAAC;wBACP,EAAE,CAAC,KAAK,CAAC;4BACP,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE;4BACpB,aAAa,EAAE,IAAI;4BACnB,EAAE,EAAE,WAAW,CAAC,GAAG,EAAE;4BACrB,IAAI,EAAE,4BAA4B;4BAClC,IAAI,EAAE,YAAY;4BAClB,QAAQ,EAAE;gCACR,gBAAgB,EAAE,OAAO,CAAC,wBAAwB,IAAI,CAAC,CAAC;gCACxD,SAAS,EAAE,CAAC,CAAC;gCACb,IAAI,EAAE,CAAC,CAAC;6BACT;4BACD,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gCACzC,OAAO,EAAE,EAAE,CAAC,OAAO;gCACnB,cAAc,EAAE,EAAE,CAAC,eAAe;gCAClC,aAAa,EAAE,EAAE,CAAC,KAAK;6BACxB,CAAC,CAAC;yBACJ,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,wEAAwE;gBACxE,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;gBAErE,IAAI,EAAE,EAAE,CAAC;oBACP,EAAE,CAAC,KAAK,CAAC;wBACP,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE;wBACpB,aAAa,EAAE,IAAI;wBACnB,EAAE,EAAE,WAAW,CAAC,GAAG,EAAE;wBACrB,IAAI,EAAE,YAAY;wBAClB,UAAU,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;4BAChC,IAAI,EAAE,GAAG;4BACT,OAAO,EAAE,CAAC,CAAC,OAAO;4BAClB,KAAK,EAAE,CAAC,CAAC,KAAK;4BACd,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;yBAClC,CAAC,CAAC;wBACH,oBAAoB,EAAE,SAAS,CAAC,MAAM;wBACtC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;wBAChE,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,QAAQ;wBACxC,IAAI,EAAE,YAAY;wBAClB,GAAG;qBACJ,CAAC,CAAC;gBACL,CAAC;gBAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,iEAAiE;oBACjE,kEAAkE;oBAClE,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;oBAClE,MAAM,UAAU,GAAG,MAAM;yBACtB,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;yBACjD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;yBACvB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;oBAE/C,OAAO;wBACL,GAAG;wBACH,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;wBACzD,MAAM,EAAE,SAAS,CAAC,MAAM;wBACxB,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBACjD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC9B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC;QAC3D,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IAC1E,CAAC;CACF;AApSD,sCAoSC"}
|
package/dist/tools/grep.d.ts
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
import type { Operation } from 'effection';
|
|
2
|
-
import { Tool } from '@lloyal-labs/lloyal-agents';
|
|
3
|
-
import type { JsonSchema } from '@lloyal-labs/lloyal-agents';
|
|
4
|
-
import type { Resource } from '../resources/types';
|
|
5
|
-
/**
|
|
6
|
-
* Exhaustive regex search across all corpus resources
|
|
7
|
-
*
|
|
8
|
-
* Scans every line of every loaded {@link Resource} for matches
|
|
9
|
-
* against a regular expression. Returns matching lines with file
|
|
10
|
-
* names and line numbers, capped at 50 results. Complements
|
|
11
|
-
* {@link SearchTool} which ranks by semantic relevance -- grep
|
|
12
|
-
* finds exact patterns exhaustively.
|
|
13
|
-
*
|
|
14
|
-
* @category Rig
|
|
15
|
-
*/
|
|
16
|
-
export declare class GrepTool extends Tool<{
|
|
17
|
-
pattern: string;
|
|
18
|
-
ignoreCase?: boolean;
|
|
19
|
-
}> {
|
|
20
|
-
readonly name = "grep";
|
|
21
|
-
readonly description = "Search the entire corpus for a regex pattern. Returns every matching line with line numbers and total match count. Complements search() which ranks by relevance \u2014 grep scans exhaustively.";
|
|
22
|
-
readonly parameters: JsonSchema;
|
|
23
|
-
private _resources;
|
|
24
|
-
private _maxResults;
|
|
25
|
-
private _lineMaxChars;
|
|
26
|
-
constructor(resources: Resource[], opts?: {
|
|
27
|
-
maxResults?: number;
|
|
28
|
-
lineMaxChars?: number;
|
|
29
|
-
});
|
|
30
|
-
execute(args: {
|
|
31
|
-
pattern: string;
|
|
32
|
-
ignoreCase?: boolean;
|
|
33
|
-
}): Operation<unknown>;
|
|
34
|
-
}
|
|
35
|
-
//# sourceMappingURL=grep.d.ts.map
|
package/dist/tools/grep.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"grep.d.ts","sourceRoot":"","sources":["../../src/tools/grep.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD;;;;;;;;;;GAUG;AACH,qBAAa,QAAS,SAAQ,IAAI,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;IAC3E,QAAQ,CAAC,IAAI,UAAU;IACvB,QAAQ,CAAC,WAAW,sMAAiM;IACrN,QAAQ,CAAC,UAAU,EAAE,UAAU,CAO7B;IAEF,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAS;gBAElB,SAAS,EAAE,QAAQ,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;IAOvF,OAAO,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,SAAS,CAAC,OAAO,CAAC;CA4C9E"}
|