@staticn0va/wigolo 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -2
- package/SKILL.md +149 -0
- package/dist/cache/change-detector.d.ts +7 -0
- package/dist/cache/change-detector.d.ts.map +1 -0
- package/dist/cache/change-detector.js +43 -0
- package/dist/cache/change-detector.js.map +1 -0
- package/dist/cache/diff-summary.d.ts +2 -0
- package/dist/cache/diff-summary.d.ts.map +1 -0
- package/dist/cache/diff-summary.js +87 -0
- package/dist/cache/diff-summary.js.map +1 -0
- package/dist/cache/store.d.ts +2 -0
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +10 -0
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/auth.d.ts +2 -0
- package/dist/cli/auth.d.ts.map +1 -0
- package/dist/cli/auth.js +95 -0
- package/dist/cli/auth.js.map +1 -0
- package/dist/cli/daemon.d.ts +6 -1
- package/dist/cli/daemon.d.ts.map +1 -1
- package/dist/cli/daemon.js +56 -3
- package/dist/cli/daemon.js.map +1 -1
- package/dist/cli/health.d.ts +1 -1
- package/dist/cli/health.d.ts.map +1 -1
- package/dist/cli/health.js +41 -3
- package/dist/cli/health.js.map +1 -1
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/plugin.d.ts +5 -0
- package/dist/cli/plugin.d.ts.map +1 -0
- package/dist/cli/plugin.js +188 -0
- package/dist/cli/plugin.js.map +1 -0
- package/dist/cli/shell.d.ts +2 -0
- package/dist/cli/shell.d.ts.map +1 -0
- package/dist/cli/shell.js +86 -0
- package/dist/cli/shell.js.map +1 -0
- package/dist/cli/warmup.d.ts +4 -0
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +48 -1
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +7 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +15 -0
- package/dist/config.js.map +1 -1
- package/dist/daemon/health-check.d.ts +16 -0
- package/dist/daemon/health-check.d.ts.map +1 -0
- package/dist/daemon/health-check.js +36 -0
- package/dist/daemon/health-check.js.map +1 -0
- package/dist/daemon/http-server.d.ts +26 -0
- package/dist/daemon/http-server.d.ts.map +1 -0
- package/dist/daemon/http-server.js +282 -0
- package/dist/daemon/http-server.js.map +1 -0
- package/dist/daemon/proxy.d.ts +10 -0
- package/dist/daemon/proxy.d.ts.map +1 -0
- package/dist/daemon/proxy.js +99 -0
- package/dist/daemon/proxy.js.map +1 -0
- package/dist/fetch/action-executor.d.ts +28 -0
- package/dist/fetch/action-executor.d.ts.map +1 -0
- package/dist/fetch/action-executor.js +86 -0
- package/dist/fetch/action-executor.js.map +1 -0
- package/dist/fetch/auth.d.ts +2 -1
- package/dist/fetch/auth.d.ts.map +1 -1
- package/dist/fetch/auth.js +30 -2
- package/dist/fetch/auth.js.map +1 -1
- package/dist/fetch/browser-pool.d.ts +30 -11
- package/dist/fetch/browser-pool.d.ts.map +1 -1
- package/dist/fetch/browser-pool.js +165 -49
- package/dist/fetch/browser-pool.js.map +1 -1
- package/dist/fetch/browser-selector.d.ts +17 -0
- package/dist/fetch/browser-selector.d.ts.map +1 -0
- package/dist/fetch/browser-selector.js +70 -0
- package/dist/fetch/browser-selector.js.map +1 -0
- package/dist/fetch/browser-types.d.ts +3 -0
- package/dist/fetch/browser-types.d.ts.map +1 -0
- package/dist/fetch/browser-types.js +45 -0
- package/dist/fetch/browser-types.js.map +1 -0
- package/dist/fetch/cdp-client.d.ts +9 -0
- package/dist/fetch/cdp-client.d.ts.map +1 -0
- package/dist/fetch/cdp-client.js +90 -0
- package/dist/fetch/cdp-client.js.map +1 -0
- package/dist/fetch/router.d.ts +4 -1
- package/dist/fetch/router.d.ts.map +1 -1
- package/dist/fetch/router.js +8 -2
- package/dist/fetch/router.js.map +1 -1
- package/dist/index.js +32 -3
- package/dist/index.js.map +1 -1
- package/dist/instructions.d.ts +26 -0
- package/dist/instructions.d.ts.map +1 -0
- package/dist/instructions.js +101 -0
- package/dist/instructions.js.map +1 -0
- package/dist/logger.d.ts +1 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/plugins/loader.d.ts +20 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +162 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/registry.d.ts +26 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +68 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/validate.d.ts +9 -0
- package/dist/plugins/validate.d.ts.map +1 -0
- package/dist/plugins/validate.js +70 -0
- package/dist/plugins/validate.js.map +1 -0
- package/dist/repl/commands/cache.d.ts +4 -0
- package/dist/repl/commands/cache.d.ts.map +1 -0
- package/dist/repl/commands/cache.js +44 -0
- package/dist/repl/commands/cache.js.map +1 -0
- package/dist/repl/commands/crawl.d.ts +7 -0
- package/dist/repl/commands/crawl.d.ts.map +1 -0
- package/dist/repl/commands/crawl.js +42 -0
- package/dist/repl/commands/crawl.js.map +1 -0
- package/dist/repl/commands/extract.d.ts +5 -0
- package/dist/repl/commands/extract.d.ts.map +1 -0
- package/dist/repl/commands/extract.js +37 -0
- package/dist/repl/commands/extract.js.map +1 -0
- package/dist/repl/commands/fetch.d.ts +5 -0
- package/dist/repl/commands/fetch.d.ts.map +1 -0
- package/dist/repl/commands/fetch.js +53 -0
- package/dist/repl/commands/fetch.js.map +1 -0
- package/dist/repl/commands/search.d.ts +5 -0
- package/dist/repl/commands/search.d.ts.map +1 -0
- package/dist/repl/commands/search.js +62 -0
- package/dist/repl/commands/search.js.map +1 -0
- package/dist/repl/commands/types.d.ts +9 -0
- package/dist/repl/commands/types.d.ts.map +1 -0
- package/dist/repl/commands/types.js +2 -0
- package/dist/repl/commands/types.js.map +1 -0
- package/dist/repl/formatters.d.ts +10 -0
- package/dist/repl/formatters.d.ts.map +1 -0
- package/dist/repl/formatters.js +209 -0
- package/dist/repl/formatters.js.map +1 -0
- package/dist/repl/parser.d.ts +9 -0
- package/dist/repl/parser.d.ts.map +1 -0
- package/dist/repl/parser.js +84 -0
- package/dist/repl/parser.js.map +1 -0
- package/dist/repl/shell.d.ts +8 -0
- package/dist/repl/shell.d.ts.map +1 -0
- package/dist/repl/shell.js +156 -0
- package/dist/repl/shell.js.map +1 -0
- package/dist/search/context-formatter.d.ts +3 -0
- package/dist/search/context-formatter.d.ts.map +1 -0
- package/dist/search/context-formatter.js +56 -0
- package/dist/search/context-formatter.js.map +1 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +204 -104
- package/dist/server.js.map +1 -1
- package/dist/tools/cache.d.ts +2 -1
- package/dist/tools/cache.d.ts.map +1 -1
- package/dist/tools/cache.js +56 -1
- package/dist/tools/cache.js.map +1 -1
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +16 -1
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +10 -0
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +48 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +36 -3
package/dist/tools/fetch.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { extractContent } from '../extraction/pipeline.js';
|
|
2
2
|
import { getCachedContent, cacheContent, isExpired } from '../cache/store.js';
|
|
3
3
|
import { extractSection } from '../extraction/markdown.js';
|
|
4
|
+
import { detectChange } from '../cache/change-detector.js';
|
|
4
5
|
import { createLogger } from '../logger.js';
|
|
5
6
|
const log = createLogger('fetch');
|
|
6
7
|
function formatCachedResponse(cached, input) {
|
|
@@ -30,7 +31,7 @@ function formatCachedResponse(cached, input) {
|
|
|
30
31
|
export async function handleFetch(input, router) {
|
|
31
32
|
try {
|
|
32
33
|
const cached = getCachedContent(input.url);
|
|
33
|
-
if (cached && !isExpired(cached)) {
|
|
34
|
+
if (cached && !isExpired(cached) && (!input.actions || input.actions.length === 0)) {
|
|
34
35
|
log.info('Serving from cache', { url: input.url });
|
|
35
36
|
return formatCachedResponse(cached, input);
|
|
36
37
|
}
|
|
@@ -39,6 +40,7 @@ export async function handleFetch(input, router) {
|
|
|
39
40
|
useAuth: input.use_auth ?? false,
|
|
40
41
|
headers: input.headers,
|
|
41
42
|
screenshot: input.screenshot,
|
|
43
|
+
actions: input.actions,
|
|
42
44
|
});
|
|
43
45
|
const extraction = await extractContent(raw.html, raw.finalUrl, {
|
|
44
46
|
maxChars: input.max_chars,
|
|
@@ -47,6 +49,13 @@ export async function handleFetch(input, router) {
|
|
|
47
49
|
contentType: raw.contentType,
|
|
48
50
|
pdfBuffer: raw.rawBuffer,
|
|
49
51
|
});
|
|
52
|
+
let changeResult;
|
|
53
|
+
try {
|
|
54
|
+
changeResult = detectChange(raw.finalUrl, extraction.markdown);
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
log.warn('change detection failed', { url: raw.finalUrl, error: String(err) });
|
|
58
|
+
}
|
|
50
59
|
cacheContent(raw, extraction);
|
|
51
60
|
return {
|
|
52
61
|
url: raw.finalUrl,
|
|
@@ -57,6 +66,12 @@ export async function handleFetch(input, router) {
|
|
|
57
66
|
images: extraction.images,
|
|
58
67
|
screenshot: raw.screenshot,
|
|
59
68
|
cached: false,
|
|
69
|
+
action_results: raw.actionResults,
|
|
70
|
+
...(changeResult?.changed ? {
|
|
71
|
+
changed: true,
|
|
72
|
+
previous_hash: changeResult.previousHash,
|
|
73
|
+
diff_summary: changeResult.diffSummary,
|
|
74
|
+
} : {}),
|
|
60
75
|
};
|
|
61
76
|
}
|
|
62
77
|
catch (err) {
|
package/dist/tools/fetch.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,SAAS,oBAAoB,CAAC,MAAqB,EAAE,KAAiB;IACpE,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC/B,IAAI,cAAmC,CAAC;IAExC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5E,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;QAC1B,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC;IAClC,CAAC;IAED,IAAI,KAAK,CAAC,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACzD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAED,OAAO;QACL,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ;QACR,QAAQ,EAAE;YACR,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;YACtC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7E;QACD,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC;QACvC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC;QACzC,MAAM,EAAE,IAAI;KACb,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAiB,EACjB,MAAmB;IAEnB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,SAAS,oBAAoB,CAAC,MAAqB,EAAE,KAAiB;IACpE,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC/B,IAAI,cAAmC,CAAC;IAExC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5E,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;QAC1B,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC;IAClC,CAAC;IAED,IAAI,KAAK,CAAC,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACzD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAED,OAAO;QACL,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ;QACR,QAAQ,EAAE;YACR,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;YACtC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7E;QACD,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC;QACvC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC;QACzC,MAAM,EAAE,IAAI;KACb,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAiB,EACjB,MAAmB;IAEnB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YACnF,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YACnD,OAAO,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,MAAM;YACnC,OAAO,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK;YAChC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE;YAC9D,QAAQ,EAAE,KAAK,CAAC,SAAS;YACzB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,YAAY,EAAE,KAAK,CAAC,aAAa;YACjC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC,CAAC;QAEH,IAAI,YAA2F,CAAC;QAChG,IAAI,CAAC;YACH,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAE9B,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,GAAG,CAAC,aAAa;YACjC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC1B,OAAO,EAAE,IAAI;gBACb,aAAa,EAAE,YAAY,CAAC,YAAY;gBACxC,YAAY,EAAE,YAAY,CAAC,WAAW;aACvC,CAAC,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO;YACL,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,EAAE;YACT,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAoB,YAAY,EAAmB,MAAM,aAAa,CAAC;AAC9G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAoB,YAAY,EAAmB,MAAM,aAAa,CAAC;AAC9G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAmBjE,wBAAsB,YAAY,CAChC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,YAAY,CAAC,CA8IvB"}
|
package/dist/tools/search.js
CHANGED
|
@@ -3,6 +3,7 @@ import { decomposeQuery } from '../search/query.js';
|
|
|
3
3
|
import { validateLinks } from '../search/validator.js';
|
|
4
4
|
import { rerankResults } from '../search/rerank.js';
|
|
5
5
|
import { applyAllFilters } from '../search/filters.js';
|
|
6
|
+
import { formatSearchContext } from '../search/context-formatter.js';
|
|
6
7
|
import { extractContent } from '../extraction/pipeline.js';
|
|
7
8
|
import { cacheSearchResults, getCachedSearchResults } from '../cache/store.js';
|
|
8
9
|
import { getConfig } from '../config.js';
|
|
@@ -33,6 +34,9 @@ export async function handleSearch(input, engines, router, backendStatus) {
|
|
|
33
34
|
const warning = backendStatus?.consumeWarning();
|
|
34
35
|
if (warning)
|
|
35
36
|
output.warning = warning;
|
|
37
|
+
if (input.format === 'context') {
|
|
38
|
+
output.context_text = formatSearchContext(output.results, maxTotalChars);
|
|
39
|
+
}
|
|
36
40
|
return output;
|
|
37
41
|
}
|
|
38
42
|
let activeEngines = engines;
|
|
@@ -86,6 +90,9 @@ export async function handleSearch(input, engines, router, backendStatus) {
|
|
|
86
90
|
const warning = backendStatus?.consumeWarning();
|
|
87
91
|
if (warning)
|
|
88
92
|
output.warning = warning;
|
|
93
|
+
if (input.format === 'context') {
|
|
94
|
+
output.context_text = '';
|
|
95
|
+
}
|
|
89
96
|
return output;
|
|
90
97
|
}
|
|
91
98
|
let merged = deduplicateResults(allRaw);
|
|
@@ -130,6 +137,9 @@ export async function handleSearch(input, engines, router, backendStatus) {
|
|
|
130
137
|
const warning = backendStatus?.consumeWarning();
|
|
131
138
|
if (warning)
|
|
132
139
|
output.warning = warning;
|
|
140
|
+
if (input.format === 'context') {
|
|
141
|
+
output.context_text = formatSearchContext(output.results, maxTotalChars);
|
|
142
|
+
}
|
|
133
143
|
return output;
|
|
134
144
|
}
|
|
135
145
|
// v1: sequential fetch for correct budget enforcement. v2: parallel fetch then apply budget in relevance order.
|
package/dist/tools/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,eAAe,GAAG,EAAE,CAAC;AAC3B,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,uBAAuB,GAAG,KAAK,CAAC;AAEtC,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAkB,EAClB,OAAuB,EACvB,MAAmB,EACnB,aAA6B;IAE7B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,mBAAmB,EAAE,eAAe,CAAC,CAAC;IACvF,MAAM,cAAc,GAAG,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC;IACrD,MAAM,eAAe,GAAG,KAAK,CAAC,iBAAiB,IAAI,yBAAyB,CAAC;IAC7E,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe,IAAI,uBAAuB,CAAC;IACvE,MAAM,cAAc,GAAG,MAAM,CAAC,oBAAoB,CAAC;IACnD,MAAM,cAAc,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAEnD,MAAM,MAAM,GAAG,sBAAsB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACtE,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC;YAC5C,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAClC,CAAC;QACF,MAAM,OAAO,GAAG,aAAa,EAAE,cAAc,EAAE,CAAC;QAChD,IAAI,OAAO;YAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,CAAC,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,aAAa,GAAG,OAAO,CAAC;IAC5B,IAAI,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5D,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,cAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5E,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;YACrG,aAAa,GAAG,OAAO,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/C,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAEtF,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,wFAAwF;IACxF,MAAM,kBAAkB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,EAAE,MAAM,IAAI,KAAK,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9F,MAAM,eAAe,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnD,MAAM,cAAc,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CACpD,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE;gBACzC,UAAU,EAAE,UAAU,GAAG,eAAe;gBACxC,SAAS,EAAE,KAAK,CAAC,UAAU;gBAC3B,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,cAAc,EAAE,KAAK,CAAC,eAAe;gBACrC,cAAc,EAAE,KAAK,CAAC,eAAe;gBACrC,QAAQ,EAAE,KAAK,CAAC,SAAS;gBACzB,MAAM,EAAE,KAAK,CAAC,OAAO;gBACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB,CAAC,CAAC;YACH,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACf,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IAEzC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,YAAY,EAAE,CAAC,GAAG,WAAW,CAAC;YAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YACjC,KAAK,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,kBAAkB;SAClE,CAAC;QACF,MAAM,OAAO,GAAG,aAAa,EAAE,cAAc,EAAE,CAAC;QAChD,IAAI,OAAO;YAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,CAAC,YAAY,GAAG,EAAE,CAAC;QAC3B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAExC,0EAA0E;IAC1E,4EAA4E;IAC5E,MAAM,GAAG,eAAe,CAAC,MAAM,EAAE;QAC/B,cAAc,EAAE,KAAK,CAAC,eAAe;QACrC,cAAc,EAAE,KAAK,CAAC,eAAe;QACrC,QAAQ,EAAE,KAAK,CAAC,SAAS;QACzB,MAAM,EAAE,KAAK,CAAC,OAAO;QACrB,QAAQ,EAAE,KAAK,CAAC,QAAQ;KACzB,CAAC,CAAC;IAEH,MAAM,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAErC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAErC,MAAM,OAAO,GAAuB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnD,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,eAAe,EAAE,CAAC,CAAC,eAAe;KACnC,CAAC,CAAC,CAAC;IAEJ,IAAI,cAAc,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,sBAAsB,CAAC,OAAO,EAAE,MAAM,EAAE;YAC5C,eAAe;YACf,aAAa;YACb,cAAc;YACd,aAAa,EAAE,KAAK,GAAG,cAAc;SACtC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC;QACH,kBAAkB,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,MAAM,GAAiB;QAC3B,OAAO;QACP,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY,EAAE,CAAC,GAAG,WAAW,CAAC;QAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAClC,CAAC;IACF,MAAM,OAAO,GAAG,aAAa,EAAE,cAAc,EAAE,CAAC;IAChD,IAAI,OAAO;QAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,CAAC,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AASD,gHAAgH;AAChH,KAAK,UAAU,sBAAsB,CACnC,OAA2B,EAC3B,MAAmB,EACnB,GAAiB;IAEjB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YACpC,MAAM,CAAC,YAAY,GAAG,eAAe,CAAC;YACtC,SAAS;QACX,CAAC;QAED,IAAI,cAAc,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;YACxC,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAChC,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAC7B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;gBAC9C,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,cAAc,CAAC,CACnE;aACF,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE;gBAC9D,QAAQ,EAAE,GAAG,CAAC,eAAe;gBAC7B,WAAW,EAAE,GAAG,CAAC,WAAW;aAC7B,CAAC,CAAC;YAEH,IAAI,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC;YAElC,MAAM,SAAS,GAAG,GAAG,CAAC,aAAa,GAAG,cAAc,CAAC;YACrD,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;gBAC/B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;gBACtC,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAClC,CAAC;YAED,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;YACjC,MAAM,CAAC,gBAAgB,GAAG,OAAO,CAAC;QACpC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,GAAG,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACnE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC;QAC5B,CAAC;IACH,CAAC;AACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,4 +1,32 @@
|
|
|
1
1
|
import type { JsonSchema } from './extraction/schema.js';
|
|
2
|
+
export type BrowserAction = {
|
|
3
|
+
type: 'click';
|
|
4
|
+
selector: string;
|
|
5
|
+
} | {
|
|
6
|
+
type: 'type';
|
|
7
|
+
selector: string;
|
|
8
|
+
text: string;
|
|
9
|
+
} | {
|
|
10
|
+
type: 'wait';
|
|
11
|
+
ms: number;
|
|
12
|
+
} | {
|
|
13
|
+
type: 'wait_for';
|
|
14
|
+
selector: string;
|
|
15
|
+
timeout?: number;
|
|
16
|
+
} | {
|
|
17
|
+
type: 'scroll';
|
|
18
|
+
direction: 'down' | 'up';
|
|
19
|
+
amount?: number;
|
|
20
|
+
} | {
|
|
21
|
+
type: 'screenshot';
|
|
22
|
+
};
|
|
23
|
+
export interface ActionResult {
|
|
24
|
+
action_index: number;
|
|
25
|
+
type: BrowserAction['type'];
|
|
26
|
+
success: boolean;
|
|
27
|
+
error?: string;
|
|
28
|
+
screenshot?: string;
|
|
29
|
+
}
|
|
2
30
|
export interface FetchInput {
|
|
3
31
|
url: string;
|
|
4
32
|
render_js?: 'auto' | 'always' | 'never';
|
|
@@ -8,6 +36,7 @@ export interface FetchInput {
|
|
|
8
36
|
section_index?: number;
|
|
9
37
|
screenshot?: boolean;
|
|
10
38
|
headers?: Record<string, string>;
|
|
39
|
+
actions?: BrowserAction[];
|
|
11
40
|
}
|
|
12
41
|
export interface FetchOutput {
|
|
13
42
|
url: string;
|
|
@@ -25,6 +54,10 @@ export interface FetchOutput {
|
|
|
25
54
|
screenshot?: string;
|
|
26
55
|
cached: boolean;
|
|
27
56
|
error?: string;
|
|
57
|
+
action_results?: ActionResult[];
|
|
58
|
+
changed?: boolean;
|
|
59
|
+
previous_hash?: string;
|
|
60
|
+
diff_summary?: string;
|
|
28
61
|
}
|
|
29
62
|
export interface RawFetchResult {
|
|
30
63
|
url: string;
|
|
@@ -36,6 +69,7 @@ export interface RawFetchResult {
|
|
|
36
69
|
headers: Record<string, string>;
|
|
37
70
|
rawBuffer?: Buffer;
|
|
38
71
|
screenshot?: string;
|
|
72
|
+
actionResults?: ActionResult[];
|
|
39
73
|
}
|
|
40
74
|
export interface ExtractionResult {
|
|
41
75
|
title: string;
|
|
@@ -57,6 +91,8 @@ export interface CDPSession {
|
|
|
57
91
|
url: string;
|
|
58
92
|
title: string;
|
|
59
93
|
webSocketDebuggerUrl: string;
|
|
94
|
+
type?: string;
|
|
95
|
+
devtoolsFrontendUrl?: string;
|
|
60
96
|
}
|
|
61
97
|
export interface CachedContent {
|
|
62
98
|
id: number;
|
|
@@ -93,6 +129,7 @@ export interface SearchInput {
|
|
|
93
129
|
from_date?: string;
|
|
94
130
|
to_date?: string;
|
|
95
131
|
category?: 'general' | 'news' | 'code' | 'docs' | 'papers' | 'images';
|
|
132
|
+
format?: 'full' | 'context';
|
|
96
133
|
}
|
|
97
134
|
export interface SearchResultItem {
|
|
98
135
|
title: string;
|
|
@@ -110,6 +147,7 @@ export interface SearchOutput {
|
|
|
110
147
|
total_time_ms: number;
|
|
111
148
|
error?: string;
|
|
112
149
|
warning?: string;
|
|
150
|
+
context_text?: string;
|
|
113
151
|
}
|
|
114
152
|
export interface RawSearchResult {
|
|
115
153
|
title: string;
|
|
@@ -173,6 +211,7 @@ export interface CacheInput {
|
|
|
173
211
|
since?: string;
|
|
174
212
|
clear?: boolean;
|
|
175
213
|
stats?: boolean;
|
|
214
|
+
check_changes?: boolean;
|
|
176
215
|
}
|
|
177
216
|
export interface CacheResultItem {
|
|
178
217
|
url: string;
|
|
@@ -191,6 +230,15 @@ export interface CacheOutput {
|
|
|
191
230
|
stats?: CacheStats;
|
|
192
231
|
cleared?: number;
|
|
193
232
|
error?: string;
|
|
233
|
+
changes?: ChangeReport[];
|
|
234
|
+
}
|
|
235
|
+
export interface ChangeReport {
|
|
236
|
+
url: string;
|
|
237
|
+
changed: boolean;
|
|
238
|
+
previous_hash?: string;
|
|
239
|
+
current_hash?: string;
|
|
240
|
+
diff_summary?: string;
|
|
241
|
+
error?: string;
|
|
194
242
|
}
|
|
195
243
|
export interface ExtractInput {
|
|
196
244
|
url?: string;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEzD,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACxC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEzD,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAC5B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GACxD;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,CAAC;AAE3B,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACxC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,eAAe,CAAC,EAAE,OAAO,CAAC;KAC3B,CAAC;IACF,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,YAAY,EAAE,CAAC;IAChC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,YAAY,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC;AAEtG,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE5D,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,oBAAoB,EAAE,MAAM,CAAC;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,GAAG,YAAY,CAAC;IACnC,aAAa,EAAE,aAAa,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/C,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAAC;CAC7D;AAID,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC/C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACtE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;CACvE;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CAClF;AAID,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC;IAC7C,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,eAAe,EAAE,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,QAAQ,CAAC;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,EAAE,GAAG,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,QAAQ,CAAC;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
package/package.json
CHANGED
|
@@ -1,26 +1,44 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@staticn0va/wigolo",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Local-first web search MCP server for AI coding agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"import": "./dist/index.js",
|
|
10
|
+
"types": "./dist/index.d.ts"
|
|
11
|
+
},
|
|
12
|
+
"./types": {
|
|
13
|
+
"import": "./dist/types.js",
|
|
14
|
+
"types": "./dist/types.d.ts"
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
"types": "dist/types.d.ts",
|
|
7
18
|
"bin": {
|
|
8
19
|
"wigolo": "dist/index.js"
|
|
9
20
|
},
|
|
10
21
|
"files": [
|
|
11
22
|
"dist",
|
|
12
23
|
"README.md",
|
|
13
|
-
"LICENSE"
|
|
24
|
+
"LICENSE",
|
|
25
|
+
"SKILL.md"
|
|
14
26
|
],
|
|
15
27
|
"keywords": [
|
|
16
28
|
"mcp",
|
|
29
|
+
"mcp-server",
|
|
17
30
|
"web-search",
|
|
18
31
|
"web-scraping",
|
|
19
32
|
"ai-agent",
|
|
20
33
|
"playwright",
|
|
21
34
|
"searxng",
|
|
22
35
|
"local-first",
|
|
23
|
-
"content-extraction"
|
|
36
|
+
"content-extraction",
|
|
37
|
+
"search",
|
|
38
|
+
"scraping",
|
|
39
|
+
"crawler",
|
|
40
|
+
"claude",
|
|
41
|
+
"agent-tools"
|
|
24
42
|
],
|
|
25
43
|
"scripts": {
|
|
26
44
|
"build": "tsc",
|
|
@@ -41,6 +59,20 @@
|
|
|
41
59
|
"type": "git",
|
|
42
60
|
"url": "https://github.com/KnockOutEZ/wigolo"
|
|
43
61
|
},
|
|
62
|
+
"mcp": {
|
|
63
|
+
"transport": "stdio",
|
|
64
|
+
"command": "npx",
|
|
65
|
+
"args": [
|
|
66
|
+
"@staticn0va/wigolo"
|
|
67
|
+
],
|
|
68
|
+
"tools": [
|
|
69
|
+
"search",
|
|
70
|
+
"fetch",
|
|
71
|
+
"crawl",
|
|
72
|
+
"cache",
|
|
73
|
+
"extract"
|
|
74
|
+
]
|
|
75
|
+
},
|
|
44
76
|
"engines": {
|
|
45
77
|
"node": ">=20"
|
|
46
78
|
},
|
|
@@ -48,6 +80,7 @@
|
|
|
48
80
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
49
81
|
"@mozilla/readability": "^0.6.0",
|
|
50
82
|
"better-sqlite3": "^12.8.0",
|
|
83
|
+
"chalk": "^5.6.2",
|
|
51
84
|
"defuddle": "^0.16.0",
|
|
52
85
|
"linkedom": "^0.18.12",
|
|
53
86
|
"pdf-parse": "^2.4.5",
|