@loopstack/web-module 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constants.js +9 -12
- package/dist/constants.js.map +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -22
- package/dist/index.js.map +1 -1
- package/dist/services/index.d.ts +3 -3
- package/dist/services/index.d.ts.map +1 -1
- package/dist/services/index.js +3 -19
- package/dist/services/index.js.map +1 -1
- package/dist/services/webfetch-fetcher.service.d.ts +1 -1
- package/dist/services/webfetch-fetcher.service.d.ts.map +1 -1
- package/dist/services/webfetch-fetcher.service.js +29 -32
- package/dist/services/webfetch-fetcher.service.js.map +1 -1
- package/dist/services/webfetch-markdown.service.js +5 -41
- package/dist/services/webfetch-markdown.service.js.map +1 -1
- package/dist/services/webfetch-summarizer.service.js +14 -17
- package/dist/services/webfetch-summarizer.service.js.map +1 -1
- package/dist/tools/index.d.ts +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +1 -17
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/web-fetch.tool.d.ts +4 -6
- package/dist/tools/web-fetch.tool.d.ts.map +1 -1
- package/dist/tools/web-fetch.tool.js +27 -35
- package/dist/tools/web-fetch.tool.js.map +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -17
- package/dist/types/index.js.map +1 -1
- package/dist/types/webfetch.types.js +1 -2
- package/dist/utils/errors.js +5 -13
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/index.d.ts +4 -4
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +4 -20
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/preapproved-hosts.js +4 -9
- package/dist/utils/preapproved-hosts.js.map +1 -1
- package/dist/utils/secondary-model-prompt.js +1 -4
- package/dist/utils/secondary-model-prompt.js.map +1 -1
- package/dist/utils/url.utils.js +6 -11
- package/dist/utils/url.utils.js.map +1 -1
- package/dist/web.module.d.ts.map +1 -1
- package/dist/web.module.js +10 -14
- package/dist/web.module.js.map +1 -1
- package/package.json +9 -16
package/dist/constants.js
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
exports.CACHE_TTL_MS = 15 * 60 * 1000;
|
|
11
|
-
exports.MAX_CACHE_SIZE_BYTES = 50 * 1024 * 1024;
|
|
12
|
-
exports.DEFAULT_USER_AGENT = 'Loopstack-WebFetch/0.1 (+https://loopstack.ai)';
|
|
1
|
+
export const MAX_URL_LENGTH = 2000;
|
|
2
|
+
export const MAX_HTTP_CONTENT_LENGTH = 10 * 1024 * 1024;
|
|
3
|
+
export const FETCH_TIMEOUT_MS = 60_000;
|
|
4
|
+
export const MAX_REDIRECTS = 10;
|
|
5
|
+
export const MAX_MARKDOWN_LENGTH = 100_000;
|
|
6
|
+
export const MAX_RESULT_SIZE_CHARS = 100_000;
|
|
7
|
+
export const CACHE_TTL_MS = 15 * 60 * 1000;
|
|
8
|
+
export const MAX_CACHE_SIZE_BYTES = 50 * 1024 * 1024;
|
|
9
|
+
export const DEFAULT_USER_AGENT = 'Loopstack-WebFetch/0.1 (+https://loopstack.ai)';
|
|
13
10
|
//# sourceMappingURL=constants.js.map
|
package/dist/constants.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC;AAEnC,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAExD,MAAM,CAAC,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEvC,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEhC,MAAM,CAAC,MAAM,mBAAmB,GAAG,OAAO,CAAC;AAE3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,OAAO,CAAC;AAE7C,MAAM,CAAC,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAErD,MAAM,CAAC,MAAM,kBAAkB,GAAG,gDAAgD,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export * from './constants';
|
|
2
|
-
export * from './services';
|
|
3
|
-
export * from './tools';
|
|
4
|
-
export * from './types';
|
|
5
|
-
export * from './utils';
|
|
6
|
-
export * from './web.module';
|
|
1
|
+
export * from './constants.js';
|
|
2
|
+
export * from './services/index.js';
|
|
3
|
+
export * from './tools/index.js';
|
|
4
|
+
export * from './types/index.js';
|
|
5
|
+
export * from './utils/index.js';
|
|
6
|
+
export * from './web.module.js';
|
|
7
7
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,23 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./constants"), exports);
|
|
18
|
-
__exportStar(require("./services"), exports);
|
|
19
|
-
__exportStar(require("./tools"), exports);
|
|
20
|
-
__exportStar(require("./types"), exports);
|
|
21
|
-
__exportStar(require("./utils"), exports);
|
|
22
|
-
__exportStar(require("./web.module"), exports);
|
|
1
|
+
export * from './constants.js';
|
|
2
|
+
export * from './services/index.js';
|
|
3
|
+
export * from './tools/index.js';
|
|
4
|
+
export * from './types/index.js';
|
|
5
|
+
export * from './utils/index.js';
|
|
6
|
+
export * from './web.module.js';
|
|
23
7
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,iBAAiB,CAAC"}
|
package/dist/services/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export * from './webfetch-fetcher.service';
|
|
2
|
-
export * from './webfetch-markdown.service';
|
|
3
|
-
export * from './webfetch-summarizer.service';
|
|
1
|
+
export * from './webfetch-fetcher.service.js';
|
|
2
|
+
export * from './webfetch-markdown.service.js';
|
|
3
|
+
export * from './webfetch-summarizer.service.js';
|
|
4
4
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/services/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/services/index.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC"}
|
package/dist/services/index.js
CHANGED
|
@@ -1,20 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./webfetch-fetcher.service"), exports);
|
|
18
|
-
__exportStar(require("./webfetch-markdown.service"), exports);
|
|
19
|
-
__exportStar(require("./webfetch-summarizer.service"), exports);
|
|
1
|
+
export * from './webfetch-fetcher.service.js';
|
|
2
|
+
export * from './webfetch-markdown.service.js';
|
|
3
|
+
export * from './webfetch-summarizer.service.js';
|
|
20
4
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/services/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/services/index.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"webfetch-fetcher.service.d.ts","sourceRoot":"","sources":["../../src/services/webfetch-fetcher.service.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"webfetch-fetcher.service.d.ts","sourceRoot":"","sources":["../../src/services/webfetch-fetcher.service.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAajE,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,cAAc,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GAC7D;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAA;CAAE,CAAC;AAEjD,qBACa,sBAAsB;IACvB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAEpE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAItB;IAEH,UAAU,IAAI,IAAI;IAIZ,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAyBrE,cAAc,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;YAQhD,2BAA2B;YA0D3B,gBAAgB;IAsB9B,OAAO,CAAC,oBAAoB;CAe7B"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
2
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
3
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -8,28 +7,26 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
8
7
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
const url_utils_1 = require("../utils/url.utils");
|
|
18
|
-
const webfetch_markdown_service_1 = require("./webfetch-markdown.service");
|
|
10
|
+
import { Inject, Injectable } from '@nestjs/common';
|
|
11
|
+
import { LRUCache } from 'lru-cache';
|
|
12
|
+
import { CACHE_TTL_MS, DEFAULT_USER_AGENT, FETCH_TIMEOUT_MS, MAX_CACHE_SIZE_BYTES, MAX_HTTP_CONTENT_LENGTH, MAX_REDIRECTS, } from '../constants.js';
|
|
13
|
+
import { ContentTooLargeError, EgressBlockedError, FetchTimeoutError, InvalidUrlError, RedirectLimitExceededError, } from '../utils/errors.js';
|
|
14
|
+
import { isBinaryContentType, isPermittedRedirect, validateURL } from '../utils/url.utils.js';
|
|
15
|
+
import { WebFetchMarkdownService } from './webfetch-markdown.service.js';
|
|
19
16
|
let WebFetchFetcherService = class WebFetchFetcherService {
|
|
20
17
|
markdownService;
|
|
21
|
-
urlCache = new
|
|
22
|
-
maxSize:
|
|
23
|
-
ttl:
|
|
18
|
+
urlCache = new LRUCache({
|
|
19
|
+
maxSize: MAX_CACHE_SIZE_BYTES,
|
|
20
|
+
ttl: CACHE_TTL_MS,
|
|
24
21
|
sizeCalculation: (entry) => Math.max(1, Buffer.byteLength(entry.content, 'utf-8')),
|
|
25
22
|
});
|
|
26
23
|
clearCache() {
|
|
27
24
|
this.urlCache.clear();
|
|
28
25
|
}
|
|
29
26
|
async fetch(url, callerSignal) {
|
|
30
|
-
const validation =
|
|
27
|
+
const validation = validateURL(url);
|
|
31
28
|
if (!validation.ok) {
|
|
32
|
-
throw new
|
|
29
|
+
throw new InvalidUrlError(url, validation.reason);
|
|
33
30
|
}
|
|
34
31
|
const cached = this.urlCache.get(url);
|
|
35
32
|
if (cached) {
|
|
@@ -53,8 +50,8 @@ let WebFetchFetcherService = class WebFetchFetcherService {
|
|
|
53
50
|
return content.content;
|
|
54
51
|
}
|
|
55
52
|
async fetchWithPermittedRedirects(url, callerSignal, depth = 0) {
|
|
56
|
-
if (depth >
|
|
57
|
-
throw new
|
|
53
|
+
if (depth > MAX_REDIRECTS) {
|
|
54
|
+
throw new RedirectLimitExceededError(MAX_REDIRECTS);
|
|
58
55
|
}
|
|
59
56
|
const composed = composeSignal(callerSignal);
|
|
60
57
|
let response;
|
|
@@ -65,7 +62,7 @@ let WebFetchFetcherService = class WebFetchFetcherService {
|
|
|
65
62
|
signal: composed,
|
|
66
63
|
headers: {
|
|
67
64
|
Accept: 'text/markdown, text/html, */*',
|
|
68
|
-
'User-Agent':
|
|
65
|
+
'User-Agent': DEFAULT_USER_AGENT,
|
|
69
66
|
},
|
|
70
67
|
});
|
|
71
68
|
}
|
|
@@ -78,7 +75,7 @@ let WebFetchFetcherService = class WebFetchFetcherService {
|
|
|
78
75
|
throw new Error(`Redirect (${response.status}) missing Location header`);
|
|
79
76
|
}
|
|
80
77
|
const redirectUrl = new URL(location, url).toString();
|
|
81
|
-
if (
|
|
78
|
+
if (isPermittedRedirect(url, redirectUrl)) {
|
|
82
79
|
await drain(response);
|
|
83
80
|
return this.fetchWithPermittedRedirects(redirectUrl, callerSignal, depth + 1);
|
|
84
81
|
}
|
|
@@ -92,19 +89,19 @@ let WebFetchFetcherService = class WebFetchFetcherService {
|
|
|
92
89
|
}
|
|
93
90
|
if (response.status === 403 && response.headers.get('x-proxy-error') === 'blocked-by-allowlist') {
|
|
94
91
|
await drain(response);
|
|
95
|
-
throw new
|
|
92
|
+
throw new EgressBlockedError(new URL(url).hostname);
|
|
96
93
|
}
|
|
97
94
|
return response;
|
|
98
95
|
}
|
|
99
96
|
async toFetchedContent(response) {
|
|
100
97
|
const contentType = response.headers.get('content-type') ?? '';
|
|
101
98
|
const declaredLength = parseInt(response.headers.get('content-length') ?? '', 10);
|
|
102
|
-
if (Number.isFinite(declaredLength) && declaredLength >
|
|
99
|
+
if (Number.isFinite(declaredLength) && declaredLength > MAX_HTTP_CONTENT_LENGTH) {
|
|
103
100
|
await drain(response);
|
|
104
|
-
throw new
|
|
101
|
+
throw new ContentTooLargeError(MAX_HTTP_CONTENT_LENGTH);
|
|
105
102
|
}
|
|
106
|
-
const buffer = await readBodyWithCap(response,
|
|
107
|
-
const isBinary =
|
|
103
|
+
const buffer = await readBodyWithCap(response, MAX_HTTP_CONTENT_LENGTH);
|
|
104
|
+
const isBinary = isBinaryContentType(contentType);
|
|
108
105
|
return {
|
|
109
106
|
content: isBinary ? '' : buffer.toString('utf-8'),
|
|
110
107
|
bytes: buffer.length,
|
|
@@ -123,19 +120,19 @@ let WebFetchFetcherService = class WebFetchFetcherService {
|
|
|
123
120
|
if (callerSignal?.aborted)
|
|
124
121
|
return error;
|
|
125
122
|
if (composed.aborted) {
|
|
126
|
-
return new
|
|
123
|
+
return new FetchTimeoutError(FETCH_TIMEOUT_MS);
|
|
127
124
|
}
|
|
128
125
|
return error;
|
|
129
126
|
}
|
|
130
127
|
};
|
|
131
|
-
exports.WebFetchFetcherService = WebFetchFetcherService;
|
|
132
128
|
__decorate([
|
|
133
|
-
|
|
134
|
-
__metadata("design:type",
|
|
129
|
+
Inject(),
|
|
130
|
+
__metadata("design:type", WebFetchMarkdownService)
|
|
135
131
|
], WebFetchFetcherService.prototype, "markdownService", void 0);
|
|
136
|
-
|
|
137
|
-
|
|
132
|
+
WebFetchFetcherService = __decorate([
|
|
133
|
+
Injectable()
|
|
138
134
|
], WebFetchFetcherService);
|
|
135
|
+
export { WebFetchFetcherService };
|
|
139
136
|
function upgradeToHttps(parsed) {
|
|
140
137
|
if (parsed.protocol === 'http:') {
|
|
141
138
|
const copy = new URL(parsed.toString());
|
|
@@ -148,7 +145,7 @@ function isRedirectStatus(status) {
|
|
|
148
145
|
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
|
149
146
|
}
|
|
150
147
|
function composeSignal(callerSignal) {
|
|
151
|
-
const timeout = AbortSignal.timeout(
|
|
148
|
+
const timeout = AbortSignal.timeout(FETCH_TIMEOUT_MS);
|
|
152
149
|
if (!callerSignal)
|
|
153
150
|
return timeout;
|
|
154
151
|
const controller = new AbortController();
|
|
@@ -181,7 +178,7 @@ async function readBodyWithCap(response, cap) {
|
|
|
181
178
|
if (!response.body) {
|
|
182
179
|
const buf = Buffer.from(await response.arrayBuffer());
|
|
183
180
|
if (buf.length > cap)
|
|
184
|
-
throw new
|
|
181
|
+
throw new ContentTooLargeError(cap);
|
|
185
182
|
return buf;
|
|
186
183
|
}
|
|
187
184
|
const reader = response.body.getReader();
|
|
@@ -201,7 +198,7 @@ async function readBodyWithCap(response, cap) {
|
|
|
201
198
|
}
|
|
202
199
|
catch {
|
|
203
200
|
}
|
|
204
|
-
throw new
|
|
201
|
+
throw new ContentTooLargeError(cap);
|
|
205
202
|
}
|
|
206
203
|
chunks.push(Buffer.from(value));
|
|
207
204
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"webfetch-fetcher.service.js","sourceRoot":"","sources":["../../src/services/webfetch-fetcher.service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"webfetch-fetcher.service.js","sourceRoot":"","sources":["../../src/services/webfetch-fetcher.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,gBAAgB,EAChB,oBAAoB,EACpB,uBAAuB,EACvB,aAAa,GACd,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAC9F,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AASlE,IAAM,sBAAsB,GAA5B,MAAM,sBAAsB;IACN,eAAe,CAA0B;IAEnD,QAAQ,GAAG,IAAI,QAAQ,CAAqB;QAC3D,OAAO,EAAE,oBAAoB;QAC7B,GAAG,EAAE,YAAY;QACjB,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;KACnF,CAAC,CAAC;IAEH,UAAU;QACR,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,YAA0B;QACjD,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,IAAI,eAAe,CAAC,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC;QAID,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,GAAG,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,WAAW,GAAG,cAAc,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QAEnF,IAAI,MAAM,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACvD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAClD,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACvC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,OAAuB;QAC1C,IAAI,OAAO,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAChC,IAAI,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5D,OAAO,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,GAAW,EACX,YAAqC,EACrC,KAAK,GAAG,CAAC;QAET,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC1B,MAAM,IAAI,0BAA0B,CAAC,aAAa,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;QAE7C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC1B,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE;oBACP,MAAM,EAAE,+BAA+B;oBACvC,YAAY,EAAE,kBAAkB;iBACjC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,CAAC,MAAM,2BAA2B,CAAC,CAAC;YAC3E,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;YAEtD,IAAI,mBAAmB,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,CAAC;gBAG1C,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACtB,OAAO,IAAI,CAAC,2BAA2B,CAAC,WAAW,EAAE,YAAY,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAChF,CAAC;YAED,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,WAAW,EAAE,GAAG;gBAChB,WAAW;gBACX,UAAU,EAAE,QAAQ,CAAC,MAAM;aAC5B,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,sBAAsB,EAAE,CAAC;YAChG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,MAAM,IAAI,kBAAkB,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,QAAkB;QAC/C,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAE/D,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAClF,IAAI,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,uBAAuB,EAAE,CAAC;YAChF,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAElD,OAAO;YACL,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;YACjD,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,IAAI,EAAE,QAAQ,CAAC,MAAM;YACrB,QAAQ,EAAE,QAAQ,CAAC,UAAU;YAC7B,WAAW;YACX,QAAQ;SACT,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,KAAc,EAAE,QAAqB,EAAE,YAA0B;QAC5F,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAE5C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC;QAC7E,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAG3B,IAAI,YAAY,EAAE,OAAO;YAAE,OAAO,KAAK,CAAC;QAExC,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACrB,OAAO,IAAI,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA5I4B;IAA1B,MAAM,EAAE;8BAAmC,uBAAuB;+DAAC;AADzD,sBAAsB;IADlC,UAAU,EAAE;GACA,sBAAsB,CA6IlC;;AAED,SAAS,cAAc,CAAC,MAAW;IACjC,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAc;IACtC,OAAO,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;AAChG,CAAC;AAED,SAAS,aAAa,CAAC,YAA0B;IAC/C,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACtD,IAAI,CAAC,YAAY;QAAE,OAAO,OAAO,CAAC;IAElC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,CAAC,MAAmB,EAAQ,EAAE;QAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO;YAAE,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAClE,CAAC,CAAC;IAEF,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,YAAY,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,SAAS,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,UAAU,CAAC,MAAM,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,QAAkB;IACrC,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;IAET,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAkB,EAAE,GAAW;IAC5D,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACtD,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG;YAAE,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAC1B,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC;oBACH,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxB,CAAC;gBAAC,MAAM,CAAC;gBAET,CAAC;gBACD,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,CAAC;YACtC,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YACH,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;QAET,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -1,50 +1,14 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
1
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
19
2
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
20
3
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
21
4
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
22
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
23
6
|
};
|
|
24
|
-
|
|
25
|
-
var ownKeys = function(o) {
|
|
26
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
27
|
-
var ar = [];
|
|
28
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
29
|
-
return ar;
|
|
30
|
-
};
|
|
31
|
-
return ownKeys(o);
|
|
32
|
-
};
|
|
33
|
-
return function (mod) {
|
|
34
|
-
if (mod && mod.__esModule) return mod;
|
|
35
|
-
var result = {};
|
|
36
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
37
|
-
__setModuleDefault(result, mod);
|
|
38
|
-
return result;
|
|
39
|
-
};
|
|
40
|
-
})();
|
|
41
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
-
exports.WebFetchMarkdownService = void 0;
|
|
43
|
-
const common_1 = require("@nestjs/common");
|
|
7
|
+
import { Injectable } from '@nestjs/common';
|
|
44
8
|
let WebFetchMarkdownService = class WebFetchMarkdownService {
|
|
45
9
|
instancePromise;
|
|
46
10
|
getService() {
|
|
47
|
-
return (this.instancePromise ??=
|
|
11
|
+
return (this.instancePromise ??= import('turndown').then((m) => {
|
|
48
12
|
const Turndown = m.default;
|
|
49
13
|
const td = new Turndown({
|
|
50
14
|
headingStyle: 'atx',
|
|
@@ -59,8 +23,8 @@ let WebFetchMarkdownService = class WebFetchMarkdownService {
|
|
|
59
23
|
return service.turndown(html);
|
|
60
24
|
}
|
|
61
25
|
};
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
(0, common_1.Injectable)()
|
|
26
|
+
WebFetchMarkdownService = __decorate([
|
|
27
|
+
Injectable()
|
|
65
28
|
], WebFetchMarkdownService);
|
|
29
|
+
export { WebFetchMarkdownService };
|
|
66
30
|
//# sourceMappingURL=webfetch-markdown.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"webfetch-markdown.service.js","sourceRoot":"","sources":["../../src/services/webfetch-markdown.service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"webfetch-markdown.service.js","sourceRoot":"","sources":["../../src/services/webfetch-markdown.service.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMrC,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;IAC1B,eAAe,CAA6B;IAE5C,UAAU;QAChB,OAAO,CAAC,IAAI,CAAC,eAAe,KAAK,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YAG7D,MAAM,QAAQ,GAAI,CAA0C,CAAC,OAAO,CAAC;YACrE,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC;gBACtB,YAAY,EAAE,KAAK;gBACnB,cAAc,EAAE,QAAQ;aACzB,CAAC,CAAC;YACH,EAAE,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;YACrD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;CACF,CAAA;AArBY,uBAAuB;IADnC,UAAU,EAAE;GACA,uBAAuB,CAqBnC"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
2
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
3
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -9,23 +8,21 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
9
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
9
|
};
|
|
11
10
|
var WebFetchSummarizerService_1;
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
const preapproved_hosts_1 = require("../utils/preapproved-hosts");
|
|
18
|
-
const secondary_model_prompt_1 = require("../utils/secondary-model-prompt");
|
|
11
|
+
import { Inject, Injectable, Logger } from '@nestjs/common';
|
|
12
|
+
import { ClaudeClientService } from '@loopstack/claude-module';
|
|
13
|
+
import { MAX_MARKDOWN_LENGTH } from '../constants.js';
|
|
14
|
+
import { isPreapprovedUrl } from '../utils/preapproved-hosts.js';
|
|
15
|
+
import { makeSecondaryModelPrompt } from '../utils/secondary-model-prompt.js';
|
|
19
16
|
const DEFAULT_SUMMARIZER_MODEL = 'claude-haiku-4-5-20251001';
|
|
20
17
|
let WebFetchSummarizerService = WebFetchSummarizerService_1 = class WebFetchSummarizerService {
|
|
21
|
-
logger = new
|
|
18
|
+
logger = new Logger(WebFetchSummarizerService_1.name);
|
|
22
19
|
claudeClientService;
|
|
23
20
|
async summarize(url, markdown, userPrompt, options) {
|
|
24
|
-
const truncated = markdown.length >
|
|
21
|
+
const truncated = markdown.length > MAX_MARKDOWN_LENGTH;
|
|
25
22
|
const content = truncated
|
|
26
|
-
? markdown.slice(0,
|
|
23
|
+
? markdown.slice(0, MAX_MARKDOWN_LENGTH) + '\n\n[Content truncated due to length...]'
|
|
27
24
|
: markdown;
|
|
28
|
-
const finalPrompt =
|
|
25
|
+
const finalPrompt = makeSecondaryModelPrompt(content, userPrompt, isPreapprovedUrl(url));
|
|
29
26
|
const client = this.claudeClientService.getClient({
|
|
30
27
|
envApiKey: options?.envApiKey,
|
|
31
28
|
});
|
|
@@ -46,12 +43,12 @@ let WebFetchSummarizerService = WebFetchSummarizerService_1 = class WebFetchSumm
|
|
|
46
43
|
return { summary: '', truncated };
|
|
47
44
|
}
|
|
48
45
|
};
|
|
49
|
-
exports.WebFetchSummarizerService = WebFetchSummarizerService;
|
|
50
46
|
__decorate([
|
|
51
|
-
|
|
52
|
-
__metadata("design:type",
|
|
47
|
+
Inject(),
|
|
48
|
+
__metadata("design:type", ClaudeClientService)
|
|
53
49
|
], WebFetchSummarizerService.prototype, "claudeClientService", void 0);
|
|
54
|
-
|
|
55
|
-
|
|
50
|
+
WebFetchSummarizerService = WebFetchSummarizerService_1 = __decorate([
|
|
51
|
+
Injectable()
|
|
56
52
|
], WebFetchSummarizerService);
|
|
53
|
+
export { WebFetchSummarizerService };
|
|
57
54
|
//# sourceMappingURL=webfetch-summarizer.service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"webfetch-summarizer.service.js","sourceRoot":"","sources":["../../src/services/webfetch-summarizer.service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"webfetch-summarizer.service.js","sourceRoot":"","sources":["../../src/services/webfetch-summarizer.service.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,EAAE,wBAAwB,EAAE,MAAM,oCAAoC,CAAC;AAS9E,MAAM,wBAAwB,GAAG,2BAA2B,CAAC;AAGtD,IAAM,yBAAyB,iCAA/B,MAAM,yBAAyB;IACnB,MAAM,GAAG,IAAI,MAAM,CAAC,2BAAyB,CAAC,IAAI,CAAC,CAAC;IAE1C,mBAAmB,CAAsB;IASpE,KAAK,CAAC,SAAS,CACb,GAAW,EACX,QAAgB,EAChB,UAAkB,EAClB,OAA0B;QAE1B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,GAAG,mBAAmB,CAAC;QACxD,MAAM,OAAO,GAAG,SAAS;YACvB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,GAAG,0CAA0C;YACrF,CAAC,CAAC,QAAQ,CAAC;QAEb,MAAM,WAAW,GAAG,wBAAwB,CAAC,OAAO,EAAE,UAAU,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC;YAChD,SAAS,EAAE,OAAO,EAAE,SAAS;SAC9B,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,IAAI,wBAAwB,CAAC;QAElG,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,KAAK;YACL,UAAU,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI;YACtC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;SACnD,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,KAAK,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;YAC7B,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QACtD,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC;IACpC,CAAC;CACF,CAAA;AA7C4B;IAA1B,MAAM,EAAE;8BAAuC,mBAAmB;sEAAC;AAHzD,yBAAyB;IADrC,UAAU,EAAE;GACA,yBAAyB,CAgDrC"}
|
package/dist/tools/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export * from './web-fetch.tool';
|
|
1
|
+
export * from './web-fetch.tool.js';
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC"}
|
package/dist/tools/index.js
CHANGED
|
@@ -1,18 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./web-fetch.tool"), exports);
|
|
1
|
+
export * from './web-fetch.tool.js';
|
|
18
2
|
//# sourceMappingURL=index.js.map
|
package/dist/tools/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC"}
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { BaseTool, ToolResult } from '@loopstack/common';
|
|
3
|
-
import { WebFetchResult } from '../types';
|
|
3
|
+
import { WebFetchResult } from '../types/index.js';
|
|
4
4
|
export declare const WebFetchSchema: z.ZodObject<{
|
|
5
5
|
url: z.ZodURL;
|
|
6
6
|
prompt: z.ZodOptional<z.ZodString>;
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
11
|
-
}, z.core.$strip>>;
|
|
7
|
+
model: z.ZodOptional<z.ZodString>;
|
|
8
|
+
envApiKey: z.ZodOptional<z.ZodString>;
|
|
9
|
+
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
12
10
|
}, z.core.$strict>;
|
|
13
11
|
type WebFetchArgs = z.infer<typeof WebFetchSchema>;
|
|
14
12
|
export declare class WebFetchTool extends BaseTool {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web-fetch.tool.d.ts","sourceRoot":"","sources":["../../src/tools/web-fetch.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAQ,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,EAAE,cAAc,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"web-fetch.tool.d.ts","sourceRoot":"","sources":["../../src/tools/web-fetch.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAQ,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,eAAO,MAAM,cAAc;;;;;;kBAchB,CAAC;AAEZ,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEnD,qBAQa,YAAa,SAAQ,QAAQ;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyB;IACjD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA4B;IAE3D,IAAI,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;CAiFpE"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
2
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
3
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -8,32 +7,25 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
8
7
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
const services_1 = require("../services");
|
|
18
|
-
exports.WebFetchSchema = zod_1.z
|
|
10
|
+
import { Inject } from '@nestjs/common';
|
|
11
|
+
import { z } from 'zod';
|
|
12
|
+
import { BaseTool, Tool } from '@loopstack/common';
|
|
13
|
+
import { MAX_MARKDOWN_LENGTH, MAX_RESULT_SIZE_CHARS } from '../constants.js';
|
|
14
|
+
import { WebFetchFetcherService, WebFetchSummarizerService } from '../services/index.js';
|
|
15
|
+
export const WebFetchSchema = z
|
|
19
16
|
.object({
|
|
20
|
-
url:
|
|
21
|
-
prompt:
|
|
17
|
+
url: z.url().describe('The URL to fetch content from'),
|
|
18
|
+
prompt: z
|
|
22
19
|
.string()
|
|
23
20
|
.optional()
|
|
24
21
|
.describe('Optional instruction applied to the fetched content by a small model. ' +
|
|
25
22
|
'When omitted, the raw Markdown is returned (truncated if very long).'),
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
envApiKey: zod_1.z.string().optional(),
|
|
30
|
-
maxTokens: zod_1.z.number().optional(),
|
|
31
|
-
})
|
|
32
|
-
.optional()
|
|
33
|
-
.describe('Model configuration for the summarization step. Only used when `prompt` is provided.'),
|
|
23
|
+
model: z.string().optional().describe('Model for the summarization step. Only used when `prompt` is provided.'),
|
|
24
|
+
envApiKey: z.string().optional(),
|
|
25
|
+
maxTokens: z.number().optional(),
|
|
34
26
|
})
|
|
35
27
|
.strict();
|
|
36
|
-
let WebFetchTool = class WebFetchTool extends
|
|
28
|
+
let WebFetchTool = class WebFetchTool extends BaseTool {
|
|
37
29
|
fetcher;
|
|
38
30
|
summarizer;
|
|
39
31
|
async call(args) {
|
|
@@ -79,16 +71,16 @@ let WebFetchTool = class WebFetchTool extends common_2.BaseTool {
|
|
|
79
71
|
let truncated = false;
|
|
80
72
|
if (args.prompt) {
|
|
81
73
|
const summary = await this.summarizer.summarize(args.url, markdown, args.prompt, {
|
|
82
|
-
model: args.
|
|
83
|
-
envApiKey: args.
|
|
84
|
-
maxTokens: args.
|
|
74
|
+
model: args.model,
|
|
75
|
+
envApiKey: args.envApiKey,
|
|
76
|
+
maxTokens: args.maxTokens,
|
|
85
77
|
});
|
|
86
78
|
result = summary.summary;
|
|
87
79
|
truncated = summary.truncated;
|
|
88
80
|
}
|
|
89
81
|
else {
|
|
90
|
-
if (markdown.length >
|
|
91
|
-
result = markdown.slice(0,
|
|
82
|
+
if (markdown.length > MAX_MARKDOWN_LENGTH) {
|
|
83
|
+
result = markdown.slice(0, MAX_MARKDOWN_LENGTH) + '\n\n[Content truncated due to length...]';
|
|
92
84
|
truncated = true;
|
|
93
85
|
}
|
|
94
86
|
else {
|
|
@@ -111,24 +103,24 @@ let WebFetchTool = class WebFetchTool extends common_2.BaseTool {
|
|
|
111
103
|
};
|
|
112
104
|
}
|
|
113
105
|
};
|
|
114
|
-
exports.WebFetchTool = WebFetchTool;
|
|
115
106
|
__decorate([
|
|
116
|
-
|
|
117
|
-
__metadata("design:type",
|
|
107
|
+
Inject(),
|
|
108
|
+
__metadata("design:type", WebFetchFetcherService)
|
|
118
109
|
], WebFetchTool.prototype, "fetcher", void 0);
|
|
119
110
|
__decorate([
|
|
120
|
-
|
|
121
|
-
__metadata("design:type",
|
|
111
|
+
Inject(),
|
|
112
|
+
__metadata("design:type", WebFetchSummarizerService)
|
|
122
113
|
], WebFetchTool.prototype, "summarizer", void 0);
|
|
123
|
-
|
|
124
|
-
|
|
114
|
+
WebFetchTool = __decorate([
|
|
115
|
+
Tool({
|
|
125
116
|
uiConfig: {
|
|
126
117
|
description: 'Fetches content from a URL, converts HTML to Markdown, and optionally summarizes it with a small Claude model against a user-provided prompt. ' +
|
|
127
118
|
'Supports HTTPS upgrade, same-origin redirect following with cross-host report, a 15-minute in-memory cache, size and redirect caps, and a preapproved-host allowlist.',
|
|
128
119
|
},
|
|
129
|
-
schema:
|
|
120
|
+
schema: WebFetchSchema,
|
|
130
121
|
})
|
|
131
122
|
], WebFetchTool);
|
|
123
|
+
export { WebFetchTool };
|
|
132
124
|
function buildRedirectMessage(originalUrl, redirectUrl, statusCode, prompt) {
|
|
133
125
|
const promptLine = prompt ? `- prompt: "${prompt}"\n` : '';
|
|
134
126
|
return (`REDIRECT DETECTED: The URL redirects to a different host.\n\n` +
|
|
@@ -156,8 +148,8 @@ function redirectStatusText(status) {
|
|
|
156
148
|
}
|
|
157
149
|
}
|
|
158
150
|
function capResultSize(text) {
|
|
159
|
-
if (text.length <=
|
|
151
|
+
if (text.length <= MAX_RESULT_SIZE_CHARS)
|
|
160
152
|
return text;
|
|
161
|
-
return text.slice(0,
|
|
153
|
+
return text.slice(0, MAX_RESULT_SIZE_CHARS) + `\n\n[Result truncated at ${MAX_RESULT_SIZE_CHARS} characters]`;
|
|
162
154
|
}
|
|
163
155
|
//# sourceMappingURL=web-fetch.tool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web-fetch.tool.js","sourceRoot":"","sources":["../../src/tools/web-fetch.tool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"web-fetch.tool.js","sourceRoot":"","sources":["../../src/tools/web-fetch.tool.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAc,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AAGzF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC;KAC5B,MAAM,CAAC;IACN,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IACtD,MAAM,EAAE,CAAC;SACN,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,wEAAwE;QACtE,sEAAsE,CACzE;IACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wEAAwE,CAAC;IAC/G,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC;KACD,MAAM,EAAE,CAAC;AAYL,IAAM,YAAY,GAAlB,MAAM,YAAa,SAAQ,QAAQ;IACb,OAAO,CAAyB;IAChC,UAAU,CAA4B;IAEjE,KAAK,CAAC,IAAI,CAAC,IAAkB;QAC3B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEnD,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAChC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;YAClE,MAAM,OAAO,GAAG,oBAAoB,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAExF,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;oBACjC,IAAI,EAAE,UAAU;oBAChB,QAAQ,EAAE,kBAAkB,CAAC,UAAU,CAAC;oBACxC,WAAW,EAAE,YAAY;oBACzB,MAAM,EAAE,OAAO;oBACf,SAAS,EAAE,KAAK;oBAChB,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK;oBACrC,QAAQ,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE;iBACnD;aACF,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAEpC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,MAAM,IAAI,GAAG,6BAA6B,OAAO,CAAC,WAAW,KAAK,OAAO,CAAC,KAAK,wCAAwC,CAAC;YACxH,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,MAAM,EAAE,IAAI;oBACZ,SAAS,EAAE,KAAK;oBAChB,MAAM;oBACN,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK;iBACtC;aACF,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAE5D,IAAI,MAAc,CAAC;QACnB,IAAI,SAAS,GAAG,KAAK,CAAC;QAEtB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE;gBAC/E,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC;YACH,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;YACzB,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,IAAI,QAAQ,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;gBAC1C,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,GAAG,0CAA0C,CAAC;gBAC7F,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,QAAQ,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE;gBACJ,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,MAAM;gBACN,SAAS;gBACT,MAAM;gBACN,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAA;AApF4B;IAA1B,MAAM,EAAE;8BAA2B,sBAAsB;6CAAC;AAChC;IAA1B,MAAM,EAAE;8BAA8B,yBAAyB;gDAAC;AAFtD,YAAY;IARxB,IAAI,CAAC;QACJ,QAAQ,EAAE;YACR,WAAW,EACT,gJAAgJ;gBAChJ,uKAAuK;SAC1K;QACD,MAAM,EAAE,cAAc;KACvB,CAAC;GACW,YAAY,CAqFxB;;AAED,SAAS,oBAAoB,CAC3B,WAAmB,EACnB,WAAmB,EACnB,UAAkB,EAClB,MAA0B;IAE1B,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,MAAM,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3D,OAAO,CACL,+DAA+D;QAC/D,iBAAiB,WAAW,IAAI;QAChC,iBAAiB,WAAW,IAAI;QAChC,WAAW,UAAU,IAAI,kBAAkB,CAAC,UAAU,CAAC,MAAM;QAC7D,wEAAwE;QACxE,WAAW,WAAW,KAAK;QAC3B,UAAU,CACX,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAc;IACxC,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,GAAG;YACN,OAAO,mBAAmB,CAAC;QAC7B,KAAK,GAAG;YACN,OAAO,OAAO,CAAC;QACjB,KAAK,GAAG;YACN,OAAO,WAAW,CAAC;QACrB,KAAK,GAAG;YACN,OAAO,oBAAoB,CAAC;QAC9B,KAAK,GAAG;YACN,OAAO,oBAAoB,CAAC;QAC9B;YACE,OAAO,UAAU,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,IAAI,CAAC,MAAM,IAAI,qBAAqB;QAAE,OAAO,IAAI,CAAC;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,GAAG,4BAA4B,qBAAqB,cAAc,CAAC;AAChH,CAAC"}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export * from './webfetch.types';
|
|
1
|
+
export * from './webfetch.types.js';
|
|
2
2
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC"}
|
package/dist/types/index.js
CHANGED
|
@@ -1,18 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./webfetch.types"), exports);
|
|
1
|
+
export * from './webfetch.types.js';
|
|
18
2
|
//# sourceMappingURL=index.js.map
|
package/dist/types/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC"}
|
package/dist/utils/errors.js
CHANGED
|
@@ -1,14 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.FetchTimeoutError = exports.EgressBlockedError = exports.RedirectLimitExceededError = exports.ContentTooLargeError = exports.InvalidUrlError = void 0;
|
|
4
|
-
class InvalidUrlError extends Error {
|
|
1
|
+
export class InvalidUrlError extends Error {
|
|
5
2
|
constructor(url, reason) {
|
|
6
3
|
super(`Invalid URL "${url}": ${reason}`);
|
|
7
4
|
this.name = 'InvalidUrlError';
|
|
8
5
|
}
|
|
9
6
|
}
|
|
10
|
-
|
|
11
|
-
class ContentTooLargeError extends Error {
|
|
7
|
+
export class ContentTooLargeError extends Error {
|
|
12
8
|
limitBytes;
|
|
13
9
|
constructor(limitBytes) {
|
|
14
10
|
super(`Response body exceeded ${limitBytes} bytes and was aborted.`);
|
|
@@ -16,15 +12,13 @@ class ContentTooLargeError extends Error {
|
|
|
16
12
|
this.name = 'ContentTooLargeError';
|
|
17
13
|
}
|
|
18
14
|
}
|
|
19
|
-
|
|
20
|
-
class RedirectLimitExceededError extends Error {
|
|
15
|
+
export class RedirectLimitExceededError extends Error {
|
|
21
16
|
constructor(limit) {
|
|
22
17
|
super(`Too many redirects (exceeded ${limit}).`);
|
|
23
18
|
this.name = 'RedirectLimitExceededError';
|
|
24
19
|
}
|
|
25
20
|
}
|
|
26
|
-
|
|
27
|
-
class EgressBlockedError extends Error {
|
|
21
|
+
export class EgressBlockedError extends Error {
|
|
28
22
|
domain;
|
|
29
23
|
constructor(domain) {
|
|
30
24
|
super(`Access to ${domain} is blocked by the network egress proxy.`);
|
|
@@ -32,12 +26,10 @@ class EgressBlockedError extends Error {
|
|
|
32
26
|
this.name = 'EgressBlockedError';
|
|
33
27
|
}
|
|
34
28
|
}
|
|
35
|
-
|
|
36
|
-
class FetchTimeoutError extends Error {
|
|
29
|
+
export class FetchTimeoutError extends Error {
|
|
37
30
|
constructor(timeoutMs) {
|
|
38
31
|
super(`Fetch timed out after ${timeoutMs}ms.`);
|
|
39
32
|
this.name = 'FetchTimeoutError';
|
|
40
33
|
}
|
|
41
34
|
}
|
|
42
|
-
exports.FetchTimeoutError = FetchTimeoutError;
|
|
43
35
|
//# sourceMappingURL=errors.js.map
|
package/dist/utils/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,YAAY,GAAW,EAAE,MAAc;QACrC,KAAK,CAAC,gBAAgB,GAAG,MAAM,MAAM,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACjB;IAA5B,YAA4B,UAAkB;QAC5C,KAAK,CAAC,0BAA0B,UAAU,yBAAyB,CAAC,CAAC;QAD3C,eAAU,GAAV,UAAU,CAAQ;QAE5C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,MAAM,OAAO,0BAA2B,SAAQ,KAAK;IACnD,YAAY,KAAa;QACvB,KAAK,CAAC,gCAAgC,KAAK,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,IAAI,GAAG,4BAA4B,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IACf;IAA5B,YAA4B,MAAc;QACxC,KAAK,CAAC,aAAa,MAAM,0CAA0C,CAAC,CAAC;QAD3C,WAAM,GAAN,MAAM,CAAQ;QAExC,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAED,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,SAAiB;QAC3B,KAAK,CAAC,yBAAyB,SAAS,KAAK,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF"}
|
package/dist/utils/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export * from './errors';
|
|
2
|
-
export * from './preapproved-hosts';
|
|
3
|
-
export * from './secondary-model-prompt';
|
|
4
|
-
export * from './url.utils';
|
|
1
|
+
export * from './errors.js';
|
|
2
|
+
export * from './preapproved-hosts.js';
|
|
3
|
+
export * from './secondary-model-prompt.js';
|
|
4
|
+
export * from './url.utils.js';
|
|
5
5
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gBAAgB,CAAC"}
|
package/dist/utils/index.js
CHANGED
|
@@ -1,21 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./errors"), exports);
|
|
18
|
-
__exportStar(require("./preapproved-hosts"), exports);
|
|
19
|
-
__exportStar(require("./secondary-model-prompt"), exports);
|
|
20
|
-
__exportStar(require("./url.utils"), exports);
|
|
1
|
+
export * from './errors.js';
|
|
2
|
+
export * from './preapproved-hosts.js';
|
|
3
|
+
export * from './secondary-model-prompt.js';
|
|
4
|
+
export * from './url.utils.js';
|
|
21
5
|
//# sourceMappingURL=index.js.map
|
package/dist/utils/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gBAAgB,CAAC"}
|
|
@@ -1,9 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.PREAPPROVED_HOSTS = void 0;
|
|
4
|
-
exports.isPreapprovedHost = isPreapprovedHost;
|
|
5
|
-
exports.isPreapprovedUrl = isPreapprovedUrl;
|
|
6
|
-
exports.PREAPPROVED_HOSTS = new Set([
|
|
1
|
+
export const PREAPPROVED_HOSTS = new Set([
|
|
7
2
|
'www.typescriptlang.org',
|
|
8
3
|
'nodejs.org',
|
|
9
4
|
'expressjs.com',
|
|
@@ -22,7 +17,7 @@ exports.PREAPPROVED_HOSTS = new Set([
|
|
|
22
17
|
const { HOSTNAME_ONLY, PATH_PREFIXES } = (() => {
|
|
23
18
|
const hosts = new Set();
|
|
24
19
|
const paths = new Map();
|
|
25
|
-
|
|
20
|
+
PREAPPROVED_HOSTS.forEach((entry) => {
|
|
26
21
|
const slash = entry.indexOf('/');
|
|
27
22
|
if (slash === -1) {
|
|
28
23
|
hosts.add(entry);
|
|
@@ -39,7 +34,7 @@ const { HOSTNAME_ONLY, PATH_PREFIXES } = (() => {
|
|
|
39
34
|
});
|
|
40
35
|
return { HOSTNAME_ONLY: hosts, PATH_PREFIXES: paths };
|
|
41
36
|
})();
|
|
42
|
-
function isPreapprovedHost(hostname, pathname) {
|
|
37
|
+
export function isPreapprovedHost(hostname, pathname) {
|
|
43
38
|
if (HOSTNAME_ONLY.has(hostname))
|
|
44
39
|
return true;
|
|
45
40
|
const prefixes = PATH_PREFIXES.get(hostname);
|
|
@@ -51,7 +46,7 @@ function isPreapprovedHost(hostname, pathname) {
|
|
|
51
46
|
}
|
|
52
47
|
return false;
|
|
53
48
|
}
|
|
54
|
-
function isPreapprovedUrl(url) {
|
|
49
|
+
export function isPreapprovedUrl(url) {
|
|
55
50
|
try {
|
|
56
51
|
const parsed = new URL(url);
|
|
57
52
|
return isPreapprovedHost(parsed.hostname, parsed.pathname);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"preapproved-hosts.js","sourceRoot":"","sources":["../../src/utils/preapproved-hosts.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"preapproved-hosts.js","sourceRoot":"","sources":["../../src/utils/preapproved-hosts.ts"],"names":[],"mappings":"AAQA,MAAM,CAAC,MAAM,iBAAiB,GAAwB,IAAI,GAAG,CAAC;IAC5D,wBAAwB;IACxB,YAAY;IACZ,eAAe;IACf,WAAW;IACX,iBAAiB;IACjB,UAAU;IACV,oBAAoB;IACpB,eAAe;IACf,gBAAgB;IAChB,aAAa;IACb,WAAW;IACX,iBAAiB;IACjB,YAAY;IACZ,cAAc;CACf,CAAC,CAAC;AAEH,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,CAAC,GAAG,EAAE;IAC7C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,iBAAiB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAClC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACnC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,QAAQ;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;gBAC7B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AACxD,CAAC,CAAC,EAAE,CAAC;AAEL,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,QAAgB;IAClE,IAAI,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7C,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,IAAI,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,iBAAiB,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.makeSecondaryModelPrompt = makeSecondaryModelPrompt;
|
|
4
|
-
function makeSecondaryModelPrompt(markdownContent, prompt, isPreapprovedDomain) {
|
|
1
|
+
export function makeSecondaryModelPrompt(markdownContent, prompt, isPreapprovedDomain) {
|
|
5
2
|
const guidelines = isPreapprovedDomain
|
|
6
3
|
? `Provide a concise response based on the content above. Include relevant details, code examples, and documentation excerpts as needed.`
|
|
7
4
|
: `Provide a concise response based only on the content above. In your response:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"secondary-model-prompt.js","sourceRoot":"","sources":["../../src/utils/secondary-model-prompt.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"secondary-model-prompt.js","sourceRoot":"","sources":["../../src/utils/secondary-model-prompt.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,wBAAwB,CACtC,eAAuB,EACvB,MAAc,EACd,mBAA4B;IAE5B,MAAM,UAAU,GAAG,mBAAmB;QACpC,CAAC,CAAC,uIAAuI;QACzI,CAAC,CAAC;;;;iDAI2C,CAAC;IAEhD,OAAO;;;EAGP,eAAe;;;EAGf,MAAM;;EAEN,UAAU;CACX,CAAC;AACF,CAAC"}
|
package/dist/utils/url.utils.js
CHANGED
|
@@ -1,12 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
exports.isBinaryContentType = isBinaryContentType;
|
|
6
|
-
const constants_1 = require("../constants");
|
|
7
|
-
function validateURL(url) {
|
|
8
|
-
if (url.length > constants_1.MAX_URL_LENGTH) {
|
|
9
|
-
return { ok: false, reason: `URL exceeds ${constants_1.MAX_URL_LENGTH} characters` };
|
|
1
|
+
import { MAX_URL_LENGTH } from '../constants.js';
|
|
2
|
+
export function validateURL(url) {
|
|
3
|
+
if (url.length > MAX_URL_LENGTH) {
|
|
4
|
+
return { ok: false, reason: `URL exceeds ${MAX_URL_LENGTH} characters` };
|
|
10
5
|
}
|
|
11
6
|
let parsed;
|
|
12
7
|
try {
|
|
@@ -27,7 +22,7 @@ function validateURL(url) {
|
|
|
27
22
|
}
|
|
28
23
|
return { ok: true, parsed };
|
|
29
24
|
}
|
|
30
|
-
function isPermittedRedirect(originalUrl, redirectUrl) {
|
|
25
|
+
export function isPermittedRedirect(originalUrl, redirectUrl) {
|
|
31
26
|
try {
|
|
32
27
|
const original = new URL(originalUrl);
|
|
33
28
|
const redirect = new URL(redirectUrl);
|
|
@@ -45,7 +40,7 @@ function isPermittedRedirect(originalUrl, redirectUrl) {
|
|
|
45
40
|
}
|
|
46
41
|
}
|
|
47
42
|
const BINARY_CONTENT_TYPE_PREFIXES = ['application/pdf', 'image/', 'audio/', 'video/', 'application/octet-stream'];
|
|
48
|
-
function isBinaryContentType(contentType) {
|
|
43
|
+
export function isBinaryContentType(contentType) {
|
|
49
44
|
const lower = contentType.toLowerCase();
|
|
50
45
|
return BINARY_CONTENT_TYPE_PREFIXES.some((prefix) => lower.startsWith(prefix));
|
|
51
46
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url.utils.js","sourceRoot":"","sources":["../../src/utils/url.utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"url.utils.js","sourceRoot":"","sources":["../../src/utils/url.utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,GAAG,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAChC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,cAAc,aAAa,EAAE,CAAC;IAC3E,CAAC;IAED,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC;IAC1D,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,yBAAyB,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAC;IAC5E,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACvC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,gDAAgD,EAAE,CAAC;IACjF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,sCAAsC,EAAE,CAAC;IACvE,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAQD,MAAM,UAAU,mBAAmB,CAAC,WAAmB,EAAE,WAAmB;IAC1E,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QAEtC,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC1D,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QAClD,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAEzD,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACrD,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,4BAA4B,GAAG,CAAC,iBAAiB,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,0BAA0B,CAAC,CAAC;AAEnH,MAAM,UAAU,mBAAmB,CAAC,WAAmB;IACrD,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IACxC,OAAO,4BAA4B,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;AACjF,CAAC"}
|
package/dist/web.module.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web.module.d.ts","sourceRoot":"","sources":["../src/web.module.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"web.module.d.ts","sourceRoot":"","sources":["../src/web.module.ts"],"names":[],"mappings":"AAKA,qBAKa,SAAS;CAAG"}
|
package/dist/web.module.js
CHANGED
|
@@ -1,25 +1,21 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
2
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
3
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
4
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
6
|
};
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const core_1 = require("@loopstack/core");
|
|
13
|
-
const services_1 = require("./services");
|
|
14
|
-
const tools_1 = require("./tools");
|
|
7
|
+
import { Module } from '@nestjs/common';
|
|
8
|
+
import { ClaudeModule } from '@loopstack/claude-module';
|
|
9
|
+
import { WebFetchFetcherService, WebFetchMarkdownService, WebFetchSummarizerService } from './services/index.js';
|
|
10
|
+
import { WebFetchTool } from './tools/index.js';
|
|
15
11
|
let WebModule = class WebModule {
|
|
16
12
|
};
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
exports: [tools_1.WebFetchTool],
|
|
13
|
+
WebModule = __decorate([
|
|
14
|
+
Module({
|
|
15
|
+
imports: [ClaudeModule],
|
|
16
|
+
providers: [WebFetchMarkdownService, WebFetchFetcherService, WebFetchSummarizerService, WebFetchTool],
|
|
17
|
+
exports: [WebFetchTool],
|
|
23
18
|
})
|
|
24
19
|
], WebModule);
|
|
20
|
+
export { WebModule };
|
|
25
21
|
//# sourceMappingURL=web.module.js.map
|
package/dist/web.module.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web.module.js","sourceRoot":"","sources":["../src/web.module.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"web.module.js","sourceRoot":"","sources":["../src/web.module.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AACjH,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAOzC,IAAM,SAAS,GAAf,MAAM,SAAS;CAAG,CAAA;AAAZ,SAAS;IALrB,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,YAAY,CAAC;QACvB,SAAS,EAAE,CAAC,uBAAuB,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,YAAY,CAAC;QACrG,OAAO,EAAE,CAAC,YAAY,CAAC;KACxB,CAAC;GACW,SAAS,CAAG"}
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"tool",
|
|
12
12
|
"webfetch"
|
|
13
13
|
],
|
|
14
|
-
"version": "0.
|
|
14
|
+
"version": "0.3.0",
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"author": {
|
|
17
17
|
"name": "Jakob Klippel",
|
|
@@ -20,7 +20,10 @@
|
|
|
20
20
|
"main": "dist/index.js",
|
|
21
21
|
"types": "dist/index.d.ts",
|
|
22
22
|
"exports": {
|
|
23
|
-
".":
|
|
23
|
+
".": {
|
|
24
|
+
"types": "./dist/index.d.ts",
|
|
25
|
+
"default": "./dist/index.js"
|
|
26
|
+
}
|
|
24
27
|
},
|
|
25
28
|
"scripts": {
|
|
26
29
|
"build": "nest build",
|
|
@@ -31,9 +34,9 @@
|
|
|
31
34
|
"watch": "nest build --watch"
|
|
32
35
|
},
|
|
33
36
|
"dependencies": {
|
|
34
|
-
"@loopstack/claude-module": "^0.
|
|
35
|
-
"@loopstack/common": "^0.
|
|
36
|
-
"@loopstack/core": "^0.
|
|
37
|
+
"@loopstack/claude-module": "^0.24.0",
|
|
38
|
+
"@loopstack/common": "^0.30.0",
|
|
39
|
+
"@loopstack/core": "^0.30.0",
|
|
37
40
|
"@nestjs/common": "^11.1.19",
|
|
38
41
|
"@nestjs/core": "^11.1.19",
|
|
39
42
|
"lru-cache": "^11.0.2",
|
|
@@ -64,15 +67,5 @@
|
|
|
64
67
|
"testEnvironment": "node",
|
|
65
68
|
"maxWorkers": 1
|
|
66
69
|
},
|
|
67
|
-
"
|
|
68
|
-
"modules": [
|
|
69
|
-
{
|
|
70
|
-
"path": "src/web.module.ts",
|
|
71
|
-
"className": "WebModule"
|
|
72
|
-
}
|
|
73
|
-
],
|
|
74
|
-
"installModes": [
|
|
75
|
-
"install"
|
|
76
|
-
]
|
|
77
|
-
}
|
|
70
|
+
"type": "module"
|
|
78
71
|
}
|