@j0hanz/superfetch 1.1.9 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/README.md +179 -469
  2. package/dist/config/constants.d.ts +19 -0
  3. package/dist/config/constants.d.ts.map +1 -0
  4. package/dist/config/constants.js +24 -0
  5. package/dist/config/constants.js.map +1 -0
  6. package/dist/config/formatting.d.ts +0 -2
  7. package/dist/config/formatting.d.ts.map +1 -1
  8. package/dist/config/formatting.js +1 -3
  9. package/dist/config/formatting.js.map +1 -1
  10. package/dist/config/index.d.ts +9 -3
  11. package/dist/config/index.d.ts.map +1 -1
  12. package/dist/config/index.js +19 -16
  13. package/dist/config/index.js.map +1 -1
  14. package/dist/config/types/content.d.ts +1 -20
  15. package/dist/config/types/content.d.ts.map +1 -1
  16. package/dist/config/types/content.js +0 -1
  17. package/dist/config/types/runtime.d.ts +7 -5
  18. package/dist/config/types/runtime.d.ts.map +1 -1
  19. package/dist/config/types/runtime.js +0 -1
  20. package/dist/config/types/tools.d.ts +5 -50
  21. package/dist/config/types/tools.d.ts.map +1 -1
  22. package/dist/config/types/tools.js +0 -1
  23. package/dist/errors/app-error.d.ts +0 -1
  24. package/dist/errors/app-error.js +0 -1
  25. package/dist/http/auth.d.ts +0 -1
  26. package/dist/http/auth.d.ts.map +1 -1
  27. package/dist/http/auth.js +17 -13
  28. package/dist/http/auth.js.map +1 -1
  29. package/dist/http/cors.d.ts +0 -1
  30. package/dist/http/cors.js +4 -1
  31. package/dist/http/cors.js.map +1 -1
  32. package/dist/http/download-routes.d.ts +14 -0
  33. package/dist/http/download-routes.d.ts.map +1 -0
  34. package/dist/http/download-routes.js +131 -0
  35. package/dist/http/download-routes.js.map +1 -0
  36. package/dist/http/mcp-routes.d.ts +1 -2
  37. package/dist/http/mcp-routes.d.ts.map +1 -1
  38. package/dist/http/mcp-routes.js +1 -2
  39. package/dist/http/mcp-routes.js.map +1 -1
  40. package/dist/http/mcp-session-helpers.d.ts +13 -0
  41. package/dist/http/mcp-session-helpers.d.ts.map +1 -0
  42. package/dist/http/mcp-session-helpers.js +64 -0
  43. package/dist/http/mcp-session-helpers.js.map +1 -0
  44. package/dist/http/mcp-session.d.ts +1 -3
  45. package/dist/http/mcp-session.d.ts.map +1 -1
  46. package/dist/http/mcp-session.js +7 -71
  47. package/dist/http/mcp-session.js.map +1 -1
  48. package/dist/http/mcp-validation.d.ts +1 -2
  49. package/dist/http/mcp-validation.d.ts.map +1 -1
  50. package/dist/http/mcp-validation.js +6 -27
  51. package/dist/http/mcp-validation.js.map +1 -1
  52. package/dist/http/rate-limit.d.ts +1 -2
  53. package/dist/http/rate-limit.d.ts.map +1 -1
  54. package/dist/http/rate-limit.js +0 -1
  55. package/dist/http/rate-limit.js.map +1 -1
  56. package/dist/http/server-middleware.d.ts +9 -0
  57. package/dist/http/server-middleware.d.ts.map +1 -0
  58. package/dist/http/server-middleware.js +111 -0
  59. package/dist/http/server-middleware.js.map +1 -0
  60. package/dist/http/server.d.ts +0 -1
  61. package/dist/http/server.d.ts.map +1 -1
  62. package/dist/http/server.js +20 -99
  63. package/dist/http/server.js.map +1 -1
  64. package/dist/http/session-cleanup.d.ts +2 -0
  65. package/dist/http/session-cleanup.d.ts.map +1 -0
  66. package/dist/http/session-cleanup.js +37 -0
  67. package/dist/http/session-cleanup.js.map +1 -0
  68. package/dist/http/sessions.d.ts +1 -2
  69. package/dist/http/sessions.d.ts.map +1 -1
  70. package/dist/http/sessions.js +0 -1
  71. package/dist/index.d.ts +0 -1
  72. package/dist/index.js +13 -6
  73. package/dist/index.js.map +1 -1
  74. package/dist/middleware/error-handler.d.ts +0 -1
  75. package/dist/middleware/error-handler.js +0 -1
  76. package/dist/resources/cached-content.d.ts +0 -1
  77. package/dist/resources/cached-content.d.ts.map +1 -1
  78. package/dist/resources/cached-content.js +76 -12
  79. package/dist/resources/cached-content.js.map +1 -1
  80. package/dist/resources/index.d.ts +0 -1
  81. package/dist/resources/index.js +0 -1
  82. package/dist/server.d.ts +0 -1
  83. package/dist/server.d.ts.map +1 -1
  84. package/dist/server.js +8 -3
  85. package/dist/server.js.map +1 -1
  86. package/dist/services/cache.d.ts +7 -4
  87. package/dist/services/cache.d.ts.map +1 -1
  88. package/dist/services/cache.js +86 -26
  89. package/dist/services/cache.js.map +1 -1
  90. package/dist/services/context.d.ts +2 -2
  91. package/dist/services/context.d.ts.map +1 -1
  92. package/dist/services/context.js +0 -1
  93. package/dist/services/extractor.d.ts +1 -2
  94. package/dist/services/extractor.d.ts.map +1 -1
  95. package/dist/services/extractor.js +45 -18
  96. package/dist/services/extractor.js.map +1 -1
  97. package/dist/services/fetcher/agents.d.ts +0 -1
  98. package/dist/services/fetcher/agents.d.ts.map +1 -1
  99. package/dist/services/fetcher/agents.js +3 -7
  100. package/dist/services/fetcher/agents.js.map +1 -1
  101. package/dist/services/fetcher/errors.d.ts +0 -1
  102. package/dist/services/fetcher/errors.js +0 -1
  103. package/dist/services/fetcher/headers.d.ts.map +1 -1
  104. package/dist/services/fetcher/headers.js +2 -24
  105. package/dist/services/fetcher/headers.js.map +1 -1
  106. package/dist/services/fetcher/interceptors.d.ts +2 -2
  107. package/dist/services/fetcher/interceptors.d.ts.map +1 -1
  108. package/dist/services/fetcher/interceptors.js +30 -21
  109. package/dist/services/fetcher/interceptors.js.map +1 -1
  110. package/dist/services/fetcher/redirects.d.ts +0 -2
  111. package/dist/services/fetcher/redirects.d.ts.map +1 -1
  112. package/dist/services/fetcher/redirects.js +20 -18
  113. package/dist/services/fetcher/redirects.js.map +1 -1
  114. package/dist/services/fetcher/response.d.ts +0 -1
  115. package/dist/services/fetcher/response.js +4 -5
  116. package/dist/services/fetcher/retry-policy.d.ts +1 -28
  117. package/dist/services/fetcher/retry-policy.d.ts.map +1 -1
  118. package/dist/services/fetcher/retry-policy.js +119 -126
  119. package/dist/services/fetcher/retry-policy.js.map +1 -1
  120. package/dist/services/fetcher.d.ts +1 -2
  121. package/dist/services/fetcher.d.ts.map +1 -1
  122. package/dist/services/fetcher.js +18 -13
  123. package/dist/services/fetcher.js.map +1 -1
  124. package/dist/services/logger.d.ts +1 -2
  125. package/dist/services/logger.d.ts.map +1 -1
  126. package/dist/services/logger.js +0 -1
  127. package/dist/services/parser.d.ts +1 -3
  128. package/dist/services/parser.d.ts.map +1 -1
  129. package/dist/services/parser.js +5 -39
  130. package/dist/services/parser.js.map +1 -1
  131. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -1
  132. package/dist/tools/handlers/fetch-links/link-extractor.js +15 -19
  133. package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -1
  134. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  135. package/dist/tools/handlers/fetch-links.tool.js +0 -2
  136. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  137. package/dist/tools/handlers/fetch-markdown.tool.d.ts +1 -2
  138. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  139. package/dist/tools/handlers/fetch-markdown.tool.js +50 -20
  140. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  141. package/dist/tools/handlers/fetch-single.shared.d.ts +14 -3
  142. package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -1
  143. package/dist/tools/handlers/fetch-single.shared.js +66 -3
  144. package/dist/tools/handlers/fetch-single.shared.js.map +1 -1
  145. package/dist/tools/handlers/fetch-url.tool.d.ts +1 -2
  146. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  147. package/dist/tools/handlers/fetch-url.tool.js +39 -17
  148. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  149. package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -1
  150. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -1
  151. package/dist/tools/handlers/fetch-urls/validation.js +1 -1
  152. package/dist/tools/handlers/fetch-urls/validation.js.map +1 -1
  153. package/dist/tools/index.d.ts +0 -1
  154. package/dist/tools/index.d.ts.map +1 -1
  155. package/dist/tools/index.js +1 -20
  156. package/dist/tools/index.js.map +1 -1
  157. package/dist/tools/schemas.d.ts +57 -250
  158. package/dist/tools/schemas.d.ts.map +1 -1
  159. package/dist/tools/schemas.js +38 -198
  160. package/dist/tools/schemas.js.map +1 -1
  161. package/dist/tools/utils/cache-vary.d.ts +0 -2
  162. package/dist/tools/utils/cache-vary.d.ts.map +1 -1
  163. package/dist/tools/utils/cache-vary.js +8 -40
  164. package/dist/tools/utils/cache-vary.js.map +1 -1
  165. package/dist/tools/utils/common.d.ts +2 -4
  166. package/dist/tools/utils/common.d.ts.map +1 -1
  167. package/dist/tools/utils/common.js +6 -7
  168. package/dist/tools/utils/common.js.map +1 -1
  169. package/dist/tools/utils/content-transform.d.ts +1 -3
  170. package/dist/tools/utils/content-transform.d.ts.map +1 -1
  171. package/dist/tools/utils/content-transform.js +65 -14
  172. package/dist/tools/utils/content-transform.js.map +1 -1
  173. package/dist/tools/utils/fetch-pipeline.d.ts +1 -2
  174. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  175. package/dist/tools/utils/fetch-pipeline.js +25 -21
  176. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  177. package/dist/tools/utils/inline-content.d.ts +3 -3
  178. package/dist/tools/utils/inline-content.d.ts.map +1 -1
  179. package/dist/tools/utils/inline-content.js +0 -1
  180. package/dist/transformers/jsonl.transformer.d.ts +1 -2
  181. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  182. package/dist/transformers/jsonl.transformer.js +0 -1
  183. package/dist/transformers/jsonl.transformer.js.map +1 -1
  184. package/dist/transformers/markdown.transformer.d.ts +1 -2
  185. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  186. package/dist/transformers/markdown.transformer.js +11 -7
  187. package/dist/transformers/markdown.transformer.js.map +1 -1
  188. package/dist/utils/code-language.d.ts +2 -0
  189. package/dist/utils/code-language.d.ts.map +1 -0
  190. package/dist/utils/code-language.js +56 -0
  191. package/dist/utils/code-language.js.map +1 -0
  192. package/dist/utils/content-cleaner.d.ts +0 -2
  193. package/dist/utils/content-cleaner.d.ts.map +1 -1
  194. package/dist/utils/content-cleaner.js +0 -4
  195. package/dist/utils/content-cleaner.js.map +1 -1
  196. package/dist/utils/crypto.d.ts +2 -0
  197. package/dist/utils/crypto.d.ts.map +1 -0
  198. package/dist/utils/crypto.js +32 -0
  199. package/dist/utils/crypto.js.map +1 -0
  200. package/dist/utils/download-url.d.ts +8 -0
  201. package/dist/utils/download-url.d.ts.map +1 -0
  202. package/dist/utils/download-url.js +27 -0
  203. package/dist/utils/download-url.js.map +1 -0
  204. package/dist/utils/error-utils.d.ts +3 -0
  205. package/dist/utils/error-utils.d.ts.map +1 -0
  206. package/dist/utils/error-utils.js +12 -0
  207. package/dist/utils/error-utils.js.map +1 -0
  208. package/dist/utils/filename-generator.d.ts +1 -0
  209. package/dist/utils/filename-generator.d.ts.map +1 -0
  210. package/dist/utils/filename-generator.js +59 -0
  211. package/dist/utils/filename-generator.js.map +1 -0
  212. package/dist/utils/header-normalizer.d.ts +7 -4
  213. package/dist/utils/header-normalizer.d.ts.map +1 -1
  214. package/dist/utils/header-normalizer.js +23 -17
  215. package/dist/utils/header-normalizer.js.map +1 -1
  216. package/dist/utils/html-truncator.d.ts +0 -1
  217. package/dist/utils/html-truncator.js +0 -1
  218. package/dist/utils/sanitizer.d.ts +0 -1
  219. package/dist/utils/sanitizer.js +0 -1
  220. package/dist/utils/tool-error-handler.d.ts +1 -3
  221. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  222. package/dist/utils/tool-error-handler.js +11 -6
  223. package/dist/utils/tool-error-handler.js.map +1 -1
  224. package/dist/utils/url-sanitizer.d.ts +2 -0
  225. package/dist/utils/url-sanitizer.d.ts.map +1 -0
  226. package/dist/utils/url-sanitizer.js +12 -0
  227. package/dist/utils/url-sanitizer.js.map +1 -0
  228. package/dist/utils/url-validator.d.ts +1 -3
  229. package/dist/utils/url-validator.d.ts.map +1 -1
  230. package/dist/utils/url-validator.js +89 -53
  231. package/dist/utils/url-validator.js.map +1 -1
  232. package/package.json +7 -9
@@ -1,71 +1,42 @@
1
1
  import { z } from 'zod';
2
2
  export declare const fetchUrlInputSchema: z.ZodObject<{
3
3
  customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
4
- timeout: z.ZodOptional<z.ZodNumber>;
5
- retries: z.ZodOptional<z.ZodNumber>;
4
+ timeout: z.ZodDefault<z.ZodNumber>;
5
+ retries: z.ZodDefault<z.ZodNumber>;
6
6
  } & {
7
7
  url: z.ZodString;
8
+ } & {
8
9
  extractMainContent: z.ZodDefault<z.ZodBoolean>;
9
10
  includeMetadata: z.ZodDefault<z.ZodBoolean>;
10
11
  maxContentLength: z.ZodOptional<z.ZodNumber>;
12
+ } & {
11
13
  format: z.ZodDefault<z.ZodEnum<["jsonl", "markdown"]>>;
12
14
  }, "strict", z.ZodTypeAny, {
13
15
  url: string;
14
16
  extractMainContent: boolean;
15
17
  includeMetadata: boolean;
18
+ retries: number;
16
19
  format: "jsonl" | "markdown";
17
- maxContentLength?: number | undefined;
20
+ timeout: number;
18
21
  customHeaders?: Record<string, string> | undefined;
19
- timeout?: number | undefined;
20
- retries?: number | undefined;
22
+ maxContentLength?: number | undefined;
21
23
  }, {
22
24
  url: string;
25
+ customHeaders?: Record<string, string> | undefined;
23
26
  extractMainContent?: boolean | undefined;
24
27
  includeMetadata?: boolean | undefined;
25
- maxContentLength?: number | undefined;
26
- format?: "jsonl" | "markdown" | undefined;
27
- customHeaders?: Record<string, string> | undefined;
28
- timeout?: number | undefined;
29
28
  retries?: number | undefined;
30
- }>;
31
- export declare const fetchLinksInputSchema: z.ZodObject<{
32
- customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
33
- timeout: z.ZodOptional<z.ZodNumber>;
34
- retries: z.ZodOptional<z.ZodNumber>;
35
- } & {
36
- url: z.ZodString;
37
- includeExternal: z.ZodDefault<z.ZodBoolean>;
38
- includeInternal: z.ZodDefault<z.ZodBoolean>;
39
- maxLinks: z.ZodOptional<z.ZodNumber>;
40
- filterPattern: z.ZodOptional<z.ZodString>;
41
- includeImages: z.ZodDefault<z.ZodBoolean>;
42
- }, "strict", z.ZodTypeAny, {
43
- url: string;
44
- includeExternal: boolean;
45
- includeInternal: boolean;
46
- includeImages: boolean;
47
- maxLinks?: number | undefined;
48
- filterPattern?: string | undefined;
49
- customHeaders?: Record<string, string> | undefined;
50
- timeout?: number | undefined;
51
- retries?: number | undefined;
52
- }, {
53
- url: string;
54
- includeExternal?: boolean | undefined;
55
- includeInternal?: boolean | undefined;
56
- maxLinks?: number | undefined;
57
- filterPattern?: string | undefined;
58
- includeImages?: boolean | undefined;
59
- customHeaders?: Record<string, string> | undefined;
29
+ format?: "jsonl" | "markdown" | undefined;
30
+ maxContentLength?: number | undefined;
60
31
  timeout?: number | undefined;
61
- retries?: number | undefined;
62
32
  }>;
63
33
  export declare const fetchMarkdownInputSchema: z.ZodObject<{
64
34
  customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
65
- timeout: z.ZodOptional<z.ZodNumber>;
66
- retries: z.ZodOptional<z.ZodNumber>;
35
+ timeout: z.ZodDefault<z.ZodNumber>;
36
+ retries: z.ZodDefault<z.ZodNumber>;
67
37
  } & {
68
38
  url: z.ZodString;
39
+ } & {
69
40
  extractMainContent: z.ZodDefault<z.ZodBoolean>;
70
41
  includeMetadata: z.ZodDefault<z.ZodBoolean>;
71
42
  maxContentLength: z.ZodOptional<z.ZodNumber>;
@@ -73,53 +44,18 @@ export declare const fetchMarkdownInputSchema: z.ZodObject<{
73
44
  url: string;
74
45
  extractMainContent: boolean;
75
46
  includeMetadata: boolean;
76
- maxContentLength?: number | undefined;
47
+ retries: number;
48
+ timeout: number;
77
49
  customHeaders?: Record<string, string> | undefined;
78
- timeout?: number | undefined;
79
- retries?: number | undefined;
50
+ maxContentLength?: number | undefined;
80
51
  }, {
81
52
  url: string;
82
- extractMainContent?: boolean | undefined;
83
- includeMetadata?: boolean | undefined;
84
- maxContentLength?: number | undefined;
85
- customHeaders?: Record<string, string> | undefined;
86
- timeout?: number | undefined;
87
- retries?: number | undefined;
88
- }>;
89
- export declare const fetchUrlsInputSchema: z.ZodObject<{
90
- customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
91
- timeout: z.ZodOptional<z.ZodNumber>;
92
- retries: z.ZodOptional<z.ZodNumber>;
93
- } & {
94
- urls: z.ZodArray<z.ZodString, "many">;
95
- extractMainContent: z.ZodDefault<z.ZodBoolean>;
96
- includeMetadata: z.ZodDefault<z.ZodBoolean>;
97
- maxContentLength: z.ZodOptional<z.ZodNumber>;
98
- format: z.ZodDefault<z.ZodEnum<["jsonl", "markdown"]>>;
99
- concurrency: z.ZodDefault<z.ZodNumber>;
100
- continueOnError: z.ZodDefault<z.ZodBoolean>;
101
- }, "strict", z.ZodTypeAny, {
102
- extractMainContent: boolean;
103
- includeMetadata: boolean;
104
- format: "jsonl" | "markdown";
105
- urls: string[];
106
- concurrency: number;
107
- continueOnError: boolean;
108
- maxContentLength?: number | undefined;
109
53
  customHeaders?: Record<string, string> | undefined;
110
- timeout?: number | undefined;
111
- retries?: number | undefined;
112
- }, {
113
- urls: string[];
114
54
  extractMainContent?: boolean | undefined;
115
55
  includeMetadata?: boolean | undefined;
56
+ retries?: number | undefined;
116
57
  maxContentLength?: number | undefined;
117
- format?: "jsonl" | "markdown" | undefined;
118
- concurrency?: number | undefined;
119
- continueOnError?: boolean | undefined;
120
- customHeaders?: Record<string, string> | undefined;
121
58
  timeout?: number | undefined;
122
- retries?: number | undefined;
123
59
  }>;
124
60
  export declare const fetchUrlOutputSchema: z.ZodObject<{
125
61
  url: z.ZodString;
@@ -128,6 +64,7 @@ export declare const fetchUrlOutputSchema: z.ZodObject<{
128
64
  fetchedAt: z.ZodString;
129
65
  format: z.ZodEnum<["jsonl", "markdown"]>;
130
66
  content: z.ZodOptional<z.ZodString>;
67
+ } & {
131
68
  contentSize: z.ZodOptional<z.ZodNumber>;
132
69
  resourceUri: z.ZodOptional<z.ZodString>;
133
70
  resourceMimeType: z.ZodOptional<z.ZodString>;
@@ -137,83 +74,52 @@ export declare const fetchUrlOutputSchema: z.ZodObject<{
137
74
  errorCode: z.ZodOptional<z.ZodString>;
138
75
  }, "strict", z.ZodTypeAny, {
139
76
  url: string;
140
- format: "jsonl" | "markdown";
77
+ contentBlocks: number;
141
78
  fetchedAt: string;
79
+ format: "jsonl" | "markdown";
142
80
  cached: boolean;
143
- contentBlocks: number;
144
- content?: string | undefined;
145
81
  error?: string | undefined;
146
- errorCode?: string | undefined;
82
+ content?: string | undefined;
147
83
  title?: string | undefined;
148
84
  truncated?: boolean | undefined;
149
- contentSize?: number | undefined;
150
85
  resourceUri?: string | undefined;
151
86
  resourceMimeType?: string | undefined;
87
+ contentSize?: number | undefined;
88
+ errorCode?: string | undefined;
152
89
  }, {
153
90
  url: string;
154
- format: "jsonl" | "markdown";
91
+ contentBlocks: number;
155
92
  fetchedAt: string;
93
+ format: "jsonl" | "markdown";
156
94
  cached: boolean;
157
- contentBlocks: number;
158
- content?: string | undefined;
159
95
  error?: string | undefined;
160
- errorCode?: string | undefined;
96
+ content?: string | undefined;
161
97
  title?: string | undefined;
162
98
  truncated?: boolean | undefined;
163
- contentSize?: number | undefined;
164
99
  resourceUri?: string | undefined;
165
100
  resourceMimeType?: string | undefined;
166
- }>;
167
- export declare const fetchLinksOutputSchema: z.ZodObject<{
168
- url: z.ZodString;
169
- linkCount: z.ZodNumber;
170
- links: z.ZodArray<z.ZodObject<{
171
- href: z.ZodString;
172
- text: z.ZodString;
173
- type: z.ZodEnum<["internal", "external", "image"]>;
174
- }, "strict", z.ZodTypeAny, {
175
- type: "image" | "internal" | "external";
176
- href: string;
177
- text: string;
178
- }, {
179
- type: "image" | "internal" | "external";
180
- href: string;
181
- text: string;
182
- }>, "many">;
183
- filtered: z.ZodOptional<z.ZodNumber>;
184
- truncated: z.ZodOptional<z.ZodBoolean>;
185
- error: z.ZodOptional<z.ZodString>;
186
- errorCode: z.ZodOptional<z.ZodString>;
187
- }, "strict", z.ZodTypeAny, {
188
- url: string;
189
- linkCount: number;
190
- links: {
191
- type: "image" | "internal" | "external";
192
- href: string;
193
- text: string;
194
- }[];
195
- error?: string | undefined;
196
- errorCode?: string | undefined;
197
- truncated?: boolean | undefined;
198
- filtered?: number | undefined;
199
- }, {
200
- url: string;
201
- linkCount: number;
202
- links: {
203
- type: "image" | "internal" | "external";
204
- href: string;
205
- text: string;
206
- }[];
207
- error?: string | undefined;
101
+ contentSize?: number | undefined;
208
102
  errorCode?: string | undefined;
209
- truncated?: boolean | undefined;
210
- filtered?: number | undefined;
211
103
  }>;
212
104
  export declare const fetchMarkdownOutputSchema: z.ZodObject<{
213
105
  url: z.ZodString;
214
106
  title: z.ZodOptional<z.ZodString>;
215
107
  fetchedAt: z.ZodString;
216
108
  markdown: z.ZodOptional<z.ZodString>;
109
+ file: z.ZodOptional<z.ZodObject<{
110
+ downloadUrl: z.ZodString;
111
+ fileName: z.ZodString;
112
+ expiresAt: z.ZodString;
113
+ }, "strip", z.ZodTypeAny, {
114
+ fileName: string;
115
+ expiresAt: string;
116
+ downloadUrl: string;
117
+ }, {
118
+ fileName: string;
119
+ expiresAt: string;
120
+ downloadUrl: string;
121
+ }>>;
122
+ } & {
217
123
  contentSize: z.ZodOptional<z.ZodNumber>;
218
124
  resourceUri: z.ZodOptional<z.ZodString>;
219
125
  resourceMimeType: z.ZodOptional<z.ZodString>;
@@ -225,133 +131,34 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
225
131
  url: string;
226
132
  fetchedAt: string;
227
133
  cached: boolean;
228
- markdown?: string | undefined;
229
134
  error?: string | undefined;
230
- errorCode?: string | undefined;
135
+ markdown?: string | undefined;
231
136
  title?: string | undefined;
232
137
  truncated?: boolean | undefined;
233
- contentSize?: number | undefined;
234
138
  resourceUri?: string | undefined;
235
139
  resourceMimeType?: string | undefined;
140
+ contentSize?: number | undefined;
141
+ file?: {
142
+ fileName: string;
143
+ expiresAt: string;
144
+ downloadUrl: string;
145
+ } | undefined;
146
+ errorCode?: string | undefined;
236
147
  }, {
237
148
  url: string;
238
149
  fetchedAt: string;
239
150
  cached: boolean;
240
- markdown?: string | undefined;
241
151
  error?: string | undefined;
242
- errorCode?: string | undefined;
152
+ markdown?: string | undefined;
243
153
  title?: string | undefined;
244
154
  truncated?: boolean | undefined;
245
- contentSize?: number | undefined;
246
155
  resourceUri?: string | undefined;
247
156
  resourceMimeType?: string | undefined;
157
+ contentSize?: number | undefined;
158
+ file?: {
159
+ fileName: string;
160
+ expiresAt: string;
161
+ downloadUrl: string;
162
+ } | undefined;
163
+ errorCode?: string | undefined;
248
164
  }>;
249
- export declare const fetchUrlsOutputSchema: z.ZodObject<{
250
- results: z.ZodArray<z.ZodObject<{
251
- url: z.ZodString;
252
- success: z.ZodBoolean;
253
- title: z.ZodOptional<z.ZodString>;
254
- content: z.ZodOptional<z.ZodString>;
255
- contentSize: z.ZodOptional<z.ZodNumber>;
256
- resourceUri: z.ZodOptional<z.ZodString>;
257
- resourceMimeType: z.ZodOptional<z.ZodString>;
258
- contentBlocks: z.ZodOptional<z.ZodNumber>;
259
- cached: z.ZodOptional<z.ZodBoolean>;
260
- truncated: z.ZodOptional<z.ZodBoolean>;
261
- error: z.ZodOptional<z.ZodString>;
262
- errorCode: z.ZodOptional<z.ZodString>;
263
- }, "strict", z.ZodTypeAny, {
264
- url: string;
265
- success: boolean;
266
- content?: string | undefined;
267
- error?: string | undefined;
268
- errorCode?: string | undefined;
269
- title?: string | undefined;
270
- truncated?: boolean | undefined;
271
- contentSize?: number | undefined;
272
- cached?: boolean | undefined;
273
- resourceUri?: string | undefined;
274
- resourceMimeType?: string | undefined;
275
- contentBlocks?: number | undefined;
276
- }, {
277
- url: string;
278
- success: boolean;
279
- content?: string | undefined;
280
- error?: string | undefined;
281
- errorCode?: string | undefined;
282
- title?: string | undefined;
283
- truncated?: boolean | undefined;
284
- contentSize?: number | undefined;
285
- cached?: boolean | undefined;
286
- resourceUri?: string | undefined;
287
- resourceMimeType?: string | undefined;
288
- contentBlocks?: number | undefined;
289
- }>, "many">;
290
- summary: z.ZodObject<{
291
- total: z.ZodNumber;
292
- successful: z.ZodNumber;
293
- failed: z.ZodNumber;
294
- cached: z.ZodNumber;
295
- totalContentBlocks: z.ZodNumber;
296
- }, "strict", z.ZodTypeAny, {
297
- cached: number;
298
- total: number;
299
- successful: number;
300
- failed: number;
301
- totalContentBlocks: number;
302
- }, {
303
- cached: number;
304
- total: number;
305
- successful: number;
306
- failed: number;
307
- totalContentBlocks: number;
308
- }>;
309
- fetchedAt: z.ZodString;
310
- }, "strict", z.ZodTypeAny, {
311
- results: {
312
- url: string;
313
- success: boolean;
314
- content?: string | undefined;
315
- error?: string | undefined;
316
- errorCode?: string | undefined;
317
- title?: string | undefined;
318
- truncated?: boolean | undefined;
319
- contentSize?: number | undefined;
320
- cached?: boolean | undefined;
321
- resourceUri?: string | undefined;
322
- resourceMimeType?: string | undefined;
323
- contentBlocks?: number | undefined;
324
- }[];
325
- summary: {
326
- cached: number;
327
- total: number;
328
- successful: number;
329
- failed: number;
330
- totalContentBlocks: number;
331
- };
332
- fetchedAt: string;
333
- }, {
334
- results: {
335
- url: string;
336
- success: boolean;
337
- content?: string | undefined;
338
- error?: string | undefined;
339
- errorCode?: string | undefined;
340
- title?: string | undefined;
341
- truncated?: boolean | undefined;
342
- contentSize?: number | undefined;
343
- cached?: boolean | undefined;
344
- resourceUri?: string | undefined;
345
- resourceMimeType?: string | undefined;
346
- contentBlocks?: number | undefined;
347
- }[];
348
- summary: {
349
- cached: number;
350
- total: number;
351
- successful: number;
352
- failed: number;
353
- totalContentBlocks: number;
354
- };
355
- fetchedAt: string;
356
- }>;
357
- //# sourceMappingURL=schemas.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../../src/tools/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAuDxB,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqBrB,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA0BvB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;EAiB1B,CAAC;AAEZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmCtB,CAAC;AAEZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsCtB,CAAC;AAEZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsBxB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkC3B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmBvB,CAAC"}
1
+ {"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../../src/tools/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAuExB,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMrB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;EAK1B,CAAC;AAEZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiBtB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgB3B,CAAC"}
@@ -1,4 +1,5 @@
1
1
  import { z } from 'zod';
2
+ import { config } from '../config/index.js';
2
3
  const requestOptionsSchema = z.object({
3
4
  customHeaders: z
4
5
  .record(z.string())
@@ -7,51 +8,17 @@ const requestOptionsSchema = z.object({
7
8
  timeout: z
8
9
  .number()
9
10
  .min(1000)
10
- .max(60000)
11
- .optional()
12
- .describe('Request timeout in milliseconds (1000-60000)'),
11
+ .max(120000)
12
+ .default(config.fetcher.timeout)
13
+ .describe('Request timeout in milliseconds (1000-120000)'),
13
14
  retries: z
14
15
  .number()
15
16
  .min(1)
16
17
  .max(10)
17
- .optional()
18
+ .default(3)
18
19
  .describe('Number of retry attempts (1-10)'),
19
20
  });
20
- const linkEntrySchema = z
21
- .object({
22
- href: z.string().describe('The link URL'),
23
- text: z.string().describe('The link anchor text'),
24
- type: z.enum(['internal', 'external', 'image']).describe('Link type'),
25
- })
26
- .strict();
27
- const batchResultSchema = z
28
- .object({
29
- url: z.string().describe('The fetched URL'),
30
- success: z.boolean().describe('Whether the fetch was successful'),
31
- title: z.string().optional().describe('Page title'),
32
- content: z.string().optional().describe('The extracted content'),
33
- contentSize: z.number().optional().describe('Content length in characters'),
34
- resourceUri: z
35
- .string()
36
- .optional()
37
- .describe('Resource URI when content is too large to inline'),
38
- resourceMimeType: z
39
- .string()
40
- .optional()
41
- .describe('MIME type for the resource URI'),
42
- contentBlocks: z
43
- .number()
44
- .optional()
45
- .describe('Number of content blocks (JSONL only)'),
46
- cached: z.boolean().optional().describe('Whether served from cache'),
47
- truncated: z.boolean().optional().describe('Whether content was truncated'),
48
- error: z.string().optional().describe('Error message if failed'),
49
- errorCode: z.string().optional().describe('Error code if failed'),
50
- })
51
- .strict();
52
- export const fetchUrlInputSchema = requestOptionsSchema
53
- .extend({
54
- url: z.string().min(1).describe('The URL to fetch'),
21
+ const extractionOptionsSchema = z.object({
55
22
  extractMainContent: z
56
23
  .boolean()
57
24
  .default(true)
@@ -65,92 +32,48 @@ export const fetchUrlInputSchema = requestOptionsSchema
65
32
  .positive()
66
33
  .optional()
67
34
  .describe('Maximum content length in characters'),
35
+ });
36
+ const formatOptionsSchema = z.object({
68
37
  format: z
69
38
  .enum(['jsonl', 'markdown'])
70
39
  .default('jsonl')
71
40
  .describe('Output format'),
72
- })
73
- .strict();
74
- export const fetchLinksInputSchema = requestOptionsSchema
75
- .extend({
76
- url: z.string().min(1).describe('The URL to extract links from'),
77
- includeExternal: z
78
- .boolean()
79
- .default(true)
80
- .describe('Include external links'),
81
- includeInternal: z
82
- .boolean()
83
- .default(true)
84
- .describe('Include internal links'),
85
- maxLinks: z
86
- .number()
87
- .positive()
88
- .max(1000)
41
+ });
42
+ const resourceFieldsSchema = z.object({
43
+ contentSize: z.number().optional().describe('Content length in characters'),
44
+ resourceUri: z
45
+ .string()
89
46
  .optional()
90
- .describe('Maximum number of links to return (1-1000)'),
91
- filterPattern: z
47
+ .describe('Resource URI when content is too large to inline'),
48
+ resourceMimeType: z
92
49
  .string()
93
50
  .optional()
94
- .describe('Regex pattern to filter links (matches against href)'),
95
- includeImages: z
51
+ .describe('MIME type for the resource URI'),
52
+ cached: z.boolean().describe('Whether the result was served from cache'),
53
+ truncated: z
96
54
  .boolean()
97
- .default(false)
98
- .describe('Include image links (img src attributes)'),
99
- })
100
- .strict();
101
- export const fetchMarkdownInputSchema = requestOptionsSchema
55
+ .optional()
56
+ .describe('Whether content was truncated by maxContentLength'),
57
+ error: z.string().optional().describe('Error message if the request failed'),
58
+ errorCode: z.string().optional().describe('Error code if the request failed'),
59
+ });
60
+ const fileDownloadSchema = z.object({
61
+ downloadUrl: z.string().describe('Relative URL to download the .md file'),
62
+ fileName: z.string().describe('Suggested filename for download'),
63
+ expiresAt: z.string().describe('ISO timestamp when download expires'),
64
+ });
65
+ export const fetchUrlInputSchema = requestOptionsSchema
102
66
  .extend({
103
67
  url: z.string().min(1).describe('The URL to fetch'),
104
- extractMainContent: z
105
- .boolean()
106
- .default(true)
107
- .describe('Extract main article content using Readability'),
108
- includeMetadata: z
109
- .boolean()
110
- .default(true)
111
- .describe('Include YAML frontmatter metadata'),
112
- maxContentLength: z
113
- .number()
114
- .positive()
115
- .optional()
116
- .describe('Maximum content length in characters'),
117
68
  })
69
+ .merge(extractionOptionsSchema)
70
+ .merge(formatOptionsSchema)
118
71
  .strict();
119
- export const fetchUrlsInputSchema = requestOptionsSchema
72
+ export const fetchMarkdownInputSchema = requestOptionsSchema
120
73
  .extend({
121
- urls: z
122
- .array(z.string().min(1))
123
- .min(1)
124
- .max(10)
125
- .describe('Array of URLs to fetch (1-10 URLs)'),
126
- extractMainContent: z
127
- .boolean()
128
- .default(true)
129
- .describe('Use Readability to extract main article content'),
130
- includeMetadata: z
131
- .boolean()
132
- .default(true)
133
- .describe('Include page metadata (title, description, etc.)'),
134
- maxContentLength: z
135
- .number()
136
- .positive()
137
- .optional()
138
- .describe('Maximum content length per URL in characters'),
139
- format: z
140
- .enum(['jsonl', 'markdown'])
141
- .default('jsonl')
142
- .describe('Output format for all URLs'),
143
- concurrency: z
144
- .number()
145
- .min(1)
146
- .max(5)
147
- .default(3)
148
- .describe('Maximum concurrent requests (1-5)'),
149
- continueOnError: z
150
- .boolean()
151
- .default(true)
152
- .describe('Continue processing if some URLs fail'),
74
+ url: z.string().min(1).describe('The URL to fetch'),
153
75
  })
76
+ .merge(extractionOptionsSchema)
154
77
  .strict();
155
78
  export const fetchUrlOutputSchema = z
156
79
  .object({
@@ -167,52 +90,8 @@ export const fetchUrlOutputSchema = z
167
90
  .string()
168
91
  .optional()
169
92
  .describe('The extracted content in JSONL or Markdown format'),
170
- contentSize: z.number().optional().describe('Content length in characters'),
171
- resourceUri: z
172
- .string()
173
- .optional()
174
- .describe('Resource URI when content is too large to inline'),
175
- resourceMimeType: z
176
- .string()
177
- .optional()
178
- .describe('MIME type for the resource URI'),
179
- cached: z.boolean().describe('Whether the result was served from cache'),
180
- truncated: z
181
- .boolean()
182
- .optional()
183
- .describe('Whether content was truncated by maxContentLength'),
184
- error: z
185
- .string()
186
- .optional()
187
- .describe('Error message if the request failed'),
188
- errorCode: z
189
- .string()
190
- .optional()
191
- .describe('Error code if the request failed'),
192
- })
193
- .strict();
194
- export const fetchLinksOutputSchema = z
195
- .object({
196
- url: z.string().describe('The source URL'),
197
- linkCount: z.number().describe('Total number of links extracted'),
198
- links: z.array(linkEntrySchema).describe('Array of extracted links'),
199
- filtered: z
200
- .number()
201
- .optional()
202
- .describe('Number of links filtered out by pattern'),
203
- truncated: z
204
- .boolean()
205
- .optional()
206
- .describe('Whether results were truncated by maxLinks'),
207
- error: z
208
- .string()
209
- .optional()
210
- .describe('Error message if the request failed'),
211
- errorCode: z
212
- .string()
213
- .optional()
214
- .describe('Error code if the request failed'),
215
93
  })
94
+ .merge(resourceFieldsSchema)
216
95
  .strict();
217
96
  export const fetchMarkdownOutputSchema = z
218
97
  .object({
@@ -225,48 +104,9 @@ export const fetchMarkdownOutputSchema = z
225
104
  .string()
226
105
  .optional()
227
106
  .describe('The extracted content in Markdown format'),
228
- contentSize: z.number().optional().describe('Content length in characters'),
229
- resourceUri: z
230
- .string()
107
+ file: fileDownloadSchema
231
108
  .optional()
232
- .describe('Resource URI when content is too large to inline'),
233
- resourceMimeType: z
234
- .string()
235
- .optional()
236
- .describe('MIME type for the resource URI'),
237
- cached: z.boolean().describe('Whether the result was served from cache'),
238
- truncated: z
239
- .boolean()
240
- .optional()
241
- .describe('Whether content was truncated by maxContentLength'),
242
- error: z
243
- .string()
244
- .optional()
245
- .describe('Error message if the request failed'),
246
- errorCode: z
247
- .string()
248
- .optional()
249
- .describe('Error code if the request failed'),
250
- })
251
- .strict();
252
- export const fetchUrlsOutputSchema = z
253
- .object({
254
- results: z
255
- .array(batchResultSchema)
256
- .describe('Array of results for each URL'),
257
- summary: z
258
- .object({
259
- total: z.number().describe('Total URLs processed'),
260
- successful: z.number().describe('Number of successful fetches'),
261
- failed: z.number().describe('Number of failed fetches'),
262
- cached: z.number().describe('Number served from cache'),
263
- totalContentBlocks: z
264
- .number()
265
- .describe('Total content blocks extracted'),
266
- })
267
- .strict()
268
- .describe('Summary statistics'),
269
- fetchedAt: z.string().describe('ISO timestamp of batch completion'),
109
+ .describe('Download information when content is cached'),
270
110
  })
111
+ .merge(resourceFieldsSchema)
271
112
  .strict();
272
- //# sourceMappingURL=schemas.js.map