@noodleseed/one 0.169.0 → 0.171.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -0
- package/dist/cli.js.map +1 -1
- package/dist/commands/capabilities.d.ts +19 -0
- package/dist/commands/capabilities.d.ts.map +1 -0
- package/dist/commands/capabilities.js +128 -0
- package/dist/commands/capabilities.js.map +1 -0
- package/dist/commands/catalog.d.ts.map +1 -1
- package/dist/commands/catalog.js +2 -1
- package/dist/commands/catalog.js.map +1 -1
- package/dist/commands/whatsapp-ops.d.ts +13 -0
- package/dist/commands/whatsapp-ops.d.ts.map +1 -0
- package/dist/commands/whatsapp-ops.js +244 -0
- package/dist/commands/whatsapp-ops.js.map +1 -0
- package/dist/commands/whatsapp-output.d.ts +3 -0
- package/dist/commands/whatsapp-output.d.ts.map +1 -0
- package/dist/commands/whatsapp-output.js +70 -0
- package/dist/commands/whatsapp-output.js.map +1 -0
- package/dist/dev.js +2 -0
- package/dist/dev.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/plugin-mode/plugin-operation-contract.d.ts +3 -3
- package/node_modules/@noodle-borg/agent-kit/dist/curated/command-groups.js +8 -0
- package/node_modules/@noodle-borg/agent-kit/dist/curated/error-fixes.js +1 -0
- package/node_modules/@noodle-borg/agent-kit/dist/generated/example-files.js +2 -2
- package/node_modules/@noodle-borg/agent-kit/dist/generated/surface.js +5 -0
- package/node_modules/@noodle-borg/agent-kit/dist/skill-authoring-refs.js +3 -2
- package/node_modules/@noodle-borg/agent-kit/dist/skill-embedded-assistant-ref.js +6 -0
- package/node_modules/@noodle-borg/agent-kit/package.json +1 -1
- package/node_modules/@noodle-borg/app-audit/dist/audit-embedded-assistant.js +2 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/assistant-configuration.js +2 -1
- package/node_modules/@noodle-borg/assistant-gateway/dist/assistant-interactive.d.ts +1 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/assistant-interactive.js +4 -1
- package/node_modules/@noodle-borg/assistant-gateway/dist/assistant-model-context.js +9 -4
- package/node_modules/@noodle-borg/assistant-gateway/dist/assistant-turn-context.d.ts +16 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/assistant-turn-context.js +19 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-coordinator.d.ts +145 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-coordinator.js +378 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-inbox.js +160 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-provider-blocks.d.ts +18 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-provider-blocks.js +69 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-spend.js +82 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-store.d.ts +42 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-store.js +76 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-types.d.ts +134 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-types.js +72 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-work.d.ts +20 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/channel-work.js +221 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/messaging-tool-dispatch.d.ts +15 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/messaging-tool-dispatch.js +34 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/model-request.d.ts +19 -1
- package/node_modules/@noodle-borg/assistant-gateway/dist/model-request.js +22 -6
- package/node_modules/@noodle-borg/assistant-gateway/dist/portable.d.ts +7 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/portable.js +7 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-assistant-appearance-store.js +83 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-assistant-elevation.js +151 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-assistant-interaction-writes.js +45 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-assistant-interactions.js +340 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-assistant.js +439 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-channel-store.js +102 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-continuity-store.js +159 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-elevation-store.js +110 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres-embed-store.js +134 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/postgres.js +11 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/public-surface.d.ts +8 -0
- package/node_modules/@noodle-borg/assistant-gateway/dist/public-surface.js +23 -0
- package/node_modules/@noodle-borg/authoring/dist/assistant.d.ts +19 -4
- package/node_modules/@noodle-borg/authoring/dist/assistant.js +51 -20
- package/node_modules/@noodle-borg/authoring/dist/index.d.ts +2 -1
- package/node_modules/@noodle-borg/authoring/dist/index.js +2 -1
- package/node_modules/@noodle-borg/authoring/dist/load-authored.js +1 -1
- package/node_modules/@noodle-borg/authoring/dist/recording.d.ts +2 -1
- package/node_modules/@noodle-borg/authoring/dist/recording.js +29 -1
- package/node_modules/@noodle-borg/authoring/dist/server.d.ts +3 -0
- package/node_modules/@noodle-borg/authoring/dist/server.js +5 -1
- package/node_modules/@noodle-borg/authoring/dist/web-capabilities.d.ts +17 -0
- package/node_modules/@noodle-borg/authoring/dist/web-capabilities.js +15 -0
- package/node_modules/@noodle-borg/authoring/package.json +1 -0
- package/node_modules/@noodle-borg/capabilities/dist/index.js +7 -0
- package/node_modules/@noodle-borg/cli-catalog/dist/catalog-data-channels.js +121 -0
- package/node_modules/@noodle-borg/cli-catalog/dist/catalog-data-hosted-observability.js +102 -0
- package/node_modules/@noodle-borg/cli-catalog/dist/index.js +1 -0
- package/node_modules/@noodle-borg/compiler/dist/artifact/assemble.js +3 -0
- package/node_modules/@noodle-borg/compiler/dist/artifact/config.js +1 -1
- package/node_modules/@noodle-borg/compiler/dist/artifact/types.d.ts +2 -0
- package/node_modules/@noodle-borg/compiler/dist/artifact/version.d.ts +2 -1
- package/node_modules/@noodle-borg/compiler/dist/artifact/version.js +2 -1
- package/node_modules/@noodle-borg/compiler/dist/compile.js +2 -0
- package/node_modules/@noodle-borg/compiler/dist/errors.d.ts +1 -1
- package/node_modules/@noodle-borg/compiler/dist/fulfilment-emit.js +18 -3
- package/node_modules/@noodle-borg/compiler/dist/manifest/schema.d.ts +106 -12
- package/node_modules/@noodle-borg/compiler/dist/manifest/schema.js +60 -29
- package/node_modules/@noodle-borg/compiler/dist/manifest/website-projection.d.ts +1 -0
- package/node_modules/@noodle-borg/compiler/dist/manifest/website-projection.js +19 -0
- package/node_modules/@noodle-borg/compiler/dist/web-capabilities.js +55 -0
- package/node_modules/@noodle-borg/compiler/package.json +1 -0
- package/node_modules/@noodle-borg/connector-http/dist/ssrf.js +5 -0
- package/node_modules/@noodle-borg/knowledge-crawl/dist/bounded-body.js +38 -0
- package/node_modules/@noodle-borg/knowledge-crawl/dist/first-party.js +2 -3
- package/node_modules/@noodle-borg/knowledge-crawl/dist/html-text.js +10 -2
- package/node_modules/@noodle-borg/knowledge-crawl/dist/index.d.ts +1 -0
- package/node_modules/@noodle-borg/knowledge-crawl/dist/index.js +1 -0
- package/node_modules/@noodle-borg/knowledge-crawl/dist/page-reader.d.ts +26 -0
- package/node_modules/@noodle-borg/knowledge-crawl/dist/page-reader.js +127 -0
- package/node_modules/@noodle-borg/knowledge-crawl/dist/robots.js +48 -23
- package/node_modules/@noodle-borg/knowledge-crawl/package.json +1 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/budget.d.ts +29 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/budget.js +124 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/contracts.d.ts +212 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/contracts.js +113 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/errors.d.ts +8 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/errors.js +21 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/executor.d.ts +26 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/executor.js +156 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/limits.d.ts +15 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/limits.js +15 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/operator-contracts.d.ts +159 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/operator-contracts.js +69 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/operator-route.d.ts +5 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/operator-route.js +160 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/policy-store.d.ts +34 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/policy-store.js +61 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/portable.d.ts +10 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/portable.js +10 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/service.d.ts +83 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/service.js +139 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/urls.d.ts +3 -0
- package/node_modules/@noodle-borg/managed-capabilities/dist/urls.js +30 -0
- package/node_modules/@noodle-borg/managed-capabilities/package.json +49 -0
- package/node_modules/@noodle-borg/module/dist/contract.d.ts +11 -0
- package/node_modules/@noodle-borg/module-audit/dist/index.js +37 -0
- package/node_modules/@noodle-borg/module-audit/dist/postgres-audit.js +126 -0
- package/node_modules/@noodle-borg/module-audit/dist/route.js +66 -0
- package/node_modules/@noodle-borg/runtime/dist/confirmation.js +29 -0
- package/node_modules/@noodle-borg/runtime/dist/connector/types.d.ts +2 -0
- package/node_modules/@noodle-borg/runtime/dist/execute.d.ts +3 -0
- package/node_modules/@noodle-borg/runtime/dist/execute.js +6 -1
- package/node_modules/@noodle-borg/runtime/dist/index.d.ts +2 -1
- package/node_modules/@noodle-borg/runtime/dist/index.js +2 -1
- package/node_modules/@noodle-borg/runtime/dist/interactive.js +10 -1
- package/node_modules/@noodle-borg/runtime/dist/managed-origins.js +8 -4
- package/node_modules/@noodle-borg/runtime/dist/operation-execution.js +3 -0
- package/node_modules/@noodle-borg/runtime/dist/postgres-state-handles.js +311 -0
- package/node_modules/@noodle-borg/runtime/dist/postgres.js +4 -0
- package/node_modules/@noodle-borg/runtime/dist/result-meta.d.ts +2 -0
- package/node_modules/@noodle-borg/runtime/dist/result-meta.js +6 -0
- package/node_modules/@noodle-borg/runtime/dist/result.d.ts +4 -0
- package/node_modules/@noodle-borg/runtime/dist/web-capabilities.d.ts +71 -0
- package/node_modules/@noodle-borg/runtime/dist/web-capabilities.js +41 -0
- package/node_modules/@noodle-borg/runtime/package.json +1 -0
- package/node_modules/@noodle-borg/service/dist/assistant-composition.js +18 -0
- package/node_modules/@noodle-borg/service/dist/build-info.js +5 -2
- package/node_modules/@noodle-borg/service/dist/business-information/source-ingestion-coordinator.js +7 -0
- package/node_modules/@noodle-borg/service/dist/channels/360dialog.js +233 -0
- package/node_modules/@noodle-borg/service/dist/channels/cipher.js +37 -0
- package/node_modules/@noodle-borg/service/dist/channels/composition.js +37 -0
- package/node_modules/@noodle-borg/service/dist/channels/inference-guard.js +53 -0
- package/node_modules/@noodle-borg/service/dist/channels/runtime.js +337 -0
- package/node_modules/@noodle-borg/service/dist/channels/worker-loop.js +50 -0
- package/node_modules/@noodle-borg/service/dist/local-options.d.ts +1 -0
- package/node_modules/@noodle-borg/service/dist/local-web-capabilities.js +13 -0
- package/node_modules/@noodle-borg/service/dist/mcp-protocol-runtime.js +21 -0
- package/node_modules/@noodle-borg/service/dist/modules/route-dispatch.js +13 -0
- package/node_modules/@noodle-borg/service/dist/registry-compile.js +3 -0
- package/node_modules/@noodle-borg/service/dist/registry-types.d.ts +1 -0
- package/node_modules/@noodle-borg/service/dist/registry.js +3 -0
- package/node_modules/@noodle-borg/service/dist/routes/assistant-agent.js +66 -35
- package/node_modules/@noodle-borg/service/dist/routes/assistant-interactions.js +7 -4
- package/node_modules/@noodle-borg/service/dist/routes/assistant-model-binding.js +6 -2
- package/node_modules/@noodle-borg/service/dist/routes/business-information-channels.js +14 -1
- package/node_modules/@noodle-borg/service/dist/routes/whatsapp.js +238 -0
- package/node_modules/@noodle-borg/service/dist/serve-local.js +14 -0
- package/node_modules/@noodle-borg/service/dist/serve-resource-auth.js +10 -0
- package/node_modules/@noodle-borg/service/dist/service.js +37 -33
- package/node_modules/@noodle-borg/service/dist/web-capabilities.js +39 -0
- package/node_modules/@noodle-borg/service/package.json +3 -1
- package/node_modules/@noodle-borg/wire-contracts/dist/application-channels.d.ts +33 -0
- package/node_modules/@noodle-borg/wire-contracts/dist/application-channels.js +9 -0
- package/node_modules/@noodle-borg/wire-contracts/dist/index.d.ts +2 -0
- package/node_modules/@noodle-borg/wire-contracts/dist/index.js +2 -0
- package/node_modules/@noodle-borg/wire-contracts/dist/service-info.d.ts +2 -0
- package/node_modules/@noodle-borg/wire-contracts/dist/service-info.js +10 -2
- package/node_modules/@noodle-borg/wire-contracts/dist/whatsapp.d.ts +426 -0
- package/node_modules/@noodle-borg/wire-contracts/dist/whatsapp.js +213 -0
- package/node_modules/@noodle-borg/wire-contracts/package.json +1 -0
- package/package.json +3 -2
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { guardedFetch } from '@noodle-borg/connector-http';
|
|
2
|
+
import { CapabilityError, publicPageUrl, WEB_EXTRACT_LIMITS, } from '@noodle-borg/managed-capabilities';
|
|
3
|
+
import { readBoundedBody } from './bounded-body.js';
|
|
4
|
+
import { extractHtmlText, extractPageContent } from './html-text.js';
|
|
5
|
+
import { parseRobots } from './robots.js';
|
|
6
|
+
const PRODUCT_TOKEN = 'NoodleSeedBot';
|
|
7
|
+
const HEADERS = {
|
|
8
|
+
'user-agent': `${PRODUCT_TOKEN}/1.0 (bounded public-page extraction)`,
|
|
9
|
+
accept: 'text/html,text/plain;q=0.9',
|
|
10
|
+
'accept-encoding': 'identity',
|
|
11
|
+
};
|
|
12
|
+
/** Single-page acquisition, not a crawler. All transport collaborators are deployment-owned. */
|
|
13
|
+
export class PublicPageReader {
|
|
14
|
+
#fetch;
|
|
15
|
+
constructor(options = {}) {
|
|
16
|
+
this.#fetch = options.fetchImpl ?? ((url, init) => guardedFetch(new URL(String(url)), init));
|
|
17
|
+
}
|
|
18
|
+
async read(request) {
|
|
19
|
+
const signal = AbortSignal.any([
|
|
20
|
+
request.signal,
|
|
21
|
+
AbortSignal.timeout(WEB_EXTRACT_LIMITS.timeoutMs),
|
|
22
|
+
]);
|
|
23
|
+
let url = publicPageUrl(request.url, request.domains);
|
|
24
|
+
const checked = new Map();
|
|
25
|
+
try {
|
|
26
|
+
for (let redirects = 0; redirects <= WEB_EXTRACT_LIMITS.maxRedirects; redirects += 1) {
|
|
27
|
+
signal.throwIfAborted();
|
|
28
|
+
let robots = checked.get(url.origin);
|
|
29
|
+
if (robots === undefined) {
|
|
30
|
+
robots = await this.#robots(url, request, signal);
|
|
31
|
+
checked.set(url.origin, robots);
|
|
32
|
+
}
|
|
33
|
+
if (!robots.isAllowed(url.pathname))
|
|
34
|
+
throw new CapabilityError('capability_source_rejected');
|
|
35
|
+
const response = await this.#request(url, request, signal);
|
|
36
|
+
try {
|
|
37
|
+
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
38
|
+
const location = response.headers.get('location');
|
|
39
|
+
if (location === null || redirects === WEB_EXTRACT_LIMITS.maxRedirects) {
|
|
40
|
+
throw new CapabilityError('capability_source_rejected');
|
|
41
|
+
}
|
|
42
|
+
url = publicPageUrl(new URL(location, url).href, request.domains);
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (response.status !== 200)
|
|
46
|
+
throw new CapabilityError('capability_source_rejected');
|
|
47
|
+
const mime = response.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase();
|
|
48
|
+
if (mime !== 'text/html')
|
|
49
|
+
throw new CapabilityError('capability_source_rejected');
|
|
50
|
+
const bytes = await readBoundedBody(response, Math.min(request.maxBytes, WEB_EXTRACT_LIMITS.maxPageBytes), signal);
|
|
51
|
+
signal.throwIfAborted();
|
|
52
|
+
const content = extractPageContent(mime, bytes);
|
|
53
|
+
if (content === undefined || content.text === '')
|
|
54
|
+
throw new CapabilityError('capability_source_rejected');
|
|
55
|
+
return {
|
|
56
|
+
url: url.href,
|
|
57
|
+
title: content.title.slice(0, 512),
|
|
58
|
+
text: content.text,
|
|
59
|
+
links: pageLinks(content.html, url, request.domains),
|
|
60
|
+
retrievedAt: new Date().toISOString(),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
await response.body?.cancel().catch(() => { });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (signal.aborted)
|
|
70
|
+
throw new CapabilityError('capability_cancelled');
|
|
71
|
+
if (error instanceof CapabilityError)
|
|
72
|
+
throw error;
|
|
73
|
+
throw new CapabilityError('capability_source_rejected');
|
|
74
|
+
}
|
|
75
|
+
throw new CapabilityError('capability_source_rejected');
|
|
76
|
+
}
|
|
77
|
+
async #request(url, request, signal) {
|
|
78
|
+
signal.throwIfAborted();
|
|
79
|
+
request.beforeRequest();
|
|
80
|
+
return this.#fetch(url.href, {
|
|
81
|
+
method: 'GET',
|
|
82
|
+
redirect: 'manual',
|
|
83
|
+
credentials: 'omit',
|
|
84
|
+
headers: HEADERS,
|
|
85
|
+
signal,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
async #robots(url, request, signal) {
|
|
89
|
+
const response = await this.#request(new URL('/robots.txt', url), request, signal);
|
|
90
|
+
try {
|
|
91
|
+
if (response.status === 404 || response.status === 410)
|
|
92
|
+
return parseRobots('', PRODUCT_TOKEN);
|
|
93
|
+
if (response.status !== 200)
|
|
94
|
+
throw new CapabilityError('capability_source_rejected');
|
|
95
|
+
const bytes = await readBoundedBody(response, WEB_EXTRACT_LIMITS.maxRobotsBytes, signal);
|
|
96
|
+
const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
97
|
+
if (text.trimStart().startsWith('<') || text.includes('\0'))
|
|
98
|
+
throw new CapabilityError('capability_source_rejected');
|
|
99
|
+
return parseRobots(text, PRODUCT_TOKEN);
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
await response.body?.cancel().catch(() => { });
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function pageLinks(html, base, domains) {
|
|
107
|
+
const links = [];
|
|
108
|
+
const seen = new Set();
|
|
109
|
+
// Bounded HTML has no evaluator. Link labels and destinations are untrusted and are not fetched here.
|
|
110
|
+
for (const match of html.matchAll(/<a\b[^>]*?href\s*=\s*["']([^"']{1,2048})["'][^>]*>([\s\S]*?)<\/a\s*>/gi)) {
|
|
111
|
+
try {
|
|
112
|
+
const url = publicPageUrl(new URL(match[1] ?? '', base).href, domains);
|
|
113
|
+
if (url.origin !== base.origin || seen.has(url.href))
|
|
114
|
+
continue;
|
|
115
|
+
seen.add(url.href);
|
|
116
|
+
const label = extractHtmlText(match[2] ?? '').slice(0, 200);
|
|
117
|
+
links.push({ url: url.href, ...(label === '' ? {} : { label }) });
|
|
118
|
+
if (links.length === WEB_EXTRACT_LIMITS.maxLinks)
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
/* Unsafe or out-of-policy links do not become candidate sources. */
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return links;
|
|
126
|
+
}
|
|
127
|
+
//# sourceMappingURL=page-reader.js.map
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* needed, verify the dependency license first (root rules).
|
|
2
|
+
* Bounded robots groups and longest-match rules. An explicit product token selects its exact
|
|
3
|
+
* groups before wildcard groups; omitted token retains the crawler's wildcard-group behavior.
|
|
4
|
+
* Supports percent-normalized paths, wildcard and end-anchor matching without a backtracking regex.
|
|
6
5
|
*/
|
|
7
|
-
export function parseRobots(text) {
|
|
8
|
-
const
|
|
6
|
+
export function parseRobots(text, productToken = '*') {
|
|
7
|
+
const groups = [];
|
|
9
8
|
const sitemaps = [];
|
|
10
|
-
let
|
|
11
|
-
|
|
9
|
+
let group = { agents: [], rules: [] };
|
|
10
|
+
groups.push(group);
|
|
11
|
+
let ruleCount = 0;
|
|
12
12
|
for (const rawLine of text.split(/\r?\n/)) {
|
|
13
13
|
const line = rawLine.replace(/#.*$/, '').trim();
|
|
14
14
|
if (line === '')
|
|
@@ -24,32 +24,31 @@ export function parseRobots(text) {
|
|
|
24
24
|
continue;
|
|
25
25
|
}
|
|
26
26
|
if (field === 'user-agent') {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
inStarGroup = false;
|
|
31
|
-
groupHasAgents = false;
|
|
27
|
+
if (group.rules.length > 0) {
|
|
28
|
+
group = { agents: [], rules: [] };
|
|
29
|
+
groups.push(group);
|
|
32
30
|
}
|
|
33
|
-
|
|
34
|
-
inStarGroup = true;
|
|
31
|
+
group.agents.push(value.toLowerCase());
|
|
35
32
|
continue;
|
|
36
33
|
}
|
|
37
34
|
if (field === 'allow' || field === 'disallow') {
|
|
38
|
-
|
|
39
|
-
if (!
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
continue;
|
|
44
|
-
rules.push({ allow: field === 'allow', path: value });
|
|
35
|
+
ruleCount += 1;
|
|
36
|
+
if (ruleCount > 4096 || value.length > 2048 || (value !== '' && !value.startsWith('/'))) {
|
|
37
|
+
throw new Error('robots_policy_invalid');
|
|
38
|
+
}
|
|
39
|
+
group.rules.push({ allow: field === 'allow', path: normalizePath(value) });
|
|
45
40
|
}
|
|
46
41
|
}
|
|
42
|
+
const exact = groups.filter((entry) => entry.agents.includes(productToken.toLowerCase()));
|
|
43
|
+
const selected = exact.length > 0 ? exact : groups.filter((entry) => entry.agents.includes('*'));
|
|
44
|
+
const rules = selected.flatMap((entry) => entry.rules).filter((rule) => rule.path !== '');
|
|
47
45
|
return {
|
|
48
46
|
sitemaps,
|
|
49
47
|
isAllowed(path) {
|
|
48
|
+
const normalized = normalizePath(path);
|
|
50
49
|
let best;
|
|
51
50
|
for (const rule of rules) {
|
|
52
|
-
if (!
|
|
51
|
+
if (!matches(normalized, rule.path))
|
|
53
52
|
continue;
|
|
54
53
|
if (best === undefined ||
|
|
55
54
|
rule.path.length > best.path.length ||
|
|
@@ -61,4 +60,30 @@ export function parseRobots(text) {
|
|
|
61
60
|
},
|
|
62
61
|
};
|
|
63
62
|
}
|
|
63
|
+
function normalizePath(path) {
|
|
64
|
+
return path.replace(/%[0-9a-f]{2}|\P{ASCII}/giu, (part) => {
|
|
65
|
+
if (!part.startsWith('%'))
|
|
66
|
+
return encodeURIComponent(part);
|
|
67
|
+
const decoded = String.fromCharCode(Number.parseInt(part.slice(1), 16));
|
|
68
|
+
return /^[a-z0-9._~-]$/i.test(decoded) ? decoded : part.toUpperCase();
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
function matches(path, pattern) {
|
|
72
|
+
const anchored = pattern.endsWith('$');
|
|
73
|
+
const parts = (anchored ? pattern.slice(0, -1) : pattern).split('*');
|
|
74
|
+
const first = parts[0] ?? '';
|
|
75
|
+
if (!path.startsWith(first))
|
|
76
|
+
return false;
|
|
77
|
+
let position = first.length;
|
|
78
|
+
for (let i = 1; i < parts.length; i += 1) {
|
|
79
|
+
const part = parts[i] ?? '';
|
|
80
|
+
if (anchored && i === parts.length - 1)
|
|
81
|
+
return path.endsWith(part) && path.length - part.length >= position;
|
|
82
|
+
const next = path.indexOf(part, position);
|
|
83
|
+
if (next === -1)
|
|
84
|
+
return false;
|
|
85
|
+
position = next + part.length;
|
|
86
|
+
}
|
|
87
|
+
return !anchored || position === path.length;
|
|
88
|
+
}
|
|
64
89
|
//# sourceMappingURL=robots.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { WebPolicy } from './contracts.js';
|
|
3
|
+
declare const snapshotSchema: z.ZodObject<{
|
|
4
|
+
startedAt: z.ZodNullable<z.ZodNumber>;
|
|
5
|
+
timeout: z.ZodNumber;
|
|
6
|
+
maxCalls: z.ZodNumber;
|
|
7
|
+
maxUrls: z.ZodNumber;
|
|
8
|
+
calls: z.ZodNumber;
|
|
9
|
+
attempts: z.ZodNumber;
|
|
10
|
+
bytes: z.ZodNumber;
|
|
11
|
+
urls: z.ZodArray<z.ZodString>;
|
|
12
|
+
}, z.core.$strict>;
|
|
13
|
+
export type CapabilityBudgetSnapshot = z.infer<typeof snapshotSchema>;
|
|
14
|
+
/** Trusted turn/invocation state. Never reconstructed from client arguments or stored in an artifact. */
|
|
15
|
+
export declare class CapabilityBudget {
|
|
16
|
+
#private;
|
|
17
|
+
constructor(snapshot?: CapabilityBudgetSnapshot);
|
|
18
|
+
/** Only for server-held continuations; no source URLs or content are retained. */
|
|
19
|
+
snapshot(): CapabilityBudgetSnapshot;
|
|
20
|
+
reserve(urls: readonly string[], policy: WebPolicy): AbortSignal;
|
|
21
|
+
beforeRequest(): void;
|
|
22
|
+
text(value: string, maxBytes: number): {
|
|
23
|
+
text: string;
|
|
24
|
+
truncated: boolean;
|
|
25
|
+
};
|
|
26
|
+
withSlot<T>(signal: AbortSignal, run: () => Promise<T>): Promise<T>;
|
|
27
|
+
}
|
|
28
|
+
export {};
|
|
29
|
+
//# sourceMappingURL=budget.d.ts.map
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { CapabilityError } from './errors.js';
|
|
4
|
+
import { WEB_EXTRACT_LIMITS as limits } from './limits.js';
|
|
5
|
+
const snapshotSchema = z
|
|
6
|
+
.object({
|
|
7
|
+
startedAt: z.number().int().nonnegative().nullable(),
|
|
8
|
+
timeout: z.number().int().min(100).max(limits.timeoutMs),
|
|
9
|
+
maxCalls: z.number().int().min(1).max(limits.maxCalls),
|
|
10
|
+
maxUrls: z.number().int().min(1).max(limits.maxUrls),
|
|
11
|
+
calls: z.number().int().min(0).max(limits.maxCalls),
|
|
12
|
+
attempts: z.number().int().min(0).max(limits.maxHttpAttempts),
|
|
13
|
+
bytes: z.number().int().min(0).max(limits.maxTurnTextBytes),
|
|
14
|
+
urls: z.array(z.string().regex(/^[a-f0-9]{64}$/)).max(limits.maxUrls),
|
|
15
|
+
})
|
|
16
|
+
.strict();
|
|
17
|
+
const urlKey = (url) => createHash('sha256').update(url).digest('hex');
|
|
18
|
+
/** Trusted turn/invocation state. Never reconstructed from client arguments or stored in an artifact. */
|
|
19
|
+
export class CapabilityBudget {
|
|
20
|
+
#startedAt;
|
|
21
|
+
#timeout = limits.timeoutMs;
|
|
22
|
+
#maxCalls = limits.maxCalls;
|
|
23
|
+
#maxUrls = limits.maxUrls;
|
|
24
|
+
#calls = 0;
|
|
25
|
+
#attempts = 0;
|
|
26
|
+
#bytes = 0;
|
|
27
|
+
#active = 0;
|
|
28
|
+
#urls = new Set();
|
|
29
|
+
#waiters = new Set();
|
|
30
|
+
constructor(snapshot) {
|
|
31
|
+
if (snapshot === undefined)
|
|
32
|
+
return;
|
|
33
|
+
const parsed = snapshotSchema.parse(snapshot);
|
|
34
|
+
this.#startedAt = parsed.startedAt ?? undefined;
|
|
35
|
+
this.#timeout = parsed.timeout;
|
|
36
|
+
this.#maxCalls = parsed.maxCalls;
|
|
37
|
+
this.#maxUrls = parsed.maxUrls;
|
|
38
|
+
this.#calls = parsed.calls;
|
|
39
|
+
this.#attempts = parsed.attempts;
|
|
40
|
+
this.#bytes = parsed.bytes;
|
|
41
|
+
for (const key of parsed.urls)
|
|
42
|
+
this.#urls.add(key);
|
|
43
|
+
}
|
|
44
|
+
/** Only for server-held continuations; no source URLs or content are retained. */
|
|
45
|
+
snapshot() {
|
|
46
|
+
return {
|
|
47
|
+
startedAt: this.#startedAt ?? null,
|
|
48
|
+
timeout: this.#timeout,
|
|
49
|
+
maxCalls: this.#maxCalls,
|
|
50
|
+
maxUrls: this.#maxUrls,
|
|
51
|
+
calls: this.#calls,
|
|
52
|
+
attempts: this.#attempts,
|
|
53
|
+
bytes: this.#bytes,
|
|
54
|
+
urls: [...this.#urls],
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
reserve(urls, policy) {
|
|
58
|
+
this.#startedAt ??= Date.now();
|
|
59
|
+
this.#timeout = Math.min(this.#timeout, policy.timeoutMs ?? limits.timeoutMs);
|
|
60
|
+
this.#maxCalls = Math.min(this.#maxCalls, policy.maxCalls ?? limits.maxCalls);
|
|
61
|
+
this.#maxUrls = Math.min(this.#maxUrls, policy.maxUrls ?? limits.maxUrls);
|
|
62
|
+
const next = new Set([...this.#urls, ...urls.map(urlKey)]);
|
|
63
|
+
if (this.#calls >= this.#maxCalls || next.size > this.#maxUrls)
|
|
64
|
+
this.#exhausted();
|
|
65
|
+
this.#checkTime();
|
|
66
|
+
this.#calls += 1;
|
|
67
|
+
for (const url of urls)
|
|
68
|
+
this.#urls.add(urlKey(url));
|
|
69
|
+
return AbortSignal.timeout(Math.max(1, this.#startedAt + this.#timeout - Date.now()));
|
|
70
|
+
}
|
|
71
|
+
beforeRequest() {
|
|
72
|
+
this.#checkTime();
|
|
73
|
+
if (this.#attempts >= limits.maxHttpAttempts)
|
|
74
|
+
this.#exhausted();
|
|
75
|
+
this.#attempts += 1;
|
|
76
|
+
}
|
|
77
|
+
text(value, maxBytes) {
|
|
78
|
+
const bytes = Buffer.from(value, 'utf8');
|
|
79
|
+
let length = Math.min(bytes.length, maxBytes, limits.maxTurnTextBytes - this.#bytes);
|
|
80
|
+
// Drop any incomplete final code point rather than introducing a replacement character.
|
|
81
|
+
if (length < bytes.length)
|
|
82
|
+
while (length > 0 && ((bytes[length] ?? 0) & 0xc0) === 0x80)
|
|
83
|
+
length -= 1;
|
|
84
|
+
if (length === 0)
|
|
85
|
+
this.#exhausted();
|
|
86
|
+
this.#bytes += length;
|
|
87
|
+
return { text: bytes.subarray(0, length).toString('utf8'), truncated: length < bytes.length };
|
|
88
|
+
}
|
|
89
|
+
async withSlot(signal, run) {
|
|
90
|
+
while (this.#active >= limits.maxConcurrency) {
|
|
91
|
+
signal.throwIfAborted();
|
|
92
|
+
await new Promise((resolve) => {
|
|
93
|
+
const wake = () => {
|
|
94
|
+
this.#waiters.delete(wake);
|
|
95
|
+
signal.removeEventListener('abort', wake);
|
|
96
|
+
resolve();
|
|
97
|
+
};
|
|
98
|
+
this.#waiters.add(wake);
|
|
99
|
+
signal.addEventListener('abort', wake, { once: true });
|
|
100
|
+
if (signal.aborted)
|
|
101
|
+
wake();
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
signal.throwIfAborted();
|
|
105
|
+
this.#checkTime();
|
|
106
|
+
this.#active += 1;
|
|
107
|
+
try {
|
|
108
|
+
return await run();
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
this.#active -= 1;
|
|
112
|
+
for (const wake of [...this.#waiters])
|
|
113
|
+
wake();
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
#checkTime() {
|
|
117
|
+
if (this.#startedAt !== undefined && Date.now() >= this.#startedAt + this.#timeout)
|
|
118
|
+
this.#exhausted();
|
|
119
|
+
}
|
|
120
|
+
#exhausted() {
|
|
121
|
+
throw new CapabilityError('capability_budget_exhausted');
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
//# sourceMappingURL=budget.js.map
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const capabilityNameSchema: z.ZodString;
|
|
3
|
+
export declare const webPolicySchema: z.ZodObject<{
|
|
4
|
+
maxUrls: z.ZodOptional<z.ZodNumber>;
|
|
5
|
+
maxCalls: z.ZodOptional<z.ZodNumber>;
|
|
6
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
7
|
+
maxTextBytes: z.ZodOptional<z.ZodNumber>;
|
|
8
|
+
domains: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
9
|
+
}, z.core.$strict>;
|
|
10
|
+
export type WebPolicy = z.infer<typeof webPolicySchema>;
|
|
11
|
+
export declare const webCapabilityBaseSchema: z.ZodObject<{
|
|
12
|
+
name: z.ZodString;
|
|
13
|
+
class: z.ZodLiteral<"web.extract.v1">;
|
|
14
|
+
title: z.ZodString;
|
|
15
|
+
description: z.ZodString;
|
|
16
|
+
provider: z.ZodObject<{
|
|
17
|
+
kind: z.ZodLiteral<"noodle-managed">;
|
|
18
|
+
}, z.core.$strict>;
|
|
19
|
+
policy: z.ZodOptional<z.ZodObject<{
|
|
20
|
+
maxUrls: z.ZodOptional<z.ZodNumber>;
|
|
21
|
+
maxCalls: z.ZodOptional<z.ZodNumber>;
|
|
22
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
23
|
+
maxTextBytes: z.ZodOptional<z.ZodNumber>;
|
|
24
|
+
domains: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
25
|
+
}, z.core.$strict>>;
|
|
26
|
+
}, z.core.$strict>;
|
|
27
|
+
export interface CapabilityAuthorization {
|
|
28
|
+
readonly discovery?: 'authorized' | 'public' | undefined;
|
|
29
|
+
readonly requiredScopes?: readonly string[] | undefined;
|
|
30
|
+
readonly allowedRoles?: readonly string[] | undefined;
|
|
31
|
+
}
|
|
32
|
+
export type WebCapability = z.infer<typeof webCapabilityBaseSchema> & {
|
|
33
|
+
readonly authorization?: CapabilityAuthorization | undefined;
|
|
34
|
+
};
|
|
35
|
+
export declare const webExtractRequestSchema: z.ZodObject<{
|
|
36
|
+
urls: z.ZodArray<z.ZodString>;
|
|
37
|
+
domains: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
38
|
+
}, z.core.$strict>;
|
|
39
|
+
export type WebExtractRequest = z.infer<typeof webExtractRequestSchema>;
|
|
40
|
+
export declare const webExtractResultSchema: z.ZodObject<{
|
|
41
|
+
status: z.ZodEnum<{
|
|
42
|
+
complete: "complete";
|
|
43
|
+
partial: "partial";
|
|
44
|
+
}>;
|
|
45
|
+
items: z.ZodArray<z.ZodObject<{
|
|
46
|
+
requestIndex: z.ZodNumber;
|
|
47
|
+
sourceRef: z.ZodString;
|
|
48
|
+
content: z.ZodObject<{
|
|
49
|
+
format: z.ZodLiteral<"text">;
|
|
50
|
+
text: z.ZodString;
|
|
51
|
+
}, z.core.$strict>;
|
|
52
|
+
links: z.ZodArray<z.ZodObject<{
|
|
53
|
+
url: z.ZodString;
|
|
54
|
+
label: z.ZodOptional<z.ZodString>;
|
|
55
|
+
}, z.core.$strict>>;
|
|
56
|
+
truncated: z.ZodBoolean;
|
|
57
|
+
}, z.core.$strict>>;
|
|
58
|
+
sources: z.ZodArray<z.ZodObject<{
|
|
59
|
+
ref: z.ZodString;
|
|
60
|
+
url: z.ZodString;
|
|
61
|
+
title: z.ZodOptional<z.ZodString>;
|
|
62
|
+
retrievedAt: z.ZodISODateTime;
|
|
63
|
+
publishedAt: z.ZodOptional<z.ZodISODateTime>;
|
|
64
|
+
observedAt: z.ZodOptional<z.ZodISODateTime>;
|
|
65
|
+
expiresAt: z.ZodOptional<z.ZodISODateTime>;
|
|
66
|
+
attribution: z.ZodOptional<z.ZodString>;
|
|
67
|
+
rights: z.ZodOptional<z.ZodString>;
|
|
68
|
+
}, z.core.$strict>>;
|
|
69
|
+
warnings: z.ZodArray<z.ZodObject<{
|
|
70
|
+
requestIndex: z.ZodNumber;
|
|
71
|
+
code: z.ZodEnum<{
|
|
72
|
+
capability_unavailable: "capability_unavailable";
|
|
73
|
+
capability_binding_invalid: "capability_binding_invalid";
|
|
74
|
+
capability_requirement_unsupported: "capability_requirement_unsupported";
|
|
75
|
+
capability_policy_denied: "capability_policy_denied";
|
|
76
|
+
capability_budget_exhausted: "capability_budget_exhausted";
|
|
77
|
+
capability_source_rejected: "capability_source_rejected";
|
|
78
|
+
capability_provider_failed: "capability_provider_failed";
|
|
79
|
+
capability_result_invalid: "capability_result_invalid";
|
|
80
|
+
capability_cancelled: "capability_cancelled";
|
|
81
|
+
content_truncated: "content_truncated";
|
|
82
|
+
}>;
|
|
83
|
+
}, z.core.$strict>>;
|
|
84
|
+
}, z.core.$strict>;
|
|
85
|
+
export type WebExtractResult = z.infer<typeof webExtractResultSchema>;
|
|
86
|
+
export declare const WEB_CONNECTOR_ID = "noodle_web";
|
|
87
|
+
export declare const WEB_CONNECTOR_VERSION = "1.0.0";
|
|
88
|
+
export declare const webOperationInputSchema: z.ZodObject<{
|
|
89
|
+
name: z.ZodString;
|
|
90
|
+
request: z.ZodObject<{
|
|
91
|
+
urls: z.ZodArray<z.ZodString>;
|
|
92
|
+
domains: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
93
|
+
}, z.core.$strict>;
|
|
94
|
+
}, z.core.$strict>;
|
|
95
|
+
export declare const WEB_OPERATION_SIGNATURE: {
|
|
96
|
+
type: "read";
|
|
97
|
+
input: z.core.ZodStandardJSONSchemaPayload<z.ZodObject<{
|
|
98
|
+
name: z.ZodString;
|
|
99
|
+
request: z.ZodObject<{
|
|
100
|
+
urls: z.ZodArray<z.ZodString>;
|
|
101
|
+
domains: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
102
|
+
}, z.core.$strict>;
|
|
103
|
+
}, z.core.$strict>>;
|
|
104
|
+
output: z.core.ZodStandardJSONSchemaPayload<z.ZodObject<{
|
|
105
|
+
status: z.ZodEnum<{
|
|
106
|
+
complete: "complete";
|
|
107
|
+
partial: "partial";
|
|
108
|
+
}>;
|
|
109
|
+
items: z.ZodArray<z.ZodObject<{
|
|
110
|
+
requestIndex: z.ZodNumber;
|
|
111
|
+
sourceRef: z.ZodString;
|
|
112
|
+
content: z.ZodObject<{
|
|
113
|
+
format: z.ZodLiteral<"text">;
|
|
114
|
+
text: z.ZodString;
|
|
115
|
+
}, z.core.$strict>;
|
|
116
|
+
links: z.ZodArray<z.ZodObject<{
|
|
117
|
+
url: z.ZodString;
|
|
118
|
+
label: z.ZodOptional<z.ZodString>;
|
|
119
|
+
}, z.core.$strict>>;
|
|
120
|
+
truncated: z.ZodBoolean;
|
|
121
|
+
}, z.core.$strict>>;
|
|
122
|
+
sources: z.ZodArray<z.ZodObject<{
|
|
123
|
+
ref: z.ZodString;
|
|
124
|
+
url: z.ZodString;
|
|
125
|
+
title: z.ZodOptional<z.ZodString>;
|
|
126
|
+
retrievedAt: z.ZodISODateTime;
|
|
127
|
+
publishedAt: z.ZodOptional<z.ZodISODateTime>;
|
|
128
|
+
observedAt: z.ZodOptional<z.ZodISODateTime>;
|
|
129
|
+
expiresAt: z.ZodOptional<z.ZodISODateTime>;
|
|
130
|
+
attribution: z.ZodOptional<z.ZodString>;
|
|
131
|
+
rights: z.ZodOptional<z.ZodString>;
|
|
132
|
+
}, z.core.$strict>>;
|
|
133
|
+
warnings: z.ZodArray<z.ZodObject<{
|
|
134
|
+
requestIndex: z.ZodNumber;
|
|
135
|
+
code: z.ZodEnum<{
|
|
136
|
+
capability_unavailable: "capability_unavailable";
|
|
137
|
+
capability_binding_invalid: "capability_binding_invalid";
|
|
138
|
+
capability_requirement_unsupported: "capability_requirement_unsupported";
|
|
139
|
+
capability_policy_denied: "capability_policy_denied";
|
|
140
|
+
capability_budget_exhausted: "capability_budget_exhausted";
|
|
141
|
+
capability_source_rejected: "capability_source_rejected";
|
|
142
|
+
capability_provider_failed: "capability_provider_failed";
|
|
143
|
+
capability_result_invalid: "capability_result_invalid";
|
|
144
|
+
capability_cancelled: "capability_cancelled";
|
|
145
|
+
content_truncated: "content_truncated";
|
|
146
|
+
}>;
|
|
147
|
+
}, z.core.$strict>>;
|
|
148
|
+
}, z.core.$strict>>;
|
|
149
|
+
};
|
|
150
|
+
export declare const WEB_CATALOG_CONNECTOR: {
|
|
151
|
+
id: string;
|
|
152
|
+
version: string;
|
|
153
|
+
kind: "builtin";
|
|
154
|
+
operations: {
|
|
155
|
+
extract: {
|
|
156
|
+
type: "read";
|
|
157
|
+
input: z.core.ZodStandardJSONSchemaPayload<z.ZodObject<{
|
|
158
|
+
name: z.ZodString;
|
|
159
|
+
request: z.ZodObject<{
|
|
160
|
+
urls: z.ZodArray<z.ZodString>;
|
|
161
|
+
domains: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
162
|
+
}, z.core.$strict>;
|
|
163
|
+
}, z.core.$strict>>;
|
|
164
|
+
output: z.core.ZodStandardJSONSchemaPayload<z.ZodObject<{
|
|
165
|
+
status: z.ZodEnum<{
|
|
166
|
+
complete: "complete";
|
|
167
|
+
partial: "partial";
|
|
168
|
+
}>;
|
|
169
|
+
items: z.ZodArray<z.ZodObject<{
|
|
170
|
+
requestIndex: z.ZodNumber;
|
|
171
|
+
sourceRef: z.ZodString;
|
|
172
|
+
content: z.ZodObject<{
|
|
173
|
+
format: z.ZodLiteral<"text">;
|
|
174
|
+
text: z.ZodString;
|
|
175
|
+
}, z.core.$strict>;
|
|
176
|
+
links: z.ZodArray<z.ZodObject<{
|
|
177
|
+
url: z.ZodString;
|
|
178
|
+
label: z.ZodOptional<z.ZodString>;
|
|
179
|
+
}, z.core.$strict>>;
|
|
180
|
+
truncated: z.ZodBoolean;
|
|
181
|
+
}, z.core.$strict>>;
|
|
182
|
+
sources: z.ZodArray<z.ZodObject<{
|
|
183
|
+
ref: z.ZodString;
|
|
184
|
+
url: z.ZodString;
|
|
185
|
+
title: z.ZodOptional<z.ZodString>;
|
|
186
|
+
retrievedAt: z.ZodISODateTime;
|
|
187
|
+
publishedAt: z.ZodOptional<z.ZodISODateTime>;
|
|
188
|
+
observedAt: z.ZodOptional<z.ZodISODateTime>;
|
|
189
|
+
expiresAt: z.ZodOptional<z.ZodISODateTime>;
|
|
190
|
+
attribution: z.ZodOptional<z.ZodString>;
|
|
191
|
+
rights: z.ZodOptional<z.ZodString>;
|
|
192
|
+
}, z.core.$strict>>;
|
|
193
|
+
warnings: z.ZodArray<z.ZodObject<{
|
|
194
|
+
requestIndex: z.ZodNumber;
|
|
195
|
+
code: z.ZodEnum<{
|
|
196
|
+
capability_unavailable: "capability_unavailable";
|
|
197
|
+
capability_binding_invalid: "capability_binding_invalid";
|
|
198
|
+
capability_requirement_unsupported: "capability_requirement_unsupported";
|
|
199
|
+
capability_policy_denied: "capability_policy_denied";
|
|
200
|
+
capability_budget_exhausted: "capability_budget_exhausted";
|
|
201
|
+
capability_source_rejected: "capability_source_rejected";
|
|
202
|
+
capability_provider_failed: "capability_provider_failed";
|
|
203
|
+
capability_result_invalid: "capability_result_invalid";
|
|
204
|
+
capability_cancelled: "capability_cancelled";
|
|
205
|
+
content_truncated: "content_truncated";
|
|
206
|
+
}>;
|
|
207
|
+
}, z.core.$strict>>;
|
|
208
|
+
}, z.core.$strict>>;
|
|
209
|
+
};
|
|
210
|
+
};
|
|
211
|
+
};
|
|
212
|
+
//# sourceMappingURL=contracts.d.ts.map
|