@fargowealth/fargo-ai-cli 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +32 -43
- package/package.json +1 -5
- package/src/cli.mjs +52 -7
- package/src/constants.mjs +1 -1
- package/src/upload-transfer.mjs +422 -0
- package/node_modules/@fargowealth/website-connector-runtime/README.md +0 -8
- package/node_modules/@fargowealth/website-connector-runtime/package.json +0 -24
- package/node_modules/@fargowealth/website-connector-runtime/src/index.d.mts +0 -116
- package/node_modules/@fargowealth/website-connector-runtime/src/index.mjs +0 -614
- package/src/website-browser-profile.mjs +0 -377
- package/src/website-cli.mjs +0 -364
- package/src/website-connector-definition.d.mts +0 -1
- package/src/website-connector-definition.mjs +0 -1
|
@@ -1,614 +0,0 @@
|
|
|
1
|
-
const CONNECTOR_ID = /^[a-z][a-z0-9-]{1,62}[a-z0-9]$/;
|
|
2
|
-
const OPERATION_ID = /^[a-z][a-z0-9_]{1,62}[a-z0-9]$/;
|
|
3
|
-
const TEMPLATE_ARGUMENT = /\{\{([a-zA-Z][a-zA-Z0-9_]*)\}\}/g;
|
|
4
|
-
const RUNTIME_CAPABILITY = /^[a-z][a-z0-9.-]{1,80}$/;
|
|
5
|
-
const SEMVER = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
6
|
-
const MAX_RESPONSE_BYTES = 1_048_576;
|
|
7
|
-
const FORBIDDEN_HEADERS = new Set([
|
|
8
|
-
"connection",
|
|
9
|
-
"content-length",
|
|
10
|
-
"cookie",
|
|
11
|
-
"host",
|
|
12
|
-
"proxy-authorization",
|
|
13
|
-
"transfer-encoding",
|
|
14
|
-
"upgrade"
|
|
15
|
-
]);
|
|
16
|
-
|
|
17
|
-
export const WEBSITE_CONNECTOR_SCHEMA_VERSION = "fargo.website-connector/v1";
|
|
18
|
-
export const WEBSITE_CONNECTOR_RUNTIME_VERSION = "1.0.0";
|
|
19
|
-
export const WEBSITE_CONNECTOR_RUNTIME_CAPABILITIES = Object.freeze([
|
|
20
|
-
"auth.browser-storage",
|
|
21
|
-
"request.http",
|
|
22
|
-
"request.json-body",
|
|
23
|
-
"request.write",
|
|
24
|
-
"response.auto",
|
|
25
|
-
"response.json",
|
|
26
|
-
"response.text"
|
|
27
|
-
]);
|
|
28
|
-
|
|
29
|
-
export class WebsiteConnectorDefinitionError extends Error {
|
|
30
|
-
constructor(code, path = "") {
|
|
31
|
-
super(path ? `${code}:${path}` : code);
|
|
32
|
-
this.name = "WebsiteConnectorDefinitionError";
|
|
33
|
-
this.code = code;
|
|
34
|
-
this.path = path;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export class WebsiteConnectorRuntimeError extends Error {
|
|
39
|
-
constructor(status, code, details) {
|
|
40
|
-
super(code);
|
|
41
|
-
this.name = "WebsiteConnectorRuntimeError";
|
|
42
|
-
this.status = status;
|
|
43
|
-
this.code = code;
|
|
44
|
-
this.details = details;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export function parseWebsiteConnectorDefinition(value) {
|
|
49
|
-
const source = record(value, "website_connector_definition_invalid", "/");
|
|
50
|
-
equal(source.schemaVersion, WEBSITE_CONNECTOR_SCHEMA_VERSION, "website_connector_schema_version_invalid", "/schemaVersion");
|
|
51
|
-
const connectorId = requiredText(source.connectorId, 64, "website_connector_id_invalid", "/connectorId");
|
|
52
|
-
if (!CONNECTOR_ID.test(connectorId)) fail("website_connector_id_invalid", "/connectorId");
|
|
53
|
-
const displayName = requiredText(source.displayName, 120, "website_connector_display_name_invalid", "/displayName");
|
|
54
|
-
const allowedOrigins = uniqueArray(source.allowedOrigins, 1, 12, "/allowedOrigins").map((origin, index) => (
|
|
55
|
-
exactHttpsOrigin(origin, "website_connector_origin_invalid", `/allowedOrigins/${index}`)
|
|
56
|
-
));
|
|
57
|
-
const loginUrl = optionalHttpsUrl(source.loginUrl, allowedOrigins, "/loginUrl");
|
|
58
|
-
const auth = parseAuthRecipe(source.auth, allowedOrigins);
|
|
59
|
-
const runtime = parseRuntimeRequirements(source.runtime);
|
|
60
|
-
if (auth.kind === "browser_storage" && !loginUrl) fail("website_connector_login_url_required", "/loginUrl");
|
|
61
|
-
const operations = uniqueArray(source.operations, 1, 80, "/operations").map((operation, index) => (
|
|
62
|
-
parseOperation(operation, allowedOrigins, index)
|
|
63
|
-
));
|
|
64
|
-
const operationIds = new Set();
|
|
65
|
-
for (const operation of operations) {
|
|
66
|
-
if (operationIds.has(operation.operationId)) fail("website_operation_id_duplicate", `/operations/${operation.operationId}`);
|
|
67
|
-
if (operation.requiresConnection && auth.kind !== "browser_storage") {
|
|
68
|
-
fail("website_operation_connection_auth_required", `/operations/${operation.operationId}/requiresConnection`);
|
|
69
|
-
}
|
|
70
|
-
operationIds.add(operation.operationId);
|
|
71
|
-
}
|
|
72
|
-
return {
|
|
73
|
-
schemaVersion: WEBSITE_CONNECTOR_SCHEMA_VERSION,
|
|
74
|
-
connectorId,
|
|
75
|
-
displayName,
|
|
76
|
-
allowedOrigins,
|
|
77
|
-
...(loginUrl ? { loginUrl } : {}),
|
|
78
|
-
...(runtime ? { runtime } : {}),
|
|
79
|
-
auth,
|
|
80
|
-
operations
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
export function connectorToolName(connectorId, operationId) {
|
|
85
|
-
const left = String(connectorId || "").replaceAll("-", "_");
|
|
86
|
-
return `${left}_${operationId}`;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
export function renderWebsiteOperation(definition, operationId, rawArguments) {
|
|
90
|
-
const parsed = parseWebsiteConnectorDefinition(definition);
|
|
91
|
-
const operation = parsed.operations.find((candidate) => candidate.operationId === operationId);
|
|
92
|
-
if (!operation) fail("website_operation_not_found", "/operationId");
|
|
93
|
-
const args = record(rawArguments ?? {}, "website_operation_arguments_invalid", "/arguments");
|
|
94
|
-
validateArguments(args, operation.inputSchema);
|
|
95
|
-
const path = renderString(operation.request.path, args, "/request/path");
|
|
96
|
-
const url = new URL(path, `${operation.request.origin}/`);
|
|
97
|
-
if (url.origin !== operation.request.origin) fail("website_operation_origin_escape", "/request/path");
|
|
98
|
-
for (const [name, argumentName] of Object.entries(operation.request.query)) {
|
|
99
|
-
const value = args[argumentName];
|
|
100
|
-
if (value !== undefined && value !== null) url.searchParams.set(name, scalarText(value, `/arguments/${argumentName}`));
|
|
101
|
-
}
|
|
102
|
-
const body = operation.request.body === undefined
|
|
103
|
-
? undefined
|
|
104
|
-
: renderBody(operation.request.body, args, "/request/body");
|
|
105
|
-
return {
|
|
106
|
-
definition: parsed,
|
|
107
|
-
operation,
|
|
108
|
-
request: {
|
|
109
|
-
body,
|
|
110
|
-
headers: { ...operation.request.headers },
|
|
111
|
-
method: operation.request.method,
|
|
112
|
-
responseMode: operation.request.responseMode,
|
|
113
|
-
url: url.toString()
|
|
114
|
-
}
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
export function websiteConnectorRuntimeDescriptor() {
|
|
119
|
-
return {
|
|
120
|
-
capabilities: [...WEBSITE_CONNECTOR_RUNTIME_CAPABILITIES],
|
|
121
|
-
runtimeVersion: WEBSITE_CONNECTOR_RUNTIME_VERSION
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
export function requiredWebsiteConnectorCapabilities(definition) {
|
|
126
|
-
const parsed = parseWebsiteConnectorDefinition(definition);
|
|
127
|
-
const required = new Set(["request.http"]);
|
|
128
|
-
if (parsed.auth.kind === "browser_storage") required.add("auth.browser-storage");
|
|
129
|
-
for (const operation of parsed.operations) {
|
|
130
|
-
required.add(`response.${operation.request.responseMode}`);
|
|
131
|
-
if (operation.request.body !== undefined) required.add("request.json-body");
|
|
132
|
-
if (operation.request.method !== "GET") required.add("request.write");
|
|
133
|
-
}
|
|
134
|
-
for (const capability of parsed.runtime?.requiredCapabilities ?? []) required.add(capability);
|
|
135
|
-
return [...required].sort();
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
export function assertWebsiteConnectorRuntimeCompatibility(definition, descriptor = websiteConnectorRuntimeDescriptor()) {
|
|
139
|
-
const parsed = parseWebsiteConnectorDefinition(definition);
|
|
140
|
-
const version = String(descriptor?.runtimeVersion || "");
|
|
141
|
-
const capabilities = new Set(Array.isArray(descriptor?.capabilities) ? descriptor.capabilities : []);
|
|
142
|
-
if (!SEMVER.test(version)) throw new WebsiteConnectorRuntimeError(500, "website_runtime_descriptor_invalid");
|
|
143
|
-
if (parsed.runtime?.minimumVersion && compareSemver(version, parsed.runtime.minimumVersion) < 0) {
|
|
144
|
-
throw new WebsiteConnectorRuntimeError(409, "website_runtime_version_incompatible", {
|
|
145
|
-
installedVersion: version,
|
|
146
|
-
minimumVersion: parsed.runtime.minimumVersion
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
const missingCapabilities = requiredWebsiteConnectorCapabilities(parsed).filter((capability) => !capabilities.has(capability));
|
|
150
|
-
if (missingCapabilities.length > 0) {
|
|
151
|
-
throw new WebsiteConnectorRuntimeError(409, "website_runtime_capability_missing", { missingCapabilities });
|
|
152
|
-
}
|
|
153
|
-
return { definition: parsed, descriptor: { capabilities: [...capabilities].sort(), runtimeVersion: version } };
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
export async function executeWebsiteConnectorOperation(definition, operationId, args, options = {}) {
|
|
157
|
-
assertWebsiteConnectorRuntimeCompatibility(definition, options.runtimeDescriptor);
|
|
158
|
-
let rendered;
|
|
159
|
-
try {
|
|
160
|
-
rendered = renderWebsiteOperation(definition, operationId, args);
|
|
161
|
-
} catch (error) {
|
|
162
|
-
const code = error instanceof WebsiteConnectorDefinitionError ? error.code : "website_operation_invalid";
|
|
163
|
-
throw new WebsiteConnectorRuntimeError(code === "website_operation_not_found" ? 404 : 400, code);
|
|
164
|
-
}
|
|
165
|
-
if (options.readOnlyOnly && !rendered.operation.readOnly) {
|
|
166
|
-
throw new WebsiteConnectorRuntimeError(409, "website_operation_not_read_only");
|
|
167
|
-
}
|
|
168
|
-
if (rendered.operation.requiresConnection && !options.storageState) {
|
|
169
|
-
throw new WebsiteConnectorRuntimeError(409, "browser_connection_required");
|
|
170
|
-
}
|
|
171
|
-
const headers = new Headers(rendered.request.headers);
|
|
172
|
-
if (rendered.operation.requiresConnection) {
|
|
173
|
-
applyBrowserAuthentication(headers, rendered.definition.auth, options.storageState, rendered.request.url);
|
|
174
|
-
}
|
|
175
|
-
let body;
|
|
176
|
-
if (rendered.request.body !== undefined) {
|
|
177
|
-
body = JSON.stringify(rendered.request.body);
|
|
178
|
-
if (!headers.has("content-type")) headers.set("Content-Type", "application/json");
|
|
179
|
-
}
|
|
180
|
-
let response;
|
|
181
|
-
try {
|
|
182
|
-
response = await (options.fetchImpl ?? fetch)(rendered.request.url, {
|
|
183
|
-
body,
|
|
184
|
-
headers,
|
|
185
|
-
method: rendered.request.method,
|
|
186
|
-
redirect: "manual",
|
|
187
|
-
signal: AbortSignal.timeout(options.timeoutMs ?? 30_000)
|
|
188
|
-
});
|
|
189
|
-
} catch {
|
|
190
|
-
throw new WebsiteConnectorRuntimeError(502, "website_operation_upstream_unavailable");
|
|
191
|
-
}
|
|
192
|
-
if (response.status >= 300 && response.status < 400) {
|
|
193
|
-
throw new WebsiteConnectorRuntimeError(409, "website_operation_redirect_forbidden");
|
|
194
|
-
}
|
|
195
|
-
if (response.status === 401 || response.status === 403) {
|
|
196
|
-
throw new WebsiteConnectorRuntimeError(409, "browser_connection_needs_reauth");
|
|
197
|
-
}
|
|
198
|
-
const raw = await readLimitedBody(response);
|
|
199
|
-
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
|
200
|
-
let data = raw;
|
|
201
|
-
const parseJson = rendered.request.responseMode === "json"
|
|
202
|
-
|| (rendered.request.responseMode === "auto" && contentType.includes("json"));
|
|
203
|
-
if (parseJson && raw) {
|
|
204
|
-
try { data = JSON.parse(raw); }
|
|
205
|
-
catch { throw new WebsiteConnectorRuntimeError(502, "website_operation_upstream_json_invalid"); }
|
|
206
|
-
}
|
|
207
|
-
const result = {
|
|
208
|
-
connectorId: rendered.definition.connectorId,
|
|
209
|
-
data,
|
|
210
|
-
fetchedAt: new Date().toISOString(),
|
|
211
|
-
ok: response.ok,
|
|
212
|
-
operationId: rendered.operation.operationId,
|
|
213
|
-
runtime: websiteConnectorRuntimeDescriptor(),
|
|
214
|
-
status: response.status
|
|
215
|
-
};
|
|
216
|
-
return options.storageState ? sanitizeWebsiteConnectorOutput(result, options.storageState) : result;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
export function browserStorageAuthenticated(state, auth, now = Date.now()) {
|
|
220
|
-
if (auth.kind !== "browser_storage") return true;
|
|
221
|
-
const seconds = now / 1000;
|
|
222
|
-
const hasCookie = !auth.sendCookies || state.cookies.some((cookie) => cookie.expires === undefined || cookie.expires < 0 || cookie.expires > seconds);
|
|
223
|
-
const hasHeaders = auth.headerSources.every((source) => {
|
|
224
|
-
const value = storageValue(state, source.origin, source.keySuffix);
|
|
225
|
-
return value !== undefined && credentialUnexpired(value, seconds);
|
|
226
|
-
});
|
|
227
|
-
return hasCookie && hasHeaders;
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
export function sanitizeWebsiteConnectorOutput(value, storageState) {
|
|
231
|
-
const secrets = [
|
|
232
|
-
...array(storageState?.cookies).map((cookie) => String(cookie?.value || "")),
|
|
233
|
-
...array(storageState?.origins).flatMap((origin) => array(origin?.localStorage).map((item) => String(item?.value || "")))
|
|
234
|
-
].filter((item) => item.length >= 4);
|
|
235
|
-
return sanitizeOutputValue(value, secrets, 0);
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
function parseRuntimeRequirements(value) {
|
|
239
|
-
if (value === undefined || value === null) return undefined;
|
|
240
|
-
const source = record(value, "website_runtime_requirements_invalid", "/runtime");
|
|
241
|
-
const minimumVersion = requiredText(source.minimumVersion, 40, "website_runtime_version_invalid", "/runtime/minimumVersion");
|
|
242
|
-
if (!SEMVER.test(minimumVersion)) fail("website_runtime_version_invalid", "/runtime/minimumVersion");
|
|
243
|
-
const requiredCapabilities = uniqueArray(source.requiredCapabilities ?? [], 0, 32, "/runtime/requiredCapabilities")
|
|
244
|
-
.map((item, index) => {
|
|
245
|
-
const capability = requiredText(item, 80, "website_runtime_capability_invalid", `/runtime/requiredCapabilities/${index}`);
|
|
246
|
-
if (!RUNTIME_CAPABILITY.test(capability)) fail("website_runtime_capability_invalid", `/runtime/requiredCapabilities/${index}`);
|
|
247
|
-
return capability;
|
|
248
|
-
}).sort();
|
|
249
|
-
return { minimumVersion, requiredCapabilities };
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
function parseAuthRecipe(value, allowedOrigins) {
|
|
253
|
-
const source = value === undefined ? { kind: "none" } : record(value, "website_auth_recipe_invalid", "/auth");
|
|
254
|
-
if (source.kind === "none") return { kind: "none" };
|
|
255
|
-
equal(source.kind, "browser_storage", "website_auth_recipe_invalid", "/auth/kind");
|
|
256
|
-
const sendCookies = source.sendCookies === true;
|
|
257
|
-
const headerSources = source.headerSources === undefined
|
|
258
|
-
? []
|
|
259
|
-
: uniqueArray(source.headerSources, 0, 12, "/auth/headerSources").map((item, index) => {
|
|
260
|
-
const binding = record(item, "website_auth_header_source_invalid", `/auth/headerSources/${index}`);
|
|
261
|
-
const name = headerName(binding.name, `/auth/headerSources/${index}/name`, true);
|
|
262
|
-
const origin = exactHttpsOrigin(binding.origin, "website_auth_header_source_invalid", `/auth/headerSources/${index}/origin`);
|
|
263
|
-
if (!allowedOrigins.includes(origin)) fail("website_auth_origin_not_allowed", `/auth/headerSources/${index}/origin`);
|
|
264
|
-
const keySuffix = requiredText(binding.keySuffix, 240, "website_auth_header_source_invalid", `/auth/headerSources/${index}/keySuffix`);
|
|
265
|
-
const prefix = optionalHeaderPrefix(binding.prefix, `/auth/headerSources/${index}/prefix`);
|
|
266
|
-
return { keySuffix, name, origin, ...(prefix ? { prefix } : {}) };
|
|
267
|
-
});
|
|
268
|
-
if (!sendCookies && headerSources.length === 0) fail("website_auth_recipe_empty", "/auth");
|
|
269
|
-
return { kind: "browser_storage", sendCookies, headerSources };
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
function parseOperation(value, allowedOrigins, index) {
|
|
273
|
-
const path = `/operations/${index}`;
|
|
274
|
-
const source = record(value, "website_operation_invalid", path);
|
|
275
|
-
const operationId = requiredText(source.operationId, 64, "website_operation_id_invalid", `${path}/operationId`);
|
|
276
|
-
if (!OPERATION_ID.test(operationId)) fail("website_operation_id_invalid", `${path}/operationId`);
|
|
277
|
-
const description = requiredText(source.description, 500, "website_operation_description_invalid", `${path}/description`);
|
|
278
|
-
const readOnly = source.readOnly === true;
|
|
279
|
-
const requiresConnection = source.requiresConnection === true;
|
|
280
|
-
const riskLevel = optionalEnum(source.riskLevel, ["low", "medium", "high", "critical"], readOnly ? "low" : "medium", `${path}/riskLevel`);
|
|
281
|
-
const inputSchema = parseInputSchema(source.inputSchema, `${path}/inputSchema`);
|
|
282
|
-
const requestSource = record(source.request, "website_operation_request_invalid", `${path}/request`);
|
|
283
|
-
const origin = exactHttpsOrigin(requestSource.origin, "website_operation_origin_invalid", `${path}/request/origin`);
|
|
284
|
-
if (!allowedOrigins.includes(origin)) fail("website_operation_origin_not_allowed", `${path}/request/origin`);
|
|
285
|
-
const method = optionalEnum(requestSource.method, ["GET", "POST", "PUT", "PATCH", "DELETE"], "GET", `${path}/request/method`);
|
|
286
|
-
const responseMode = optionalEnum(requestSource.responseMode, ["auto", "json", "text"], "auto", `${path}/request/responseMode`);
|
|
287
|
-
if (readOnly && method !== "GET") fail("website_operation_read_only_method_invalid", `${path}/request/method`);
|
|
288
|
-
const requestPath = requiredText(requestSource.path, 2_000, "website_operation_path_invalid", `${path}/request/path`);
|
|
289
|
-
if (!requestPath.startsWith("/") || requestPath.startsWith("//") || requestPath.includes("\\")) {
|
|
290
|
-
fail("website_operation_path_invalid", `${path}/request/path`);
|
|
291
|
-
}
|
|
292
|
-
const pathArguments = [...requestPath.matchAll(TEMPLATE_ARGUMENT)].map((match) => match[1]);
|
|
293
|
-
const properties = record(inputSchema.properties ?? {}, "website_operation_input_schema_invalid", `${path}/inputSchema/properties`);
|
|
294
|
-
for (const argument of pathArguments) {
|
|
295
|
-
if (!Object.hasOwn(properties, argument)) fail("website_operation_template_argument_unknown", `${path}/request/path`);
|
|
296
|
-
}
|
|
297
|
-
const query = requestSource.query === undefined
|
|
298
|
-
? {}
|
|
299
|
-
: Object.fromEntries(Object.entries(record(requestSource.query, "website_operation_query_invalid", `${path}/request/query`)).map(([name, argument]) => {
|
|
300
|
-
if (!/^[A-Za-z0-9_.~-]{1,120}$/.test(name)) fail("website_operation_query_name_invalid", `${path}/request/query/${name}`);
|
|
301
|
-
const argumentName = requiredText(argument, 120, "website_operation_query_argument_invalid", `${path}/request/query/${name}`);
|
|
302
|
-
if (!Object.hasOwn(properties, argumentName)) fail("website_operation_query_argument_unknown", `${path}/request/query/${name}`);
|
|
303
|
-
return [name, argumentName];
|
|
304
|
-
}));
|
|
305
|
-
const headers = requestSource.headers === undefined
|
|
306
|
-
? {}
|
|
307
|
-
: Object.fromEntries(Object.entries(record(requestSource.headers, "website_operation_headers_invalid", `${path}/request/headers`)).map(([name, headerValue]) => {
|
|
308
|
-
const normalized = requiredText(headerValue, 500, "website_operation_header_value_invalid", `${path}/request/headers/${name}`);
|
|
309
|
-
if (/[\r\n\0]/.test(normalized)) fail("website_operation_header_value_invalid", `${path}/request/headers/${name}`);
|
|
310
|
-
return [headerName(name, `${path}/request/headers/${name}`, false), normalized];
|
|
311
|
-
}));
|
|
312
|
-
const body = requestSource.body;
|
|
313
|
-
if (body !== undefined && method === "GET") fail("website_operation_get_body_invalid", `${path}/request/body`);
|
|
314
|
-
if (body !== undefined) validateBodyTemplate(body, properties, `${path}/request/body`);
|
|
315
|
-
return {
|
|
316
|
-
operationId,
|
|
317
|
-
description,
|
|
318
|
-
readOnly,
|
|
319
|
-
requiresConnection,
|
|
320
|
-
riskLevel,
|
|
321
|
-
inputSchema,
|
|
322
|
-
request: {
|
|
323
|
-
origin,
|
|
324
|
-
method,
|
|
325
|
-
path: requestPath,
|
|
326
|
-
query,
|
|
327
|
-
headers,
|
|
328
|
-
responseMode,
|
|
329
|
-
...(body === undefined ? {} : { body })
|
|
330
|
-
}
|
|
331
|
-
};
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
function optionalHeaderPrefix(value, path) {
|
|
335
|
-
if (value === undefined || value === null || value === "") return "";
|
|
336
|
-
if (typeof value !== "string" || value.length > 80 || /[\r\n\0]/.test(value)) {
|
|
337
|
-
fail("website_auth_header_source_invalid", path);
|
|
338
|
-
}
|
|
339
|
-
return value;
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
function parseInputSchema(value, path) {
|
|
343
|
-
const source = value === undefined
|
|
344
|
-
? { additionalProperties: false, properties: {}, type: "object" }
|
|
345
|
-
: record(value, "website_operation_input_schema_invalid", path);
|
|
346
|
-
equal(source.type, "object", "website_operation_input_schema_invalid", `${path}/type`);
|
|
347
|
-
if (source.additionalProperties !== false) fail("website_operation_input_schema_open", `${path}/additionalProperties`);
|
|
348
|
-
const properties = record(source.properties ?? {}, "website_operation_input_schema_invalid", `${path}/properties`);
|
|
349
|
-
const normalizedProperties = {};
|
|
350
|
-
for (const [name, candidate] of Object.entries(properties)) {
|
|
351
|
-
if (!/^[A-Za-z][A-Za-z0-9_]{0,119}$/.test(name)) fail("website_operation_input_name_invalid", `${path}/properties/${name}`);
|
|
352
|
-
const property = record(candidate, "website_operation_input_property_invalid", `${path}/properties/${name}`);
|
|
353
|
-
const type = optionalEnum(property.type, ["string", "number", "integer", "boolean"], "string", `${path}/properties/${name}/type`);
|
|
354
|
-
normalizedProperties[name] = {
|
|
355
|
-
type,
|
|
356
|
-
...(typeof property.maxLength === "number" ? { maxLength: boundedInteger(property.maxLength, 1, 20_000, `${path}/properties/${name}/maxLength`) } : {}),
|
|
357
|
-
...(Array.isArray(property.enum) ? { enum: property.enum.map((item) => scalar(item, `${path}/properties/${name}/enum`)) } : {})
|
|
358
|
-
};
|
|
359
|
-
}
|
|
360
|
-
const required = source.required === undefined ? [] : uniqueArray(source.required, 0, Object.keys(normalizedProperties).length, `${path}/required`).map(String);
|
|
361
|
-
for (const name of required) if (!Object.hasOwn(normalizedProperties, name)) fail("website_operation_required_unknown", `${path}/required`);
|
|
362
|
-
return { additionalProperties: false, properties: normalizedProperties, required, type: "object" };
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
function validateArguments(args, schema) {
|
|
366
|
-
for (const required of schema.required) if (!Object.hasOwn(args, required)) fail("website_operation_argument_required", `/arguments/${required}`);
|
|
367
|
-
for (const [name, value] of Object.entries(args)) {
|
|
368
|
-
const property = schema.properties[name];
|
|
369
|
-
if (!property) fail("website_operation_argument_unknown", `/arguments/${name}`);
|
|
370
|
-
if (property.type === "string" && typeof value !== "string") fail("website_operation_argument_type_invalid", `/arguments/${name}`);
|
|
371
|
-
if (property.type === "number" && (typeof value !== "number" || !Number.isFinite(value))) fail("website_operation_argument_type_invalid", `/arguments/${name}`);
|
|
372
|
-
if (property.type === "integer" && !Number.isInteger(value)) fail("website_operation_argument_type_invalid", `/arguments/${name}`);
|
|
373
|
-
if (property.type === "boolean" && typeof value !== "boolean") fail("website_operation_argument_type_invalid", `/arguments/${name}`);
|
|
374
|
-
if (property.maxLength && typeof value === "string" && value.length > property.maxLength) fail("website_operation_argument_too_long", `/arguments/${name}`);
|
|
375
|
-
if (property.enum && !property.enum.some((candidate) => Object.is(candidate, value))) fail("website_operation_argument_enum_invalid", `/arguments/${name}`);
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
function validateBodyTemplate(value, properties, path) {
|
|
380
|
-
if (typeof value === "string") {
|
|
381
|
-
if (value.startsWith("$") && !Object.hasOwn(properties, value.slice(1))) fail("website_operation_body_argument_unknown", path);
|
|
382
|
-
return;
|
|
383
|
-
}
|
|
384
|
-
if (value === null || typeof value === "number" || typeof value === "boolean") return;
|
|
385
|
-
if (Array.isArray(value)) {
|
|
386
|
-
value.forEach((item, index) => validateBodyTemplate(item, properties, `${path}/${index}`));
|
|
387
|
-
return;
|
|
388
|
-
}
|
|
389
|
-
const source = record(value, "website_operation_body_invalid", path);
|
|
390
|
-
for (const [name, item] of Object.entries(source)) validateBodyTemplate(item, properties, `${path}/${name}`);
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
function renderBody(value, args, path) {
|
|
394
|
-
if (typeof value === "string") {
|
|
395
|
-
if (value.startsWith("$")) return args[value.slice(1)];
|
|
396
|
-
return renderString(value, args, path);
|
|
397
|
-
}
|
|
398
|
-
if (Array.isArray(value)) return value.map((item, index) => renderBody(item, args, `${path}/${index}`));
|
|
399
|
-
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([name, item]) => [name, renderBody(item, args, `${path}/${name}`)]));
|
|
400
|
-
return value;
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
function renderString(value, args, path) {
|
|
404
|
-
return String(value).replace(TEMPLATE_ARGUMENT, (_match, name) => encodeURIComponent(scalarText(args[name], `/arguments/${name || path}`)));
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
function scalarText(value, path) {
|
|
408
|
-
scalar(value, path);
|
|
409
|
-
return String(value);
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
function scalar(value, path) {
|
|
413
|
-
if (!["string", "number", "boolean"].includes(typeof value) || (typeof value === "number" && !Number.isFinite(value))) {
|
|
414
|
-
fail("website_operation_argument_scalar_required", path);
|
|
415
|
-
}
|
|
416
|
-
return value;
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
function headerName(value, path, allowAuthorization) {
|
|
420
|
-
const name = requiredText(value, 100, "website_operation_header_name_invalid", path);
|
|
421
|
-
const normalized = name.toLowerCase();
|
|
422
|
-
if (!/^[A-Za-z][A-Za-z0-9-]*$/.test(name) || FORBIDDEN_HEADERS.has(normalized) || (!allowAuthorization && normalized === "authorization")) {
|
|
423
|
-
fail("website_operation_header_name_invalid", path);
|
|
424
|
-
}
|
|
425
|
-
return name;
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
function exactHttpsOrigin(value, code, path) {
|
|
429
|
-
const text = requiredText(value, 500, code, path);
|
|
430
|
-
try {
|
|
431
|
-
const url = new URL(text);
|
|
432
|
-
if (url.protocol !== "https:" || url.origin !== text || url.username || url.password || !publicHostname(url.hostname)) fail(code, path);
|
|
433
|
-
return url.origin;
|
|
434
|
-
} catch (error) {
|
|
435
|
-
if (error instanceof WebsiteConnectorDefinitionError) throw error;
|
|
436
|
-
fail(code, path);
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
function publicHostname(value) {
|
|
441
|
-
const hostname = String(value).toLowerCase().replace(/^\[|\]$/g, "");
|
|
442
|
-
if (!hostname.includes(".") || /^[0-9.]+$/.test(hostname) || hostname.includes(":")) return false;
|
|
443
|
-
return !hostname.endsWith(".localhost")
|
|
444
|
-
&& !hostname.endsWith(".local")
|
|
445
|
-
&& !hostname.endsWith(".internal")
|
|
446
|
-
&& hostname !== "metadata.google.internal";
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
function optionalHttpsUrl(value, allowedOrigins, path) {
|
|
450
|
-
if (value === undefined || value === null || value === "") return undefined;
|
|
451
|
-
try {
|
|
452
|
-
const url = new URL(requiredText(value, 2_000, "website_connector_login_url_invalid", path));
|
|
453
|
-
if (url.protocol !== "https:" || !allowedOrigins.includes(url.origin) || url.username || url.password || url.hash) fail("website_connector_login_url_invalid", path);
|
|
454
|
-
return url.toString();
|
|
455
|
-
} catch (error) {
|
|
456
|
-
if (error instanceof WebsiteConnectorDefinitionError) throw error;
|
|
457
|
-
fail("website_connector_login_url_invalid", path);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
function record(value, code, path) {
|
|
462
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) fail(code, path);
|
|
463
|
-
return value;
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
function uniqueArray(value, min, max, path) {
|
|
467
|
-
if (!Array.isArray(value) || value.length < min || value.length > max) fail("website_connector_array_invalid", path);
|
|
468
|
-
const serialized = value.map((item) => typeof item === "string" ? item : JSON.stringify(item));
|
|
469
|
-
if (new Set(serialized).size !== serialized.length) fail("website_connector_array_duplicate", path);
|
|
470
|
-
return value;
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
function requiredText(value, max, code, path) {
|
|
474
|
-
const text = typeof value === "string" ? value.trim() : "";
|
|
475
|
-
if (!text || text.length > max) fail(code, path);
|
|
476
|
-
return text;
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
function optionalText(value, max, code, path) {
|
|
480
|
-
if (value === undefined || value === null || value === "") return undefined;
|
|
481
|
-
return requiredText(value, max, code, path);
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
function optionalEnum(value, allowed, fallback, path) {
|
|
485
|
-
const candidate = value === undefined ? fallback : value;
|
|
486
|
-
if (!allowed.includes(candidate)) fail("website_connector_enum_invalid", path);
|
|
487
|
-
return candidate;
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
function boundedInteger(value, min, max, path) {
|
|
491
|
-
if (!Number.isInteger(value) || value < min || value > max) fail("website_connector_integer_invalid", path);
|
|
492
|
-
return value;
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
function equal(actual, expected, code, path) {
|
|
496
|
-
if (actual !== expected) fail(code, path);
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
function compareSemver(left, right) {
|
|
500
|
-
const a = left.split(".").map(Number);
|
|
501
|
-
const b = right.split(".").map(Number);
|
|
502
|
-
for (let index = 0; index < 3; index += 1) {
|
|
503
|
-
if (a[index] !== b[index]) return a[index] < b[index] ? -1 : 1;
|
|
504
|
-
}
|
|
505
|
-
return 0;
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
function applyBrowserAuthentication(headers, auth, state, requestUrl) {
|
|
509
|
-
if (auth.kind !== "browser_storage") {
|
|
510
|
-
throw new WebsiteConnectorRuntimeError(503, "website_connector_auth_invalid");
|
|
511
|
-
}
|
|
512
|
-
if (auth.sendCookies) {
|
|
513
|
-
const cookie = cookieHeader(state, new URL(requestUrl));
|
|
514
|
-
if (!cookie) throw new WebsiteConnectorRuntimeError(409, "browser_connection_needs_reauth");
|
|
515
|
-
headers.set("Cookie", cookie);
|
|
516
|
-
}
|
|
517
|
-
for (const source of auth.headerSources) {
|
|
518
|
-
const value = storageValue(state, source.origin, source.keySuffix);
|
|
519
|
-
if (value === undefined || !credentialUnexpired(value, Date.now() / 1000)) {
|
|
520
|
-
throw new WebsiteConnectorRuntimeError(409, "browser_connection_needs_reauth");
|
|
521
|
-
}
|
|
522
|
-
headers.set(source.name, `${source.prefix ?? ""}${value}`);
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
function storageValue(state, origin, keySuffix) {
|
|
527
|
-
const matches = array(state?.origins)
|
|
528
|
-
.filter((candidate) => candidate?.origin === origin)
|
|
529
|
-
.flatMap((candidate) => array(candidate?.localStorage))
|
|
530
|
-
.filter((candidate) => String(candidate?.name || "").endsWith(keySuffix));
|
|
531
|
-
return matches.length === 1 && matches[0]?.value ? String(matches[0].value) : undefined;
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
function credentialUnexpired(value, nowSeconds) {
|
|
535
|
-
const parts = String(value).split(".");
|
|
536
|
-
if (parts.length !== 3) return true;
|
|
537
|
-
try {
|
|
538
|
-
const base64 = parts[1].replaceAll("-", "+").replaceAll("_", "/");
|
|
539
|
-
const padded = base64.padEnd(Math.ceil(base64.length / 4) * 4, "=");
|
|
540
|
-
const payload = JSON.parse(atob(padded));
|
|
541
|
-
return typeof payload.exp !== "number" || payload.exp > nowSeconds + 30;
|
|
542
|
-
} catch {
|
|
543
|
-
return true;
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
function cookieHeader(state, url) {
|
|
548
|
-
const now = Date.now() / 1000;
|
|
549
|
-
return array(state?.cookies)
|
|
550
|
-
.filter((cookie) => {
|
|
551
|
-
const domain = String(cookie?.domain || "").replace(/^\./, "").toLowerCase();
|
|
552
|
-
const hostname = url.hostname.toLowerCase();
|
|
553
|
-
const domainMatch = hostname === domain || hostname.endsWith(`.${domain}`);
|
|
554
|
-
const pathMatch = url.pathname.startsWith(cookie?.path || "/");
|
|
555
|
-
const unexpired = cookie?.expires === undefined || cookie.expires < 0 || cookie.expires > now;
|
|
556
|
-
return domainMatch && pathMatch && unexpired && (!cookie?.secure || url.protocol === "https:");
|
|
557
|
-
})
|
|
558
|
-
.sort((left, right) => (right.path?.length ?? 0) - (left.path?.length ?? 0))
|
|
559
|
-
.map((cookie) => `${cookie.name}=${cookie.value}`)
|
|
560
|
-
.join("; ");
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
async function readLimitedBody(response) {
|
|
564
|
-
const declared = Number(response.headers.get("content-length") || 0);
|
|
565
|
-
if (declared > MAX_RESPONSE_BYTES) throw new WebsiteConnectorRuntimeError(502, "website_operation_response_too_large");
|
|
566
|
-
if (!response.body) return "";
|
|
567
|
-
const reader = response.body.getReader();
|
|
568
|
-
const chunks = [];
|
|
569
|
-
let size = 0;
|
|
570
|
-
while (true) {
|
|
571
|
-
const { done, value } = await reader.read();
|
|
572
|
-
if (done) break;
|
|
573
|
-
size += value.byteLength;
|
|
574
|
-
if (size > MAX_RESPONSE_BYTES) {
|
|
575
|
-
await reader.cancel();
|
|
576
|
-
throw new WebsiteConnectorRuntimeError(502, "website_operation_response_too_large");
|
|
577
|
-
}
|
|
578
|
-
chunks.push(value);
|
|
579
|
-
}
|
|
580
|
-
const body = new Uint8Array(size);
|
|
581
|
-
let offset = 0;
|
|
582
|
-
for (const chunk of chunks) {
|
|
583
|
-
body.set(chunk, offset);
|
|
584
|
-
offset += chunk.byteLength;
|
|
585
|
-
}
|
|
586
|
-
return new TextDecoder().decode(body);
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
function sanitizeOutputValue(value, secrets, depth) {
|
|
590
|
-
if (depth > 12) return "[truncated]";
|
|
591
|
-
if (typeof value === "string") {
|
|
592
|
-
let result = value.slice(0, 100_000);
|
|
593
|
-
for (const secret of secrets) result = result.replaceAll(secret, "[redacted]");
|
|
594
|
-
return result;
|
|
595
|
-
}
|
|
596
|
-
if (typeof value === "number" || typeof value === "boolean" || value === null) return value;
|
|
597
|
-
if (Array.isArray(value)) return value.slice(0, 1_000).map((item) => sanitizeOutputValue(item, secrets, depth + 1));
|
|
598
|
-
if (!value || typeof value !== "object") return undefined;
|
|
599
|
-
return Object.fromEntries(Object.entries(value).flatMap(([key, child]) => (
|
|
600
|
-
sensitiveKey(key) ? [] : [[key, sanitizeOutputValue(child, secrets, depth + 1)]]
|
|
601
|
-
)));
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
function sensitiveKey(key) {
|
|
605
|
-
return /cookie|password|authorization|access.?token|refresh.?token|session.?state|storage.?state|credential/i.test(key);
|
|
606
|
-
}
|
|
607
|
-
|
|
608
|
-
function array(value) {
|
|
609
|
-
return Array.isArray(value) ? value : [];
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
function fail(code, path) {
|
|
613
|
-
throw new WebsiteConnectorDefinitionError(code, path);
|
|
614
|
-
}
|