@lerianstudio/matcher-mcp 1.2.0 → 1.2.1-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +0 -33
- package/dist/config.js.map +1 -1
- package/dist/spec/enum-policy.js +83 -0
- package/dist/spec/enum-policy.js.map +1 -0
- package/dist/spec/openapi.yaml +15 -20
- package/dist/spec/schema-parity.js +393 -0
- package/dist/spec/schema-parity.js.map +1 -0
- package/dist/tools/context/create.js +18 -3
- package/dist/tools/context/create.js.map +1 -1
- package/dist/tools/context/update.js +20 -5
- package/dist/tools/context/update.js.map +1 -1
- package/dist/tools/exception/bulk-dispatch.js +11 -13
- package/dist/tools/exception/bulk-dispatch.js.map +1 -1
- package/dist/tools/exception/list.js +12 -3
- package/dist/tools/exception/list.js.map +1 -1
- package/dist/tools/fee-rule/create.js +36 -13
- package/dist/tools/fee-rule/create.js.map +1 -1
- package/dist/tools/generic/describe-operation.js +3 -2
- package/dist/tools/generic/describe-operation.js.map +1 -1
- package/dist/tools/ingestion/index.js +13 -8
- package/dist/tools/ingestion/index.js.map +1 -1
- package/dist/tools/ingestion/upload-begin.js +116 -0
- package/dist/tools/ingestion/upload-begin.js.map +1 -0
- package/dist/tools/ingestion/upload.js +47 -97
- package/dist/tools/ingestion/upload.js.map +1 -1
- package/dist/tools/source/create.js +6 -3
- package/dist/tools/source/create.js.map +1 -1
- package/package.json +1 -1
- package/dist/tools/ingestion/fetch-source.js +0 -105
- package/dist/tools/ingestion/fetch-source.js.map +0 -1
|
@@ -2,28 +2,23 @@
|
|
|
2
2
|
// /v1/imports/contexts/{contextId}/sources/{sourceId}/upload.
|
|
3
3
|
//
|
|
4
4
|
// The backend endpoint is multipart/form-data, which a JSON MCP tool can't
|
|
5
|
-
// carry directly. This tool accepts the file body
|
|
6
|
-
// relay rebuilds the real multipart request
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
5
|
+
// carry directly. This tool accepts the file body inline as base64 and the
|
|
6
|
+
// relay rebuilds the real multipart request: wrap the bytes in a `FormData`
|
|
7
|
+
// and let `matcher/client.ts`'s FormData branch hand it to `fetch` untouched
|
|
8
|
+
// (no JSON.stringify, no content-type override — the runtime sets the
|
|
9
|
+
// multipart boundary itself).
|
|
10
10
|
//
|
|
11
|
-
// - `contentBase64`: the file inline, base64-encoded.
|
|
12
|
-
//
|
|
11
|
+
// - `contentBase64`: the file inline, base64-encoded. This is the path for
|
|
12
|
+
// SMALL, programmatically-generated payloads — an LLM client must emit the
|
|
13
13
|
// whole encoded string in its OUTPUT tokens, and base64 inflates bytes by
|
|
14
14
|
// ~33% on top of that (a 1000-row file is easily ~150k tokens). Practical
|
|
15
15
|
// ceiling: the relay's inbound JSON body cap (1 MiB default,
|
|
16
16
|
// MAX_BODY_BYTES override) means raw files up to ~750 KB upload cleanly.
|
|
17
|
-
// - `fileUrl`: a URL to the file. The RELAY fetches the bytes server-side
|
|
18
|
-
// (see fetch-source.ts) — the client only ever emits a short URL, so this
|
|
19
|
-
// is the path for real files. The fetch is SSRF-guarded: only hostnames
|
|
20
|
-
// in MCP_UPLOAD_ALLOWED_HOSTS (exact match, empty by default) are
|
|
21
|
-
// reachable, redirects are rejected, and the response is size-capped by
|
|
22
|
-
// MCP_UPLOAD_MAX_FETCH_BYTES.
|
|
23
17
|
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
18
|
+
// For a real LOCAL file (bytes on the operator's disk), use
|
|
19
|
+
// ingestion_upload_begin (upload-begin.ts) instead, which brokers a direct
|
|
20
|
+
// client→matcher upload so the bytes never traverse the relay or the token
|
|
21
|
+
// budget.
|
|
27
22
|
//
|
|
28
23
|
// Field order matters: `format` MUST be appended BEFORE `file`. The backend's
|
|
29
24
|
// streaming multipart parser binds the declared format from whichever field
|
|
@@ -36,10 +31,13 @@
|
|
|
36
31
|
// validates both against the relayed token's tenant. No tenant field is
|
|
37
32
|
// accepted here either.
|
|
38
33
|
import { z } from 'zod';
|
|
39
|
-
import { loadConfig } from '../../config.js';
|
|
40
|
-
import { fetchSourceFile } from './fetch-source.js';
|
|
41
34
|
import { contextSourceUploadPath, dispatchIngestion } from './shared.js';
|
|
42
|
-
/**
|
|
35
|
+
/** Current known parse formats the backend accepts for an ingestion upload.
|
|
36
|
+
* Not a closed enum on the input schema: ingestion formats are a GROWING set
|
|
37
|
+
* (the connector-parity roadmap adds parsers), so the field is typed as a free
|
|
38
|
+
* string and this list only feeds the description as a hint — a closed enum
|
|
39
|
+
* would lag the backend and reject a valid new format at the client. See
|
|
40
|
+
* docs/enum-policy.md (registered in INTENTIONALLY_OPEN_FIELDS). */
|
|
43
41
|
const UPLOAD_FORMATS = ['csv', 'json', 'xml', 'camt053'];
|
|
44
42
|
/** Zod raw shape for `ingestion_upload` input — path ids + format + file. */
|
|
45
43
|
export const ingestionUploadInputShape = {
|
|
@@ -55,103 +53,55 @@ export const ingestionUploadInputShape = {
|
|
|
55
53
|
.describe('The source id (UUID) within the context to upload into. Must already ' +
|
|
56
54
|
'exist (create it first with source_create).'),
|
|
57
55
|
format: z
|
|
58
|
-
.
|
|
59
|
-
.
|
|
60
|
-
'
|
|
61
|
-
'
|
|
62
|
-
'
|
|
56
|
+
.string()
|
|
57
|
+
.min(1)
|
|
58
|
+
.describe('The declared parse format of the file. Free-form string — the backend ' +
|
|
59
|
+
'validates and the accepted set grows as new parsers are added. ' +
|
|
60
|
+
'Current known formats: ' +
|
|
61
|
+
UPLOAD_FORMATS.join(' | ') +
|
|
62
|
+
'. Sent as the first multipart field — matcher binds the format from ' +
|
|
63
|
+
'it before reading the file, so always pass the true format rather ' +
|
|
64
|
+
'than relying on filename-extension inference.'),
|
|
63
65
|
filename: z
|
|
64
66
|
.string()
|
|
65
67
|
.min(1)
|
|
66
68
|
.describe('The original file name (e.g. "transactions.csv"), sent as the ' +
|
|
67
69
|
'multipart filename for the file part.'),
|
|
68
70
|
contentBase64: z
|
|
69
|
-
.string()
|
|
70
|
-
.base64()
|
|
71
|
-
.optional()
|
|
72
|
-
.describe('The file bytes, base64-encoded. For SMALL files only — base64 inflates ' +
|
|
73
|
-
'bytes by ~33% and the LLM client must emit the whole encoded string ' +
|
|
74
|
-
'as output tokens. The relay\'s inbound JSON body is capped (1 MiB by ' +
|
|
75
|
-
'default, MAX_BODY_BYTES env override), so raw files up to ~750 KB ' +
|
|
76
|
-
'upload cleanly under the default. Exactly one of contentBase64 / ' +
|
|
77
|
-
'fileUrl is required; for real files prefer fileUrl.'),
|
|
78
|
-
fileUrl: z
|
|
79
71
|
.string()
|
|
80
72
|
.min(1)
|
|
81
|
-
.
|
|
82
|
-
.describe('
|
|
83
|
-
'
|
|
84
|
-
'
|
|
85
|
-
'
|
|
86
|
-
'
|
|
87
|
-
'
|
|
88
|
-
'
|
|
89
|
-
'of contentBase64 / fileUrl is required.'),
|
|
73
|
+
.base64()
|
|
74
|
+
.describe('The file bytes, base64-encoded (required). For SMALL, programmatically-' +
|
|
75
|
+
'generated payloads only — base64 inflates bytes by ~33% and the LLM ' +
|
|
76
|
+
'client must emit the whole encoded string as output tokens. The ' +
|
|
77
|
+
'relay\'s inbound JSON body is capped (1 MiB by default, MAX_BODY_BYTES ' +
|
|
78
|
+
'env override), so raw files up to ~750 KB upload cleanly under the ' +
|
|
79
|
+
'default. For a real LOCAL file use ingestion_upload_begin instead (it ' +
|
|
80
|
+
'brokers a direct upload that bypasses this token/body cap).'),
|
|
90
81
|
};
|
|
91
82
|
/**
|
|
92
|
-
* Register `ingestion_upload`.
|
|
93
|
-
* `
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
* context, 400 malformed file) surfaces via the shared toToolError.
|
|
83
|
+
* Register `ingestion_upload`. Decodes the file bytes from `contentBase64`,
|
|
84
|
+
* then rebuilds a `FormData` with `format` appended before `file` (order is
|
|
85
|
+
* load-bearing — see module comment) and dispatches the POST through the shared
|
|
86
|
+
* fail-closed relay; a matcher error (e.g. 403 inactive context, 400 malformed
|
|
87
|
+
* file) surfaces via the shared toToolError.
|
|
98
88
|
*/
|
|
99
89
|
export function registerIngestionUploadTool(server) {
|
|
100
90
|
server.registerTool('ingestion_upload', {
|
|
101
|
-
description: 'Upload a file into a source for ingestion
|
|
102
|
-
'
|
|
103
|
-
'
|
|
104
|
-
'
|
|
105
|
-
'
|
|
106
|
-
'
|
|
107
|
-
'
|
|
108
|
-
'hostname allowlist (MCP_UPLOAD_ALLOWED_HOSTS, empty by default) and ' +
|
|
109
|
-
'size-capped (MCP_UPLOAD_MAX_FETCH_BYTES); a non-allowlisted host is ' +
|
|
110
|
-
'rejected before any network call. Either way the relay rebuilds the ' +
|
|
111
|
-
'multipart/form-data upload the matcher backend expects. ' +
|
|
91
|
+
description: 'Upload a file into a source for ingestion by passing the file body ' +
|
|
92
|
+
'inline as contentBase64 — base64 in JSON, for SMALL programmatic ' +
|
|
93
|
+
'payloads only (raw files up to ~750 KB; larger payloads burn the ' +
|
|
94
|
+
'calling LLM\'s output tokens and risk the relay\'s inbound body cap). ' +
|
|
95
|
+
'For a real LOCAL file use ingestion_upload_begin instead (a direct ' +
|
|
96
|
+
'client→matcher upload that bypasses this relay entirely). The relay ' +
|
|
97
|
+
'rebuilds the multipart/form-data upload the matcher backend expects. ' +
|
|
112
98
|
'Prerequisites: the context must be ACTIVE and the sourceId must ' +
|
|
113
99
|
'already exist. No tenant field is accepted. Returns the 202 job ' +
|
|
114
100
|
'response, or a structured RFC 9457 error (e.g. 403 if the context ' +
|
|
115
101
|
'is not ACTIVE).',
|
|
116
102
|
inputSchema: ingestionUploadInputShape,
|
|
117
103
|
}, async (args, extra) => {
|
|
118
|
-
const
|
|
119
|
-
const hasFileUrl = args.fileUrl !== undefined;
|
|
120
|
-
if (hasBase64 === hasFileUrl) {
|
|
121
|
-
return {
|
|
122
|
-
isError: true,
|
|
123
|
-
content: [
|
|
124
|
-
{
|
|
125
|
-
type: 'text',
|
|
126
|
-
text: 'ingestion_upload requires exactly one of contentBase64 or ' +
|
|
127
|
-
`fileUrl, got ${hasBase64 ? 'both' : 'neither'}.`,
|
|
128
|
-
},
|
|
129
|
-
],
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
|
-
let bytes;
|
|
133
|
-
if (hasBase64) {
|
|
134
|
-
bytes = Buffer.from(args.contentBase64, 'base64');
|
|
135
|
-
}
|
|
136
|
-
else {
|
|
137
|
-
const cfg = loadConfig();
|
|
138
|
-
try {
|
|
139
|
-
bytes = await fetchSourceFile(args.fileUrl, {
|
|
140
|
-
allowedHosts: cfg.uploadAllowedHosts,
|
|
141
|
-
maxBytes: cfg.uploadMaxFetchBytes,
|
|
142
|
-
timeoutMs: cfg.timeoutMs,
|
|
143
|
-
});
|
|
144
|
-
}
|
|
145
|
-
catch (err) {
|
|
146
|
-
const message = err instanceof Error ? err.message : 'unknown error';
|
|
147
|
-
return {
|
|
148
|
-
isError: true,
|
|
149
|
-
content: [
|
|
150
|
-
{ type: 'text', text: `failed to fetch fileUrl: ${message}` },
|
|
151
|
-
],
|
|
152
|
-
};
|
|
153
|
-
}
|
|
154
|
-
}
|
|
104
|
+
const bytes = Buffer.from(args.contentBase64, 'base64');
|
|
155
105
|
const form = new FormData();
|
|
156
106
|
// 'format' MUST precede 'file' — see module comment.
|
|
157
107
|
form.append('format', args.format);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload.js","sourceRoot":"","sources":["../../../src/tools/ingestion/upload.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,8DAA8D;AAC9D,EAAE;AACF,2EAA2E;AAC3E,
|
|
1
|
+
{"version":3,"file":"upload.js","sourceRoot":"","sources":["../../../src/tools/ingestion/upload.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,8DAA8D;AAC9D,EAAE;AACF,2EAA2E;AAC3E,2EAA2E;AAC3E,4EAA4E;AAC5E,6EAA6E;AAC7E,sEAAsE;AACtE,8BAA8B;AAC9B,EAAE;AACF,6EAA6E;AAC7E,+EAA+E;AAC/E,8EAA8E;AAC9E,8EAA8E;AAC9E,iEAAiE;AACjE,6EAA6E;AAC7E,EAAE;AACF,4DAA4D;AAC5D,2EAA2E;AAC3E,2EAA2E;AAC3E,UAAU;AACV,EAAE;AACF,8EAA8E;AAC9E,4EAA4E;AAC5E,wEAAwE;AACxE,0EAA0E;AAC1E,6EAA6E;AAC7E,6CAA6C;AAC7C,EAAE;AACF,8EAA8E;AAC9E,wEAAwE;AACxE,wBAAwB;AAExB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAIvB,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAExE;;;;;qEAKqE;AACrE,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAU,CAAA;AAEjE,6EAA6E;AAC7E,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,yEAAyE;QACvE,sEAAsE;QACtE,4CAA4C,CAC/C;IACH,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,uEAAuE;QACrE,6CAA6C,CAChD;IACH,MAAM,EAAE,CAAC;SACN,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,wEAAwE;QACtE,iEAAiE;QACjE,yBAAyB;QACzB,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC;QAC1B,sEAAsE;QACtE,oEAAoE;QACpE,+CAA+C,CAClD;IACH,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,gEAAgE;QAC9D,uCAAuC,CAC1C;IACH,aAAa,EAAE,CAAC;SACb,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,MAAM,EAAE;SACR,QAAQ,CACP,yEAAyE;QACvE,sEAAsE;QACtE,kEAAkE;QAClE,yEAAyE;QACzE,qEAAqE;QACrE,wEAAwE;QACxE,6DAA6D,CAChE;CACJ,CAAA;AAOD;;;;;;GAMG;AACH,MAAM,UAAU,2BAA2B,CAAC,MAAiB;IAC3D,MAAM,CAAC,YAAY,CACjB,kBAAkB,EAClB;QACE,WAAW,EACT,qEAAqE;YACrE,mEAAmE;YACnE,mEAAmE;YACnE,wEAAwE;YACxE,qEAAqE;YACrE,sEAAsE;YACtE,uEAAuE;YACvE,kEAAkE;YAClE,kEAAkE;YAClE,oEAAoE;YACpE,iBAAiB;QACnB,WAAW,EAAE,yBAAyB;KACvC,EACD,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QACpB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;QAEvD,MAAM,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAA;QAC3B,qDAAqD;QACrD,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAClC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;QAErD,OAAO,iBAAiB,CACtB;YACE,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,uBAAuB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC;YAC5D,IAAI,EAAE,IAAI;SACX,EACD,KAAK,CACN,CAAA;IACH,CAAC,CACF,CAAA;AACH,CAAC"}
|
|
@@ -46,8 +46,10 @@ export const createSourceInputShape = {
|
|
|
46
46
|
'reconciled against each other.'),
|
|
47
47
|
config: z
|
|
48
48
|
.record(z.string(), z.unknown())
|
|
49
|
+
.optional()
|
|
49
50
|
.describe('Source-specific configuration object (connection and parsing settings). ' +
|
|
50
|
-
'
|
|
51
|
+
'Optional; defaults to an empty object server-side when omitted. Its ' +
|
|
52
|
+
'shape depends on the source type and is validated server-side.'),
|
|
51
53
|
};
|
|
52
54
|
/**
|
|
53
55
|
* Build the POST body from the input minus the path-only `contextId`. Only the
|
|
@@ -68,8 +70,9 @@ export function registerSourceCreateTool(server) {
|
|
|
68
70
|
server.registerTool('source_create', {
|
|
69
71
|
description: 'Create a reconciliation source under a context. Provide contextId ' +
|
|
70
72
|
'(UUID path param — not a tenant field), name (1–50), type ' +
|
|
71
|
-
'(LEDGER|BANK|GATEWAY|CUSTOM|FETCHER), side (LEFT|RIGHT), and
|
|
72
|
-
'type-specific config object (validated server-side
|
|
73
|
+
'(LEDGER|BANK|GATEWAY|CUSTOM|FETCHER), side (LEFT|RIGHT), and an ' +
|
|
74
|
+
'optional type-specific config object (validated server-side; defaults ' +
|
|
75
|
+
'to an empty object when omitted). The source is ' +
|
|
73
76
|
"created for your token's tenant (no tenant field is accepted). Returns " +
|
|
74
77
|
'the created source, or a structured RFC 9457 error.',
|
|
75
78
|
inputSchema: createSourceInputShape,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create.js","sourceRoot":"","sources":["../../../src/tools/source/create.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,iFAAiF;AACjF,6EAA6E;AAC7E,gFAAgF;AAChF,mBAAmB;AACnB,EAAE;AACF,2EAA2E;AAC3E,6EAA6E;AAC7E,4EAA4E;AAE5E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAIvB,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEhE,+FAA+F;AAC/F,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,QAAQ;IACR,MAAM;IACN,SAAS;IACT,QAAQ;IACR,SAAS;CACD,CAAA;AAEV,qFAAqF;AACrF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,OAAO,CAAU,CAAA;AAEtD,6EAA6E;AAC7E,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,wEAAwE;QACtE,yEAAyE;QACzE,iCAAiC,CACpC;IACH,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,CAAC,iDAAiD,CAAC;IAC9D,IAAI,EAAE,CAAC;SACJ,IAAI,CAAC,YAAY,CAAC;SAClB,QAAQ,CACP,2EAA2E;QACzE,0EAA0E;QAC1E,qCAAqC,CACxC;IACH,IAAI,EAAE,CAAC;SACJ,IAAI,CAAC,YAAY,CAAC;SAClB,QAAQ,CACP,oEAAoE;QAClE,gCAAgC,CACnC;IACH,MAAM,EAAE,CAAC;SACN,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;SAC/B,QAAQ,CACP,0EAA0E;QACxE,
|
|
1
|
+
{"version":3,"file":"create.js","sourceRoot":"","sources":["../../../src/tools/source/create.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,iFAAiF;AACjF,6EAA6E;AAC7E,gFAAgF;AAChF,mBAAmB;AACnB,EAAE;AACF,2EAA2E;AAC3E,6EAA6E;AAC7E,4EAA4E;AAE5E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAIvB,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEhE,+FAA+F;AAC/F,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,QAAQ;IACR,MAAM;IACN,SAAS;IACT,QAAQ;IACR,SAAS;CACD,CAAA;AAEV,qFAAqF;AACrF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,OAAO,CAAU,CAAA;AAEtD,6EAA6E;AAC7E,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,wEAAwE;QACtE,yEAAyE;QACzE,iCAAiC,CACpC;IACH,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,CAAC,iDAAiD,CAAC;IAC9D,IAAI,EAAE,CAAC;SACJ,IAAI,CAAC,YAAY,CAAC;SAClB,QAAQ,CACP,2EAA2E;QACzE,0EAA0E;QAC1E,qCAAqC,CACxC;IACH,IAAI,EAAE,CAAC;SACJ,IAAI,CAAC,YAAY,CAAC;SAClB,QAAQ,CACP,oEAAoE;QAClE,gCAAgC,CACnC;IACH,MAAM,EAAE,CAAC;SACN,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;SAC/B,QAAQ,EAAE;SACV,QAAQ,CACP,0EAA0E;QACxE,sEAAsE;QACtE,gEAAgE,CACnE;CACJ,CAAA;AAOD;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,IAAuB;IAEvB,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,UAAU,CAAA;IACf,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB,CAAC,MAAiB;IACxD,MAAM,CAAC,YAAY,CACjB,eAAe,EACf;QACE,WAAW,EACT,oEAAoE;YACpE,4DAA4D;YAC5D,kEAAkE;YAClE,wEAAwE;YACxE,kDAAkD;YAClD,yEAAyE;YACzE,qDAAqD;QACvD,WAAW,EAAE,sBAAsB;KACpC,EACD,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QACpB,OAAO,cAAc,CACnB;YACE,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC;YACxC,IAAI,EAAE,eAAe,CAAC,IAAI,CAAC;SAC5B,EACD,KAAK,CACN,CAAA;IACH,CAAC,CACF,CAAA;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
// SSRF-guarded fetch for `ingestion_upload`'s upload-by-reference path
|
|
2
|
-
// (`fileUrl`). Base64-in-JSON doesn't scale — a real file inflates ~33% and
|
|
3
|
-
// burns the calling LLM's output tokens one-for-one. This lets the client pass
|
|
4
|
-
// a short URL instead; the relay fetches the bytes server-side and rebuilds
|
|
5
|
-
// the multipart upload exactly as the base64 path does.
|
|
6
|
-
//
|
|
7
|
-
// The relay is a network-reachable, credential-bearing process: an unguarded
|
|
8
|
-
// "fetch whatever URL the client gives you" is textbook SSRF (cloud metadata
|
|
9
|
-
// endpoints, internal services, redirect chains to either). Every check below
|
|
10
|
-
// is fail-closed — on ambiguity or partial information, reject.
|
|
11
|
-
/**
|
|
12
|
-
* Fetch a source file from `fileUrl` under a strict SSRF allowlist, returning
|
|
13
|
-
* the raw bytes. Throws a plain `Error` with a clear, secret-free message on
|
|
14
|
-
* any rejection — there is no token on this outbound call to begin with (it
|
|
15
|
-
* targets an external resource, not matcher), so messages may freely include
|
|
16
|
-
* the URL/hostname for operator debugging.
|
|
17
|
-
*/
|
|
18
|
-
export async function fetchSourceFile(fileUrl, opts) {
|
|
19
|
-
let url;
|
|
20
|
-
try {
|
|
21
|
-
url = new URL(fileUrl);
|
|
22
|
-
}
|
|
23
|
-
catch {
|
|
24
|
-
throw new Error(`invalid fileUrl: ${fileUrl} (not a parseable URL)`);
|
|
25
|
-
}
|
|
26
|
-
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
27
|
-
throw new Error(`invalid fileUrl: ${fileUrl} (scheme must be http or https, got ${url.protocol})`);
|
|
28
|
-
}
|
|
29
|
-
// Exact, case-insensitive hostname match only — no substring/suffix
|
|
30
|
-
// matching. An empty allowlist (the default) rejects every host.
|
|
31
|
-
// NOTE: this validates the hostname string, not the resolved IP. Only
|
|
32
|
-
// allowlist non-rebindable/static internal names (e.g. host.docker.internal),
|
|
33
|
-
// never external domains whose DNS you don't control (DNS-rebinding vector).
|
|
34
|
-
const hostname = url.hostname.toLowerCase();
|
|
35
|
-
const allowed = opts.allowedHosts.some((h) => h.toLowerCase() === hostname);
|
|
36
|
-
if (!allowed) {
|
|
37
|
-
throw new Error(`fileUrl host "${url.hostname}" is not in the upload allowlist ` +
|
|
38
|
-
'(MCP_UPLOAD_ALLOWED_HOSTS) — add it there to permit this host');
|
|
39
|
-
}
|
|
40
|
-
const controller = new AbortController();
|
|
41
|
-
const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
|
|
42
|
-
let res;
|
|
43
|
-
try {
|
|
44
|
-
res = await fetch(fileUrl, {
|
|
45
|
-
// A redirect must NOT be silently followed — redirect-to-internal (e.g.
|
|
46
|
-
// an allowlisted host 302-ing to a metadata IP) is a classic SSRF
|
|
47
|
-
// bypass. 'error' makes fetch reject the promise on any redirect.
|
|
48
|
-
redirect: 'error',
|
|
49
|
-
signal: controller.signal,
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
catch (err) {
|
|
53
|
-
if (controller.signal.aborted) {
|
|
54
|
-
throw new Error(`fetching fileUrl timed out after ${opts.timeoutMs}ms: ${fileUrl}`, { cause: err });
|
|
55
|
-
}
|
|
56
|
-
const detail = err instanceof Error ? err.message : 'unknown fetch error';
|
|
57
|
-
throw new Error(`failed to fetch fileUrl (redirects are rejected; this may be why): ${fileUrl} — ${detail}`, { cause: err });
|
|
58
|
-
}
|
|
59
|
-
finally {
|
|
60
|
-
clearTimeout(timer);
|
|
61
|
-
}
|
|
62
|
-
if (!res.ok) {
|
|
63
|
-
throw new Error(`fileUrl returned ${res.status} ${res.statusText}: ${fileUrl}`);
|
|
64
|
-
}
|
|
65
|
-
// Reject on a declared oversize BEFORE reading the body.
|
|
66
|
-
const declaredLength = res.headers.get('content-length');
|
|
67
|
-
if (declaredLength !== undefined && declaredLength !== null) {
|
|
68
|
-
const declared = Number(declaredLength);
|
|
69
|
-
if (Number.isFinite(declared) && declared > opts.maxBytes) {
|
|
70
|
-
throw new Error(`fileUrl response is ${declared} bytes, exceeding the ${opts.maxBytes}-byte limit (MCP_UPLOAD_MAX_FETCH_BYTES)`);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
if (!res.body) {
|
|
74
|
-
return new Uint8Array(0);
|
|
75
|
-
}
|
|
76
|
-
// Enforce the cap WHILE streaming so a source that omits or understates
|
|
77
|
-
// Content-Length can't force an unbounded read into the shared relay.
|
|
78
|
-
const reader = res.body.getReader();
|
|
79
|
-
const chunks = [];
|
|
80
|
-
let total = 0;
|
|
81
|
-
try {
|
|
82
|
-
for (;;) {
|
|
83
|
-
const { done, value } = await reader.read();
|
|
84
|
-
if (done)
|
|
85
|
-
break;
|
|
86
|
-
total += value.byteLength;
|
|
87
|
-
if (total > opts.maxBytes) {
|
|
88
|
-
await reader.cancel();
|
|
89
|
-
throw new Error(`fileUrl response exceeds the ${opts.maxBytes}-byte limit (MCP_UPLOAD_MAX_FETCH_BYTES)`);
|
|
90
|
-
}
|
|
91
|
-
chunks.push(value);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
finally {
|
|
95
|
-
reader.releaseLock();
|
|
96
|
-
}
|
|
97
|
-
const buf = new Uint8Array(total);
|
|
98
|
-
let offset = 0;
|
|
99
|
-
for (const chunk of chunks) {
|
|
100
|
-
buf.set(chunk, offset);
|
|
101
|
-
offset += chunk.byteLength;
|
|
102
|
-
}
|
|
103
|
-
return buf;
|
|
104
|
-
}
|
|
105
|
-
//# sourceMappingURL=fetch-source.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-source.js","sourceRoot":"","sources":["../../../src/tools/ingestion/fetch-source.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,4EAA4E;AAC5E,+EAA+E;AAC/E,4EAA4E;AAC5E,wDAAwD;AACxD,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,gEAAgE;AAYhE;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,IAAwB;IAExB,IAAI,GAAQ,CAAA;IACZ,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,wBAAwB,CAAC,CAAA;IACtE,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,oBAAoB,OAAO,uCAAuC,GAAG,CAAC,QAAQ,GAAG,CAClF,CAAA;IACH,CAAC;IAED,oEAAoE;IACpE,iEAAiE;IACjE,sEAAsE;IACtE,8EAA8E;IAC9E,6EAA6E;IAC7E,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,CAAA;IAC3E,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,iBAAiB,GAAG,CAAC,QAAQ,mCAAmC;YAC9D,+DAA+D,CAClE,CAAA;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IAElE,IAAI,GAAa,CAAA;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YACzB,wEAAwE;YACxE,kEAAkE;YAClE,kEAAkE;YAClE,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,oCAAoC,IAAI,CAAC,SAAS,OAAO,OAAO,EAAE,EAClE,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAA;QACH,CAAC;QACD,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAA;QACzE,MAAM,IAAI,KAAK,CACb,sEAAsE,OAAO,MAAM,MAAM,EAAE,EAC3F,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAA;IACH,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAA;IACrB,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,oBAAoB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,KAAK,OAAO,EAAE,CAC/D,CAAA;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;IACxD,IAAI,cAAc,KAAK,SAAS,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC,CAAA;QACvC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CACb,uBAAuB,QAAQ,yBAAyB,IAAI,CAAC,QAAQ,0CAA0C,CAChH,CAAA;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,CAAA;IAC1B,CAAC;IAED,wEAAwE;IACxE,sEAAsE;IACtE,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAA;IACnC,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,CAAC;QACH,SAAS,CAAC;YACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YAC3C,IAAI,IAAI;gBAAE,MAAK;YACf,KAAK,IAAI,KAAK,CAAC,UAAU,CAAA;YACzB,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC1B,MAAM,MAAM,CAAC,MAAM,EAAE,CAAA;gBACrB,MAAM,IAAI,KAAK,CACb,gCAAgC,IAAI,CAAC,QAAQ,0CAA0C,CACxF,CAAA;YACH,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,WAAW,EAAE,CAAA;IACtB,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAA;IACjC,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACtB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAA;IAC5B,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC"}
|