@apifuse/provider-sdk 2.2.0-beta.58 → 2.2.0-beta.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +25 -0
- package/CHANGELOG.md +8 -0
- package/README.md +18 -2
- package/bin/apifuse-dev.ts +2 -1
- package/bin/apifuse-pack-types.ts +49 -0
- package/bin/apifuse-perf.ts +41 -4
- package/bin/apifuse-record.ts +9 -2
- package/dist/cli/templates/provider/README.md.tpl +15 -3
- package/dist/config/loader.d.ts +7 -0
- package/dist/config/loader.js +43 -18
- package/dist/declaration-validation.d.ts +2 -1
- package/dist/declaration-validation.js +39 -0
- package/dist/define.d.ts +4 -4
- package/dist/define.js +25 -1
- package/dist/engine.d.ts +16 -0
- package/dist/engine.js +59 -3
- package/dist/error-resolution.js +13 -0
- package/dist/index.d.ts +7 -4
- package/dist/index.js +4 -2
- package/dist/lint.js +38 -17
- package/dist/provider.d.ts +3 -3
- package/dist/provider.js +2 -2
- package/dist/runtime/bounded-expiring-map.d.ts +21 -0
- package/dist/runtime/bounded-expiring-map.js +64 -0
- package/dist/runtime/browser.js +2 -1
- package/dist/runtime/cache.d.ts +2 -0
- package/dist/runtime/cache.js +11 -3
- package/dist/runtime/diagnostic-env-values.d.ts +5 -0
- package/dist/runtime/diagnostic-env-values.js +57 -0
- package/dist/runtime/diagnostic-env.d.ts +11 -0
- package/dist/runtime/diagnostic-env.js +35 -0
- package/dist/runtime/diagnostic-redactor.d.ts +49 -0
- package/dist/runtime/diagnostic-redactor.js +569 -0
- package/dist/runtime/egress-lease.d.ts +65 -0
- package/dist/runtime/egress-lease.js +236 -0
- package/dist/runtime/env.js +2 -1
- package/dist/runtime/http.js +1 -0
- package/dist/runtime/instrumentation.js +3 -0
- package/dist/runtime/ocr.js +2 -1
- package/dist/runtime/otlp.js +5 -4
- package/dist/runtime/proxy-nodemaven.js +3 -2
- package/dist/runtime/proxy-retry-policy.d.ts +4 -1
- package/dist/runtime/proxy-retry-policy.js +11 -2
- package/dist/runtime/request-options.d.ts +7 -0
- package/dist/runtime/request-options.js +62 -22
- package/dist/runtime/request-telemetry.d.ts +7 -9
- package/dist/runtime/resolver-config.d.ts +7 -0
- package/dist/runtime/resolver-config.js +8 -0
- package/dist/runtime/resolver-public.d.ts +3 -1
- package/dist/runtime/resolver-public.js +1 -1
- package/dist/runtime/resolver-shared.d.ts +9 -0
- package/dist/runtime/resolver-shared.js +57 -0
- package/dist/runtime/resolver-telemetry.d.ts +134 -0
- package/dist/runtime/resolver-telemetry.js +204 -0
- package/dist/runtime/resolver-usage.d.ts +24 -0
- package/dist/runtime/resolver-usage.js +60 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +5 -0
- package/dist/runtime/resolver-vendors/bindings.js +5 -0
- package/dist/runtime/resolver-vendors/browser.js +0 -1
- package/dist/runtime/resolver-vendors/capsolver.d.ts +2 -2
- package/dist/runtime/resolver-vendors/capsolver.js +21 -9
- package/dist/runtime/resolver-vendors/hypersolutions.d.ts +27 -0
- package/dist/runtime/resolver-vendors/hypersolutions.js +369 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +2 -2
- package/dist/runtime/resolver-vendors/twocaptcha.js +26 -10
- package/dist/runtime/resolver-vendors/types.d.ts +35 -5
- package/dist/runtime/resolver-vendors/types.js +20 -10
- package/dist/runtime/resolver.d.ts +5 -1
- package/dist/runtime/resolver.js +400 -111
- package/dist/runtime/stealth-akamai-sbsd.d.ts +54 -0
- package/dist/runtime/stealth-akamai-sbsd.js +142 -0
- package/dist/runtime/stealth.js +547 -53
- package/dist/runtime/stt.js +3 -2
- package/dist/runtime/trace.d.ts +5 -0
- package/dist/runtime/trace.js +115 -20
- package/dist/schema.d.ts +20 -0
- package/dist/schema.js +72 -2
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +1 -0
- package/dist/server/self-test-redaction.js +8 -33
- package/dist/server/self-test.d.ts +4 -0
- package/dist/server/self-test.js +10 -1
- package/dist/server/sensitive-values.d.ts +26 -0
- package/dist/server/sensitive-values.js +131 -0
- package/dist/server/serve-implementation.d.ts +7 -1
- package/dist/server/serve-implementation.js +394 -125
- package/dist/server/trace-output.d.ts +2 -1
- package/dist/server/trace-output.js +46 -14
- package/dist/server/types.d.ts +6 -0
- package/dist/server/types.js +14 -0
- package/dist/stealth/profiles.d.ts +7 -1
- package/dist/stealth/profiles.js +9 -0
- package/dist/testing/run.js +6 -0
- package/dist/trace-sanitization.d.ts +4 -3
- package/dist/trace-sanitization.js +45 -22
- package/dist/types.d.ts +106 -10
- package/package.json +2 -1
- package/src/cli/templates/provider/README.md.tpl +15 -3
- package/src/config/loader.ts +47 -18
- package/src/declaration-validation.ts +47 -1
- package/src/define.ts +45 -3
- package/src/engine.ts +78 -3
- package/src/error-resolution.ts +13 -0
- package/src/index.ts +34 -1
- package/src/lint.ts +44 -18
- package/src/provider.ts +10 -0
- package/src/runtime/bounded-expiring-map.ts +67 -0
- package/src/runtime/browser.ts +2 -1
- package/src/runtime/cache.ts +13 -3
- package/src/runtime/diagnostic-env-values.ts +56 -0
- package/src/runtime/diagnostic-env.ts +55 -0
- package/src/runtime/diagnostic-redactor.ts +662 -0
- package/src/runtime/egress-lease.ts +341 -0
- package/src/runtime/env.ts +2 -1
- package/src/runtime/http.ts +1 -0
- package/src/runtime/instrumentation.ts +3 -0
- package/src/runtime/ocr.ts +2 -1
- package/src/runtime/otlp.ts +5 -4
- package/src/runtime/proxy-nodemaven.ts +3 -2
- package/src/runtime/proxy-retry-policy.ts +12 -1
- package/src/runtime/request-options.ts +76 -28
- package/src/runtime/request-telemetry.ts +5 -7
- package/src/runtime/resolver-config.ts +12 -0
- package/src/runtime/resolver-public.ts +21 -0
- package/src/runtime/resolver-shared.ts +75 -0
- package/src/runtime/resolver-telemetry.ts +362 -0
- package/src/runtime/resolver-usage.ts +96 -0
- package/src/runtime/resolver-vendors/bindings.ts +5 -0
- package/src/runtime/resolver-vendors/browser.ts +0 -1
- package/src/runtime/resolver-vendors/capsolver.ts +22 -3
- package/src/runtime/resolver-vendors/hypersolutions.ts +480 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +26 -12
- package/src/runtime/resolver-vendors/types.ts +60 -12
- package/src/runtime/resolver.ts +468 -126
- package/src/runtime/stealth-akamai-sbsd.ts +207 -0
- package/src/runtime/stealth.ts +727 -95
- package/src/runtime/stt.ts +3 -2
- package/src/runtime/trace.ts +168 -26
- package/src/schema.ts +80 -2
- package/src/server/index.ts +21 -0
- package/src/server/self-test-redaction.ts +10 -40
- package/src/server/self-test.ts +13 -0
- package/src/server/sensitive-values.ts +200 -0
- package/src/server/serve-implementation.ts +561 -136
- package/src/server/trace-output.ts +77 -17
- package/src/server/types.ts +14 -0
- package/src/stealth/profiles.ts +10 -0
- package/src/testing/run.ts +6 -0
- package/src/trace-sanitization.ts +70 -23
- package/src/types.ts +124 -19
package/src/runtime/stt.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readDiagnosticEnv } from "./diagnostic-env.js";
|
|
1
2
|
import { ProviderError, TransportError, ValidationError } from "../errors.js";
|
|
2
3
|
import type {
|
|
3
4
|
Bcp47Locale,
|
|
@@ -69,7 +70,7 @@ export function createUnsupportedSttClient(reason?: string): SttContext {
|
|
|
69
70
|
}
|
|
70
71
|
|
|
71
72
|
function normalizedEnvValue(env: EnvLike, key: string): string | undefined {
|
|
72
|
-
const value = env
|
|
73
|
+
const value = readDiagnosticEnv(key, env)?.trim();
|
|
73
74
|
return value ? value : undefined;
|
|
74
75
|
}
|
|
75
76
|
|
|
@@ -99,7 +100,7 @@ export function createSttClientFromEnv(
|
|
|
99
100
|
}
|
|
100
101
|
|
|
101
102
|
const accountId = env.APIFUSE__CLOUDFLARE__ACCOUNT_ID?.trim();
|
|
102
|
-
const apiToken = env
|
|
103
|
+
const apiToken = readDiagnosticEnv(APIFUSE__STT__CLOUDFLARE_API_TOKEN_ENV, env)?.trim();
|
|
103
104
|
if (!accountId || !apiToken) {
|
|
104
105
|
return createUnsupportedSttClient(
|
|
105
106
|
`STT backend ${backend} requires ${CLOUDFLARE_ACCOUNT_ID_ENV} and ${APIFUSE__STT__CLOUDFLARE_API_TOKEN_ENV}`,
|
package/src/runtime/trace.ts
CHANGED
|
@@ -1,12 +1,25 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
-
|
|
2
|
+
import { sanitizeSpanNameForOutput } from "../trace-sanitization.js";
|
|
3
3
|
import type {
|
|
4
4
|
TraceContext as BaseTraceContext,
|
|
5
5
|
TraceAttributeValue,
|
|
6
6
|
TraceConfig,
|
|
7
7
|
TraceSpan,
|
|
8
8
|
} from "../types.js";
|
|
9
|
+
import {
|
|
10
|
+
type DiagnosticRedactor,
|
|
11
|
+
diagnosticStructuredRedactor,
|
|
12
|
+
redactDiagnosticAttributeKey,
|
|
13
|
+
copyDiagnosticAttributeTypes,
|
|
14
|
+
redactedKeyAllocator,
|
|
15
|
+
isRedactedKey,
|
|
16
|
+
redactCriticalDiagnosticText,
|
|
17
|
+
diagnosticAttributeRedactor,
|
|
18
|
+
REDACTION_FAILED,
|
|
19
|
+
redactDiagnosticText,
|
|
20
|
+
} from "./diagnostic-redactor.js";
|
|
9
21
|
import { exportSpansOTLP, type OTLPExportOptions } from "./otlp.js";
|
|
22
|
+
import { REDACTED_QUERY_VALUE } from "./request-options.js";
|
|
10
23
|
|
|
11
24
|
export type SpanAttributeValue = TraceAttributeValue;
|
|
12
25
|
|
|
@@ -19,6 +32,8 @@ export interface TraceContext extends BaseTraceContext {
|
|
|
19
32
|
export interface CreateTraceContextOptions {
|
|
20
33
|
maxSpans?: number;
|
|
21
34
|
onSpan?: (span: Span) => void;
|
|
35
|
+
/** Redacts diagnostic attribute values and errors before recording or calling onSpan. */
|
|
36
|
+
redact?: DiagnosticRedactor;
|
|
22
37
|
exportOptions?: OTLPExportOptions;
|
|
23
38
|
resourceAttributes?: Record<string, string>;
|
|
24
39
|
/** W3C-compatible 32-character lowercase hexadecimal trace id used for export. */
|
|
@@ -42,7 +57,7 @@ type PendingSpan = {
|
|
|
42
57
|
startedAt: number;
|
|
43
58
|
parentId?: string;
|
|
44
59
|
sequence: number;
|
|
45
|
-
attributes: Record<string, SpanAttributeValue>;
|
|
60
|
+
attributes: Record<string, SpanAttributeValue | bigint>;
|
|
46
61
|
};
|
|
47
62
|
|
|
48
63
|
type CompletedSpanEntry = {
|
|
@@ -52,14 +67,22 @@ type CompletedSpanEntry = {
|
|
|
52
67
|
exported: boolean;
|
|
53
68
|
};
|
|
54
69
|
|
|
70
|
+
const TRACE_RESOURCE_ATTRIBUTE_SANITIZER = Symbol.for(
|
|
71
|
+
"@apifuse/provider-sdk/runtime/trace-resource-attribute-sanitizer",
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
type ResourceAttributesWithSanitizer = Record<string, string> & {
|
|
75
|
+
[TRACE_RESOURCE_ATTRIBUTE_SANITIZER]?: (
|
|
76
|
+
attributes: Record<string, string>,
|
|
77
|
+
) => Record<string, string>;
|
|
78
|
+
};
|
|
79
|
+
|
|
55
80
|
export interface TraceRecorder {
|
|
56
81
|
runSpan<T>(name: string, fn: () => Promise<T> | T, options?: SpanHookOptions<T>): Promise<T>;
|
|
57
82
|
}
|
|
58
83
|
|
|
59
84
|
export const TRACE_RECORDER = Symbol.for("@apifuse/provider-sdk/runtime/trace-recorder");
|
|
60
|
-
const TRACE_EXPORT_METADATA = Symbol.for(
|
|
61
|
-
"@apifuse/provider-sdk/runtime/trace-export-metadata",
|
|
62
|
-
);
|
|
85
|
+
const TRACE_EXPORT_METADATA = Symbol.for("@apifuse/provider-sdk/runtime/trace-export-metadata");
|
|
63
86
|
|
|
64
87
|
type TraceExportMetadata = {
|
|
65
88
|
update(input: { traceId?: string; resourceAttributes?: Record<string, string> }): void;
|
|
@@ -109,30 +132,104 @@ export function resolveTraceContextOptions(config?: TraceConfig): CreateTraceCon
|
|
|
109
132
|
};
|
|
110
133
|
}
|
|
111
134
|
|
|
112
|
-
function normalizeAttributeValue(
|
|
113
|
-
|
|
114
|
-
|
|
135
|
+
function normalizeAttributeValue(
|
|
136
|
+
value: unknown,
|
|
137
|
+
redact?: DiagnosticRedactor,
|
|
138
|
+
retainBigints = false,
|
|
139
|
+
): SpanAttributeValue | bigint | undefined {
|
|
140
|
+
if (typeof value === "number" || typeof value === "boolean") {
|
|
141
|
+
// Typed columns are contracts: unlike string tokens, number/boolean/bigint
|
|
142
|
+
// values only consult the registry at >=8 characters in string form.
|
|
143
|
+
const text = String(value);
|
|
144
|
+
if (text.length < 8) return value;
|
|
145
|
+
const redacted = redactDiagnosticText(text, diagnosticStructuredRedactor(redact));
|
|
146
|
+
return redacted === text ? value : redacted;
|
|
147
|
+
}
|
|
148
|
+
if (typeof value === "bigint") {
|
|
149
|
+
// Bigints are not valid OTLP attribute primitives, so retain their prior
|
|
150
|
+
// string normalization while still checking their diagnostic form.
|
|
151
|
+
const text = String(value);
|
|
152
|
+
if (text.length < 8) return retainBigints ? value : text;
|
|
153
|
+
const redacted = redactDiagnosticText(text, diagnosticStructuredRedactor(redact));
|
|
154
|
+
return retainBigints && redacted === text ? value : redacted;
|
|
115
155
|
}
|
|
116
156
|
|
|
117
157
|
if (value === null || value === undefined) {
|
|
118
158
|
return undefined;
|
|
119
159
|
}
|
|
120
160
|
|
|
121
|
-
return String(value);
|
|
161
|
+
return redactDiagnosticText(String(value), redact);
|
|
122
162
|
}
|
|
123
163
|
|
|
164
|
+
function normalizeAttributes(
|
|
165
|
+
attributes: Record<string, unknown> | undefined,
|
|
166
|
+
redact: DiagnosticRedactor | undefined,
|
|
167
|
+
retainBigints: true,
|
|
168
|
+
): Record<string, SpanAttributeValue | bigint>;
|
|
169
|
+
function normalizeAttributes(
|
|
170
|
+
attributes?: Record<string, unknown>,
|
|
171
|
+
redact?: DiagnosticRedactor,
|
|
172
|
+
): Record<string, SpanAttributeValue>;
|
|
124
173
|
function normalizeAttributes(
|
|
125
174
|
attributes?: Record<string, unknown>,
|
|
126
|
-
|
|
175
|
+
redact?: DiagnosticRedactor,
|
|
176
|
+
retainBigints = false,
|
|
177
|
+
): Record<string, SpanAttributeValue | bigint> {
|
|
127
178
|
if (!attributes) {
|
|
128
179
|
return {};
|
|
129
180
|
}
|
|
130
181
|
|
|
182
|
+
const nextKey = redactedKeyAllocator(Object.keys(attributes));
|
|
131
183
|
const normalizedEntries = Object.entries(attributes)
|
|
132
|
-
.map(([key, value]) =>
|
|
133
|
-
|
|
184
|
+
.map(([key, value]) => {
|
|
185
|
+
const redactedKey = isRedactedKey(key) ? key : redactDiagnosticAttributeKey(key, redact);
|
|
186
|
+
const keyChanged = redactedKey !== key;
|
|
187
|
+
const normalizedKey = keyChanged ? nextKey() : key;
|
|
188
|
+
// Once a key changes, its original sensitivity classification is no
|
|
189
|
+
// longer available to later output sanitizers. Suppress the value here.
|
|
190
|
+
const normalizedValue = keyChanged
|
|
191
|
+
? redactedKey === REDACTION_FAILED
|
|
192
|
+
? REDACTION_FAILED
|
|
193
|
+
: REDACTED_QUERY_VALUE
|
|
194
|
+
: normalizeAttributeValue(value, diagnosticAttributeRedactor(key, redact), retainBigints);
|
|
195
|
+
return [normalizedKey, normalizedValue] as const;
|
|
196
|
+
})
|
|
197
|
+
.filter(
|
|
198
|
+
(entry): entry is readonly [string, SpanAttributeValue | bigint] => entry[1] !== undefined,
|
|
199
|
+
);
|
|
200
|
+
|
|
201
|
+
return copyDiagnosticAttributeTypes(attributes, Object.fromEntries(normalizedEntries));
|
|
202
|
+
}
|
|
134
203
|
|
|
135
|
-
|
|
204
|
+
function prepareResourceAttributesForExport(
|
|
205
|
+
attributes: Record<string, string> | undefined,
|
|
206
|
+
redact?: DiagnosticRedactor,
|
|
207
|
+
sanitize?: (attributes: Record<string, string>) => Record<string, string>,
|
|
208
|
+
): Record<string, string> | undefined {
|
|
209
|
+
if (!attributes) return undefined;
|
|
210
|
+
if (sanitize) {
|
|
211
|
+
const sanitized = sanitize({ ...attributes });
|
|
212
|
+
if (!sanitized || typeof sanitized !== "object") {
|
|
213
|
+
throw new Error("resource attribute sanitizer returned no attributes");
|
|
214
|
+
}
|
|
215
|
+
return sanitized;
|
|
216
|
+
}
|
|
217
|
+
const nextKey = redactedKeyAllocator(Object.keys(attributes));
|
|
218
|
+
return Object.fromEntries(
|
|
219
|
+
Object.entries(attributes).map(([key, value]) => {
|
|
220
|
+
const redactedKey = isRedactedKey(key) ? key : redactDiagnosticAttributeKey(key, redact);
|
|
221
|
+
const keyChanged = redactedKey !== key;
|
|
222
|
+
const sanitizedKey = keyChanged ? nextKey() : sanitizeSpanNameForOutput(key);
|
|
223
|
+
return [
|
|
224
|
+
sanitizedKey,
|
|
225
|
+
keyChanged
|
|
226
|
+
? redactedKey === REDACTION_FAILED
|
|
227
|
+
? REDACTION_FAILED
|
|
228
|
+
: REDACTED_QUERY_VALUE
|
|
229
|
+
: sanitizeSpanNameForOutput(value, diagnosticAttributeRedactor(key, redact)),
|
|
230
|
+
];
|
|
231
|
+
}),
|
|
232
|
+
);
|
|
136
233
|
}
|
|
137
234
|
|
|
138
235
|
function insertCompletedSpan(
|
|
@@ -158,7 +255,10 @@ function prepareSpanForExport(
|
|
|
158
255
|
span: Span,
|
|
159
256
|
sanitize: CreateTraceContextOptions["sanitizeSpanForExport"],
|
|
160
257
|
): Span {
|
|
161
|
-
const copy: Span = {
|
|
258
|
+
const copy: Span = {
|
|
259
|
+
...span,
|
|
260
|
+
attributes: copyDiagnosticAttributeTypes(span.attributes, { ...span.attributes }),
|
|
261
|
+
};
|
|
162
262
|
if (!sanitize) return copy;
|
|
163
263
|
const sanitized = sanitize(copy);
|
|
164
264
|
if (!sanitized || typeof sanitized !== "object") {
|
|
@@ -171,6 +271,25 @@ export function getTraceRecorder(trace: BaseTraceContext): TraceRecorder | null
|
|
|
171
271
|
return (trace as Partial<InternalTraceContext>)[TRACE_RECORDER] ?? null;
|
|
172
272
|
}
|
|
173
273
|
|
|
274
|
+
const droppedOTLPSpans = new Map<string, number>();
|
|
275
|
+
|
|
276
|
+
/** Internal, read-only process counters; no reset can re-arm a warning. */
|
|
277
|
+
export function getDroppedOTLPSpanCount(reason: string): number {
|
|
278
|
+
return droppedOTLPSpans.get(reason) ?? 0;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function warnDroppedOTLPSpans(reason: string, count: number): void {
|
|
282
|
+
const previous = getDroppedOTLPSpanCount(reason);
|
|
283
|
+
const total = previous + count;
|
|
284
|
+
droppedOTLPSpans.set(reason, total);
|
|
285
|
+
if (previous !== 0) return;
|
|
286
|
+
try {
|
|
287
|
+
console.warn(`[apifuse] OTLP export skipped; reason=${reason}; dropped_spans=${total}`);
|
|
288
|
+
} catch {
|
|
289
|
+
// Warning sinks are best-effort. The counter/latch survives a failed sink.
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
174
293
|
export function createTraceContext(options: CreateTraceContextOptions = {}): TraceContext {
|
|
175
294
|
if (options.traceId !== undefined) assertValidTraceId(options.traceId);
|
|
176
295
|
const maxSpans = options.maxSpans ?? 1000;
|
|
@@ -180,9 +299,14 @@ export function createTraceContext(options: CreateTraceContextOptions = {}): Tra
|
|
|
180
299
|
// Export configuration (which can carry collector credentials) stays in this closure;
|
|
181
300
|
// the context object handed to provider code never exposes it.
|
|
182
301
|
const exportOptions = options.exportOptions;
|
|
183
|
-
const
|
|
184
|
-
|
|
302
|
+
const resourceAttributeOptions = options.resourceAttributes as
|
|
303
|
+
| ResourceAttributesWithSanitizer
|
|
304
|
+
| undefined;
|
|
305
|
+
const exportResourceAttributes = resourceAttributeOptions
|
|
306
|
+
? { ...resourceAttributeOptions }
|
|
185
307
|
: undefined;
|
|
308
|
+
const exportResourceAttributeSanitizer =
|
|
309
|
+
resourceAttributeOptions?.[TRACE_RESOURCE_ATTRIBUTE_SANITIZER];
|
|
186
310
|
// One trace id per context so every export batch of this request shares it and
|
|
187
311
|
// two processes can never mint the same id.
|
|
188
312
|
let exportTraceId = options.traceId ?? crypto.randomUUID().replace(/-/g, "");
|
|
@@ -203,15 +327,30 @@ export function createTraceContext(options: CreateTraceContextOptions = {}): Tra
|
|
|
203
327
|
if (pending.length === 0) return;
|
|
204
328
|
// Sanitization runs off the request path; a faulty sanitizer drops the batch, never the request.
|
|
205
329
|
let spans: Span[];
|
|
330
|
+
let resourceAttributes: Record<string, string> | undefined;
|
|
206
331
|
try {
|
|
207
332
|
spans = pending.map((entry) =>
|
|
208
333
|
prepareSpanForExport(entry.span, options.sanitizeSpanForExport),
|
|
209
334
|
);
|
|
335
|
+
resourceAttributes = prepareResourceAttributesForExport(
|
|
336
|
+
exportResourceAttributes,
|
|
337
|
+
options.redact,
|
|
338
|
+
exportResourceAttributeSanitizer,
|
|
339
|
+
);
|
|
210
340
|
} catch {
|
|
211
|
-
|
|
341
|
+
try {
|
|
342
|
+
console.warn("[apifuse] OTLP export skipped; span sanitization failed.");
|
|
343
|
+
} catch {
|
|
344
|
+
// A broken diagnostic sink must not escape this detached callback.
|
|
345
|
+
}
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
// A matching trace ID cannot be replaced with non-hex text on the wire. Emit less.
|
|
349
|
+
if (redactCriticalDiagnosticText(exportTraceId, options.redact) !== exportTraceId) {
|
|
350
|
+
warnDroppedOTLPSpans("unverifiable_trace_id", pending.length);
|
|
212
351
|
return;
|
|
213
352
|
}
|
|
214
|
-
void exportSpansOTLP(spans, exportOptions,
|
|
353
|
+
void exportSpansOTLP(spans, exportOptions, resourceAttributes, exportTraceId);
|
|
215
354
|
});
|
|
216
355
|
};
|
|
217
356
|
|
|
@@ -219,11 +358,11 @@ export function createTraceContext(options: CreateTraceContextOptions = {}): Tra
|
|
|
219
358
|
async runSpan(name, fn, spanOptions = {}) {
|
|
220
359
|
const pendingSpan: PendingSpan = {
|
|
221
360
|
id: crypto.randomUUID(),
|
|
222
|
-
name,
|
|
361
|
+
name: redactCriticalDiagnosticText(name, options.redact),
|
|
223
362
|
startedAt: Date.now(),
|
|
224
363
|
parentId: activeSpanStorage.getStore()?.id,
|
|
225
364
|
sequence: sequence++,
|
|
226
|
-
attributes: normalizeAttributes(spanOptions.attributes),
|
|
365
|
+
attributes: normalizeAttributes(spanOptions.attributes, options.redact, true),
|
|
227
366
|
};
|
|
228
367
|
|
|
229
368
|
const finalize = (
|
|
@@ -235,7 +374,7 @@ export function createTraceContext(options: CreateTraceContextOptions = {}): Tra
|
|
|
235
374
|
const duration = endedAt - pendingSpan.startedAt;
|
|
236
375
|
const attributes = {
|
|
237
376
|
...pendingSpan.attributes,
|
|
238
|
-
...
|
|
377
|
+
...extraAttributes,
|
|
239
378
|
};
|
|
240
379
|
|
|
241
380
|
if (attributes.duration_ms === undefined) {
|
|
@@ -243,15 +382,18 @@ export function createTraceContext(options: CreateTraceContextOptions = {}): Tra
|
|
|
243
382
|
}
|
|
244
383
|
|
|
245
384
|
const span: Span = {
|
|
246
|
-
id: pendingSpan.id,
|
|
247
|
-
|
|
385
|
+
id: redactCriticalDiagnosticText(pendingSpan.id, options.redact),
|
|
386
|
+
// Recheck at recording: harvest and live env reads may follow span start.
|
|
387
|
+
name: redactCriticalDiagnosticText(pendingSpan.name, options.redact),
|
|
248
388
|
startedAt: pendingSpan.startedAt,
|
|
249
389
|
endedAt,
|
|
250
390
|
duration_ms: duration,
|
|
251
391
|
status,
|
|
252
|
-
attributes,
|
|
253
|
-
...(error ? { error } : {}),
|
|
254
|
-
...(pendingSpan.parentId
|
|
392
|
+
attributes: normalizeAttributes(attributes, options.redact),
|
|
393
|
+
...(error ? { error: redactDiagnosticText(error, options.redact) } : {}),
|
|
394
|
+
...(pendingSpan.parentId
|
|
395
|
+
? { parentId: redactCriticalDiagnosticText(pendingSpan.parentId, options.redact) }
|
|
396
|
+
: {}),
|
|
255
397
|
};
|
|
256
398
|
|
|
257
399
|
insertCompletedSpan(
|
package/src/schema.ts
CHANGED
|
@@ -83,6 +83,8 @@ export function safeParseSchemaSync(
|
|
|
83
83
|
export const APIFUSE_SENSITIVE_META_KEY = "x-apifuse-sensitive";
|
|
84
84
|
export const APIFUSE_SENSITIVE_KIND_META_KEY = "x-apifuse-sensitive-kind";
|
|
85
85
|
export const APIFUSE_DESCRIPTION_KEY_META_KEY = "x-apifuse-description-key";
|
|
86
|
+
export const APIFUSE_CONTENT_TRUST_META_KEY = "x-apifuse-content-trust";
|
|
87
|
+
export const APIFUSE_CONTENT_PROVENANCE_META_KEY = "x-apifuse-content-provenance";
|
|
86
88
|
export const APIFUSE_REDACTION_MARKER = "<redacted>";
|
|
87
89
|
|
|
88
90
|
export type SensitivePathSegment = string | "*";
|
|
@@ -103,6 +105,8 @@ export type SensitiveFieldKind =
|
|
|
103
105
|
export interface SensitiveFieldOptions {
|
|
104
106
|
/**
|
|
105
107
|
* Mark this schema as sensitive. Defaults to true for the helper presets.
|
|
108
|
+
* An explicit `false` emits `x-apifuse-sensitive: false` (reviewed public
|
|
109
|
+
* data, never redacted) and drops any sensitivity kind.
|
|
106
110
|
*/
|
|
107
111
|
sensitive?: boolean;
|
|
108
112
|
/**
|
|
@@ -127,6 +131,21 @@ export function describeKey<TSchema extends ZodType>(
|
|
|
127
131
|
});
|
|
128
132
|
}
|
|
129
133
|
|
|
134
|
+
/**
|
|
135
|
+
* Mark an output field as externally authored text (listing bodies, reviews,
|
|
136
|
+
* posts) that the provider relays verbatim. Sets both content-trust JSON Schema
|
|
137
|
+
* meta keys on the field node so the platform can enforce the boundary when it
|
|
138
|
+
* assembles tool output; validation and the parsed value are unchanged.
|
|
139
|
+
*/
|
|
140
|
+
export function untrustedContent<TSchema extends ZodType>(schema: TSchema): TSchema {
|
|
141
|
+
const metadata = schema.meta() ?? {};
|
|
142
|
+
return schema.meta({
|
|
143
|
+
...metadata,
|
|
144
|
+
[APIFUSE_CONTENT_TRUST_META_KEY]: "untrusted",
|
|
145
|
+
[APIFUSE_CONTENT_PROVENANCE_META_KEY]: "external",
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
|
|
130
149
|
declare module "zod" {
|
|
131
150
|
interface ZodType {
|
|
132
151
|
describeKey(key: ProviderLocaleKey | string): this;
|
|
@@ -191,10 +210,27 @@ export function field<TSchema extends ZodType>(
|
|
|
191
210
|
? schema.describe(options.description)
|
|
192
211
|
: schema;
|
|
193
212
|
const metadata = described.meta() ?? {};
|
|
213
|
+
const isSensitive = options.sensitive ?? true;
|
|
214
|
+
// A public declaration only rewrites the meta of the node it receives;
|
|
215
|
+
// z.toJSONSchema folds that over the inner leaf, so `false` on a wrapper
|
|
216
|
+
// around a sensitive() leaf would publish `false` next to the leaf's kind
|
|
217
|
+
// while collectSensitivePaths() still redacts. Refuse the contradiction.
|
|
218
|
+
if (!isSensitive && wrapsSensitiveDeclaration(described)) {
|
|
219
|
+
throw new TypeError(
|
|
220
|
+
"publicField()/field({ sensitive: false }) wraps a sensitive() declaration; apply it to the leaf, before .optional(), .nullable(), .default(), .pipe(), z.lazy() or similar wrappers.",
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
// Zod merges meta with the parent chain, so an inherited kind can only be
|
|
224
|
+
// cleared by writing undefined over it (dropped from JSON Schema output).
|
|
225
|
+
const kind = isSensitive
|
|
226
|
+
? (options.kind ?? Reflect.get(metadata, APIFUSE_SENSITIVE_KIND_META_KEY))
|
|
227
|
+
: undefined;
|
|
194
228
|
return described.meta({
|
|
195
229
|
...metadata,
|
|
196
|
-
|
|
197
|
-
...(
|
|
230
|
+
[APIFUSE_SENSITIVE_META_KEY]: isSensitive,
|
|
231
|
+
...(kind !== undefined || APIFUSE_SENSITIVE_KIND_META_KEY in metadata
|
|
232
|
+
? { [APIFUSE_SENSITIVE_KIND_META_KEY]: kind }
|
|
233
|
+
: {}),
|
|
198
234
|
});
|
|
199
235
|
}
|
|
200
236
|
|
|
@@ -205,6 +241,18 @@ export function sensitive<TSchema extends ZodType>(
|
|
|
205
241
|
return field(schema, { sensitive: true, kind });
|
|
206
242
|
}
|
|
207
243
|
|
|
244
|
+
/**
|
|
245
|
+
* Declare a field as reviewed public data (business, facility, product, or
|
|
246
|
+
* place names and contacts) so a key-based privacy classifier hit counts as
|
|
247
|
+
* declared instead of undeclared. Emits `x-apifuse-sensitive: false` on the
|
|
248
|
+
* JSON Schema leaf; nothing is redacted. Use `sensitive()` for anything that
|
|
249
|
+
* identifies a person. Apply it to the leaf, before `.optional()` or
|
|
250
|
+
* `.nullable()`; wrapping a `sensitive()` declaration throws at definition time.
|
|
251
|
+
*/
|
|
252
|
+
export function publicField<TSchema extends ZodType>(schema: TSchema): TSchema {
|
|
253
|
+
return field(schema, { sensitive: false });
|
|
254
|
+
}
|
|
255
|
+
|
|
208
256
|
function sensitiveString(
|
|
209
257
|
kind: SensitiveFieldKind,
|
|
210
258
|
description: string,
|
|
@@ -365,6 +413,36 @@ function readZodMetadata(schema: unknown): object | undefined {
|
|
|
365
413
|
return metadata && typeof metadata === "object" ? metadata : undefined;
|
|
366
414
|
}
|
|
367
415
|
|
|
416
|
+
// Same-path wrappers (optional, nullable, default, catch, readonly, pipe,
|
|
417
|
+
// lazy, ...) share the JSON Schema leaf with the schema they wrap.
|
|
418
|
+
function wrapsSensitiveDeclaration(schema: unknown, seen = new Set<unknown>()): boolean {
|
|
419
|
+
const def = readZodDef(schema);
|
|
420
|
+
if (!def || seen.has(schema)) return false;
|
|
421
|
+
seen.add(schema);
|
|
422
|
+
return sameLeafChildren(def).some(
|
|
423
|
+
(child) => isSensitiveSchema(child) || wrapsSensitiveDeclaration(child, seen),
|
|
424
|
+
);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
function sameLeafChildren(def: object): unknown[] {
|
|
428
|
+
switch (Reflect.get(def, "type")) {
|
|
429
|
+
case "pipe":
|
|
430
|
+
return [Reflect.get(def, "in"), Reflect.get(def, "out")];
|
|
431
|
+
case "lazy": {
|
|
432
|
+
// A forward-reference getter can throw before its target is
|
|
433
|
+
// initialised; treat that as no declaration rather than failing.
|
|
434
|
+
const getter = Reflect.get(def, "getter");
|
|
435
|
+
try {
|
|
436
|
+
return typeof getter === "function" ? [getter()] : [];
|
|
437
|
+
} catch {
|
|
438
|
+
return [];
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
default:
|
|
442
|
+
return [Reflect.get(def, "innerType")];
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
368
446
|
function readZodDef(schema: unknown): object | undefined {
|
|
369
447
|
if (!schema || typeof schema !== "object") return undefined;
|
|
370
448
|
const def = Reflect.get(schema, "def") ?? Reflect.get(schema, "_def");
|
package/src/server/index.ts
CHANGED
|
@@ -23,6 +23,27 @@ export {
|
|
|
23
23
|
type TelemetryKey,
|
|
24
24
|
type TenantNeutral,
|
|
25
25
|
} from "../runtime/request-telemetry.js";
|
|
26
|
+
export {
|
|
27
|
+
ResolverTelemetryCollector,
|
|
28
|
+
type ResolverAttemptSample,
|
|
29
|
+
type ResolverCacheReadTelemetryEvent,
|
|
30
|
+
type ResolverCacheWriteTelemetryEvent,
|
|
31
|
+
type ResolverFailoverTelemetryEvent,
|
|
32
|
+
type ResolverIdentityTelemetryEvent,
|
|
33
|
+
type ResolverOutcomeTelemetryEvent,
|
|
34
|
+
type ResolverTelemetryAttemptOutcome,
|
|
35
|
+
type ResolverTelemetryCacheStatus,
|
|
36
|
+
type ResolverTelemetryCacheWriteReason,
|
|
37
|
+
type ResolverTelemetryErrorClass,
|
|
38
|
+
type ResolverTelemetryHeaderPayload,
|
|
39
|
+
type ResolverTelemetryIdentitySource,
|
|
40
|
+
type ResolverTelemetryLogPayload,
|
|
41
|
+
type ResolverTelemetryOutcome,
|
|
42
|
+
type ResolverTelemetryPhase,
|
|
43
|
+
type ResolverTelemetrySink,
|
|
44
|
+
type ResolverVendorAttemptTelemetryEvent,
|
|
45
|
+
} from "../runtime/resolver-telemetry.js";
|
|
46
|
+
export type { ResolverVendorUnavailableReason } from "../runtime/resolver-vendors/types.js";
|
|
26
47
|
export type { Span, TraceContext } from "../runtime/trace.js";
|
|
27
48
|
export {
|
|
28
49
|
createServerApp,
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { ProviderDefinition } from "../types.js";
|
|
2
2
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
} from "./
|
|
3
|
+
collectDiagnosticSensitiveValues,
|
|
4
|
+
SELF_TEST_SENSITIVE_SOURCE_KINDS,
|
|
5
|
+
} from "./sensitive-values.js";
|
|
6
6
|
|
|
7
7
|
export const SELF_TEST_REDACTED_PLACEHOLDER = "[REDACTED]";
|
|
8
8
|
|
|
@@ -31,39 +31,12 @@ export function collectSelfTestSensitiveValues(
|
|
|
31
31
|
credentialInputs?: Readonly<Record<string, string>>;
|
|
32
32
|
} = {},
|
|
33
33
|
): string[] {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
envNames.add(secret.name);
|
|
41
|
-
}
|
|
42
|
-
const healthProbe = provider.healthProbe ?? provider.healthMonitor;
|
|
43
|
-
for (const name of healthProbe?.requiredSecrets ?? []) {
|
|
44
|
-
envNames.add(name);
|
|
45
|
-
}
|
|
46
|
-
for (const name of Object.values(healthProbe?.credentialInputs ?? {})) {
|
|
47
|
-
envNames.add(name);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
const values = new Set<string>();
|
|
51
|
-
for (const name of envNames) {
|
|
52
|
-
const value = env[name];
|
|
53
|
-
if (typeof value === "string" && value.length >= MIN_SENSITIVE_VALUE_LENGTH) {
|
|
54
|
-
values.add(value);
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
for (const value of Object.values(options.credentialInputs ?? {})) {
|
|
58
|
-
if (value.length >= MIN_SENSITIVE_VALUE_LENGTH) {
|
|
59
|
-
values.add(value);
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
return [...values];
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function escapeRegExp(value: string): string {
|
|
66
|
-
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
34
|
+
return collectDiagnosticSensitiveValues(provider, {
|
|
35
|
+
...options,
|
|
36
|
+
sourceKinds: SELF_TEST_SENSITIVE_SOURCE_KINDS,
|
|
37
|
+
minimumLength: MIN_SENSITIVE_VALUE_LENGTH,
|
|
38
|
+
includeResolvedEnvVariants: false,
|
|
39
|
+
});
|
|
67
40
|
}
|
|
68
41
|
|
|
69
42
|
/**
|
|
@@ -79,10 +52,7 @@ export function redactSelfTestText(
|
|
|
79
52
|
let redacted = text;
|
|
80
53
|
for (const value of sensitiveValues) {
|
|
81
54
|
if (value.length < MIN_SENSITIVE_VALUE_LENGTH) continue;
|
|
82
|
-
redacted = redacted.
|
|
83
|
-
new RegExp(escapeRegExp(value), "g"),
|
|
84
|
-
SELF_TEST_REDACTED_PLACEHOLDER,
|
|
85
|
-
);
|
|
55
|
+
redacted = redacted.replaceAll(value, SELF_TEST_REDACTED_PLACEHOLDER);
|
|
86
56
|
}
|
|
87
57
|
for (const pattern of HEADER_SHAPED_SECRETS) {
|
|
88
58
|
redacted = redacted.replace(
|
package/src/server/self-test.ts
CHANGED
|
@@ -117,6 +117,8 @@ export type SelfTestAuthFlowInvoke = (args: {
|
|
|
117
117
|
externalRef?: string;
|
|
118
118
|
input?: Record<string, unknown>;
|
|
119
119
|
context?: Record<string, unknown>;
|
|
120
|
+
/** Opaque engine state from the previous turn (`engine` in the auth response), sent verbatim. */
|
|
121
|
+
engine?: { egressLease: string };
|
|
120
122
|
signal?: AbortSignal;
|
|
121
123
|
}) => Promise<{
|
|
122
124
|
status: number;
|
|
@@ -343,6 +345,7 @@ export function createSelfTestAuthFlowInvoke(app: {
|
|
|
343
345
|
externalRef,
|
|
344
346
|
input,
|
|
345
347
|
context,
|
|
348
|
+
engine,
|
|
346
349
|
signal,
|
|
347
350
|
}) => {
|
|
348
351
|
const response = await app.request(`/auth/${route}`, {
|
|
@@ -351,10 +354,14 @@ export function createSelfTestAuthFlowInvoke(app: {
|
|
|
351
354
|
body: JSON.stringify({
|
|
352
355
|
requestId,
|
|
353
356
|
flowId,
|
|
357
|
+
// Engine ceremony leases are scoped to a tenant; the self-test has no
|
|
358
|
+
// customer tenant, so it presents a fixed one that never collides.
|
|
359
|
+
tenantId: "self-test",
|
|
354
360
|
...(connectionId ? { connectionId } : {}),
|
|
355
361
|
...(externalRef ? { externalRef } : {}),
|
|
356
362
|
...(input ? { input } : {}),
|
|
357
363
|
...(context ? { context } : {}),
|
|
364
|
+
...(engine ? { engine } : {}),
|
|
358
365
|
}),
|
|
359
366
|
...(signal ? { signal } : {}),
|
|
360
367
|
});
|
|
@@ -550,6 +557,7 @@ type ParsedAuthFlowTurn =
|
|
|
550
557
|
ok: true;
|
|
551
558
|
turn: { kind: string; data?: unknown; expectedInput?: unknown };
|
|
552
559
|
contextPatch?: Record<string, unknown>;
|
|
560
|
+
engine?: { egressLease: string };
|
|
553
561
|
}
|
|
554
562
|
| { ok: false; code: string; message: string; httpStatus: number };
|
|
555
563
|
|
|
@@ -578,6 +586,7 @@ function parseAuthFlowResponse(result: { status: number; body: unknown }): Parse
|
|
|
578
586
|
};
|
|
579
587
|
}
|
|
580
588
|
const contextPatch = objectProperty(result.body, "contextPatch");
|
|
589
|
+
const egressLease = objectProperty(objectProperty(result.body, "engine"), "egressLease");
|
|
581
590
|
return {
|
|
582
591
|
ok: true,
|
|
583
592
|
turn: {
|
|
@@ -588,6 +597,7 @@ function parseAuthFlowResponse(result: { status: number; body: unknown }): Parse
|
|
|
588
597
|
...(contextPatch && typeof contextPatch === "object" && !Array.isArray(contextPatch)
|
|
589
598
|
? { contextPatch: contextPatch as Record<string, unknown> }
|
|
590
599
|
: {}),
|
|
600
|
+
...(typeof egressLease === "string" && egressLease ? { engine: { egressLease } } : {}),
|
|
591
601
|
};
|
|
592
602
|
}
|
|
593
603
|
|
|
@@ -777,6 +787,9 @@ async function materializeFlowCredential(
|
|
|
777
787
|
...(options.externalRef ? { externalRef: options.externalRef } : {}),
|
|
778
788
|
input: submitInputs,
|
|
779
789
|
...(Object.keys(flowContext).length > 0 ? { context: flowContext } : {}),
|
|
790
|
+
// The ceremony egress lease is engine state, not flow context: carry it the
|
|
791
|
+
// way the gateway does so the continue turn stays on the start turn's egress.
|
|
792
|
+
...(started.engine ? { engine: started.engine } : {}),
|
|
780
793
|
...(options.signal ? { signal: options.signal } : {}),
|
|
781
794
|
}),
|
|
782
795
|
);
|