@apifuse/provider-sdk 2.2.0-beta.67 → 2.2.0-beta.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +28 -1
- package/CHANGELOG.md +4 -0
- package/README.md +8 -4
- package/bin/apifuse-record.ts +81 -21
- package/bin/apifuse-submit-check.ts +170 -28
- package/dist/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +138 -23
- package/dist/cli/templates/provider/README.md.tpl +17 -9
- package/dist/config/loader.d.ts +16 -0
- package/dist/config/loader.js +25 -8
- package/dist/error-resolution.d.ts +13 -0
- package/dist/error-resolution.js +72 -35
- package/dist/lint.d.ts +2 -0
- package/dist/lint.js +493 -2
- package/dist/runtime/executor.js +3 -0
- package/dist/runtime/http.d.ts +2 -0
- package/dist/runtime/http.js +2 -1
- package/dist/runtime/provider.d.ts +7 -0
- package/dist/runtime/provider.js +14 -0
- package/dist/server/index.d.ts +1 -0
- package/dist/server/serve-implementation.d.ts +2 -1
- package/dist/server/serve-implementation.js +43 -29
- package/dist/server/state-backend-report.d.ts +70 -0
- package/dist/server/state-backend-report.js +113 -0
- package/package.json +1 -1
- package/src/cli/templates/provider/.agents/skills/health-checks-and-fail-closed/SKILL.md.tpl +138 -23
- package/src/cli/templates/provider/README.md.tpl +17 -9
- package/src/config/loader.ts +49 -12
- package/src/error-resolution.ts +89 -35
- package/src/lint.ts +541 -1
- package/src/runtime/executor.ts +3 -0
- package/src/runtime/http.ts +2 -1
- package/src/runtime/provider.ts +15 -0
- package/src/server/index.ts +4 -0
- package/src/server/serve-implementation.ts +56 -30
- package/src/server/state-backend-report.ts +185 -0
package/AUTHORING.md
CHANGED
|
@@ -680,6 +680,31 @@ The registered mappings are:
|
|
|
680
680
|
| Unregistered input `ValidationError` code | 400 |
|
|
681
681
|
| Other unregistered `ProviderError` code | 500 |
|
|
682
682
|
|
|
683
|
+
A registered code that names an upstream outcome also carries a category. When
|
|
684
|
+
a `ProviderError` (or any subclass, `ValidationError` included) with one of
|
|
685
|
+
these codes sets no `category`, the server reports the code's category in
|
|
686
|
+
`X-ApiFuse-Error-Observability`, and the public `source` follows from it:
|
|
687
|
+
|
|
688
|
+
| Error code | Category when unset | `source` |
|
|
689
|
+
| --- | --- | --- |
|
|
690
|
+
| `NOT_FOUND`, `not_found`, `NO_DATA` | `upstream_http` | `upstream_failure` |
|
|
691
|
+
| `RATE_LIMITED`, `UPSTREAM_RATE_LIMIT`, `LIMITED_NUMBER_OF_SERVICE_REQUESTS_EXCEEDS_ERROR` | `upstream_rate_limited` | `upstream_rule` |
|
|
692
|
+
| `UPSTREAM_REJECTED` | `upstream_rejected` | `upstream_rule` |
|
|
693
|
+
| `UPSTREAM_ERROR` | `upstream_http` | `upstream_failure` |
|
|
694
|
+
| `BLOCKED` | `anti_bot_blocked` | `upstream_failure` |
|
|
695
|
+
| `UPSTREAM_AUTH_ERROR` | `upstream_auth` | `upstream_failure` |
|
|
696
|
+
| `UPSTREAM_SCHEMA_ERROR` | `upstream_schema_drift` | `upstream_failure` |
|
|
697
|
+
|
|
698
|
+
So `throw new ProviderError("…", { code: "UPSTREAM_SCHEMA_ERROR" })` is enough
|
|
699
|
+
for the health monitor to grade a drifted upstream `down`; without the
|
|
700
|
+
category it would be served as `source: "apifuse"` and graded unknown. An
|
|
701
|
+
explicit `category` always wins, an operation-declared rejection status
|
|
702
|
+
(409/410/422) still classifies as `upstream_rejected`, and retryability does
|
|
703
|
+
not follow the category: it stays the instance option, then the declaration,
|
|
704
|
+
then `false`. Every other code, including `AUTH_REQUIRED`, `INVALID_REQUEST`,
|
|
705
|
+
and your own codes, is classified as before (`provider_error` unless you set a
|
|
706
|
+
category).
|
|
707
|
+
|
|
683
708
|
An unregistered non-validation `ProviderError` code returns HTTP 500 and emits
|
|
684
709
|
the greppable `unregistered_provider_error_code` signal with the code in the
|
|
685
710
|
structured failure log. A matching operation declaration, including one that
|
|
@@ -701,7 +726,9 @@ SDK-owned and keep their 502/504 mapping.
|
|
|
701
726
|
refused a *platform-managed* service key. The caller holds no credential, so
|
|
702
727
|
401 ("re-authenticate") asks for something the caller cannot do, and 502
|
|
703
728
|
("the upstream is sick") promises a recovery that will never arrive. It is a
|
|
704
|
-
deployment/config defect, exactly like `MISSING_SECRET`.
|
|
729
|
+
deployment/config defect, exactly like `MISSING_SECRET`. The status is about
|
|
730
|
+
what the caller can do; the attribution is still the upstream's, since the
|
|
731
|
+
upstream is what refused the key, so the category is `upstream_auth`.
|
|
705
732
|
- `UPSTREAM_SCHEMA_ERROR` is non-retryable: a retry returns the same payload
|
|
706
733
|
the provider already could not normalize.
|
|
707
734
|
- `INVALID_REQUEST` is the spelling for caller-side bad input. `INVALID_INPUT`,
|
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -309,10 +309,14 @@ apifuse bounty-check [path]
|
|
|
309
309
|
apifuse perf providers/korea-air-quality --operation realtime --params '{"stationName":"종로구"}'
|
|
310
310
|
```
|
|
311
311
|
|
|
312
|
-
`apifuse record` is for real upstream-backed operations that
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
312
|
+
`apifuse record` is for real upstream-backed operations that call the upstream
|
|
313
|
+
through `ctx.http` or `ctx.stealth`. Absolute request URLs need no declaration; a
|
|
314
|
+
relative URL resolves the way `serve` resolves it — against the operation's
|
|
315
|
+
`upstream.baseUrl`, else the provider's first declared one (`ctx.stealth` then
|
|
316
|
+
falls back to `https://<allowedHosts[0]>`). A relative `ctx.http` URL with no base
|
|
317
|
+
is refused with the `upstream.baseUrl` to declare. The generated local-only `ping`
|
|
318
|
+
operation intentionally has no upstream and should be replaced before recording
|
|
319
|
+
fixtures.
|
|
316
320
|
|
|
317
321
|
## Bounty submission readiness
|
|
318
322
|
|
package/bin/apifuse-record.ts
CHANGED
|
@@ -36,6 +36,8 @@ import {
|
|
|
36
36
|
resolveLocalConnection,
|
|
37
37
|
} from "../src/cli/local-connection.js";
|
|
38
38
|
import { createCliResolverRuntime } from "../src/cli/resolver-runtime.js";
|
|
39
|
+
import { isAbsoluteUrl } from "../src/runtime/http.js";
|
|
40
|
+
import { getProviderBaseUrl, getProviderStealthBaseUrl } from "../src/runtime/provider.js";
|
|
39
41
|
import { assertProcessEngineModeSupported } from "../src/runtime/engine-mode.js";
|
|
40
42
|
import type { JsonValue } from "../src/contract-json.js";
|
|
41
43
|
import {
|
|
@@ -93,6 +95,11 @@ const HELP_TEXT = `Usage: apifuse record [path] --operation <operation> --params
|
|
|
93
95
|
|
|
94
96
|
Calls a real upstream-backed operation through ctx.http or ctx.stealth and writes __fixtures__/raw.json.
|
|
95
97
|
|
|
98
|
+
Absolute request URLs need no declaration. A relative URL resolves the way serve resolves it: against
|
|
99
|
+
the operation's upstream.baseUrl, else the provider's first declared one (ctx.stealth then falls back
|
|
100
|
+
to https://<allowedHosts[0]>). A relative ctx.http URL with no base anywhere is refused and names
|
|
101
|
+
the upstream.baseUrl to declare.
|
|
102
|
+
|
|
96
103
|
Streaming responses are recorded as evidence (status, selected headers, full-body SHA-256 and byte
|
|
97
104
|
count, plus a ${STREAM_PREVIEW_BYTES}-byte base64 preview). Test replay is evidence-only: ctx.http.stream exposes the
|
|
98
105
|
preview as its body and the original body_sha256/body_bytes as response metadata.
|
|
@@ -137,7 +144,7 @@ export async function main() {
|
|
|
137
144
|
|
|
138
145
|
capture = createCaptureContext(
|
|
139
146
|
provider,
|
|
140
|
-
|
|
147
|
+
resolveRecordBaseUrls(provider, operationName),
|
|
141
148
|
args.sanitize,
|
|
142
149
|
connection,
|
|
143
150
|
);
|
|
@@ -148,6 +155,7 @@ export async function main() {
|
|
|
148
155
|
try {
|
|
149
156
|
result = await executeOperation(provider, operationName, capture.ctx, parsedParams);
|
|
150
157
|
} catch (operationError) {
|
|
158
|
+
capture.throwIfRefused();
|
|
151
159
|
let partial: unknown;
|
|
152
160
|
try {
|
|
153
161
|
partial = await capture.getCapturedRaw();
|
|
@@ -172,6 +180,9 @@ export async function main() {
|
|
|
172
180
|
);
|
|
173
181
|
if (telemetryLine) console.log(telemetryLine);
|
|
174
182
|
}
|
|
183
|
+
// Checked on success too: an operation that caught the refusal and carried
|
|
184
|
+
// on would otherwise save a fixture missing the request it never made.
|
|
185
|
+
capture.throwIfRefused();
|
|
175
186
|
const captured = await capture.getCapturedRaw();
|
|
176
187
|
|
|
177
188
|
if (captured === undefined) {
|
|
@@ -534,23 +545,58 @@ async function parseParams(
|
|
|
534
545
|
return operation.input ? parseSchema(operation.input, parsed, "record.params") : parsed;
|
|
535
546
|
}
|
|
536
547
|
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
548
|
+
/**
|
|
549
|
+
* The bases the recorder's `ctx.http` and `ctx.stealth` resolve relative
|
|
550
|
+
* request URLs against. An absent base means only absolute URLs resolve.
|
|
551
|
+
*/
|
|
552
|
+
export type RecordBaseUrls = {
|
|
553
|
+
readonly http?: string;
|
|
554
|
+
readonly stealth?: string;
|
|
555
|
+
};
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* `upstream.baseUrl` only resolves relative request URLs (and seeds the stealth
|
|
559
|
+
* cookie jar's default origin); the recorder uses it for nothing else, so an
|
|
560
|
+
* operation that requests absolute URLs records without one. The operation's
|
|
561
|
+
* own base wins; otherwise the recorder binds the bases `serve` binds, so a
|
|
562
|
+
* relative URL resolves in a recording exactly as it does in production.
|
|
563
|
+
*/
|
|
564
|
+
export function resolveRecordBaseUrls(
|
|
565
|
+
provider: ProviderRuntime,
|
|
566
|
+
operationName: string,
|
|
567
|
+
): RecordBaseUrls {
|
|
568
|
+
const declared = provider.operations[operationName]?.upstream?.baseUrl;
|
|
569
|
+
const own = typeof declared === "string" && declared.length > 0 ? declared : undefined;
|
|
570
|
+
const http = own ?? getProviderBaseUrl(provider);
|
|
571
|
+
const stealth = own ?? getProviderStealthBaseUrl(provider);
|
|
572
|
+
return { ...(http ? { http } : {}), ...(stealth ? { stealth } : {}) };
|
|
573
|
+
}
|
|
544
574
|
|
|
545
|
-
|
|
575
|
+
/** A request the recorder cannot resolve without a declaration; fails the run even if the operation catches it. */
|
|
576
|
+
class RecordRefusalError extends Error {
|
|
577
|
+
constructor(message: string) {
|
|
578
|
+
super(message);
|
|
579
|
+
this.name = "RecordRefusalError";
|
|
580
|
+
}
|
|
546
581
|
}
|
|
547
582
|
|
|
548
583
|
export function createCaptureContext(
|
|
549
584
|
provider: ProviderRuntime,
|
|
550
|
-
|
|
585
|
+
baseUrls: RecordBaseUrls,
|
|
551
586
|
sanitize: boolean,
|
|
552
587
|
connection?: LocalConnection,
|
|
553
588
|
) {
|
|
589
|
+
let refusal: RecordRefusalError | undefined;
|
|
590
|
+
const refuse = (message: string): never => {
|
|
591
|
+
refusal ??= new RecordRefusalError(message);
|
|
592
|
+
throw refusal;
|
|
593
|
+
};
|
|
594
|
+
const assertResolvableHttpUrl = (url: string) => {
|
|
595
|
+
if (baseUrls.http || isAbsoluteUrl(url)) return;
|
|
596
|
+
refuse(
|
|
597
|
+
`ctx.http cannot resolve the relative URL "${requestPathForFixture(url)}": no operation of provider "${provider.id}" declares upstream.baseUrl. Declare upstream.baseUrl on the recorded operation, or request an absolute URL.`,
|
|
598
|
+
);
|
|
599
|
+
};
|
|
554
600
|
let nextCaptureOrder = 0;
|
|
555
601
|
let nextStreamOrdinal = 0;
|
|
556
602
|
let capturedRaw: JsonValue | undefined;
|
|
@@ -610,7 +656,8 @@ export function createCaptureContext(
|
|
|
610
656
|
rawCaptures.push({ order, value: json });
|
|
611
657
|
};
|
|
612
658
|
|
|
613
|
-
const http = captureHttpClient(createHttpClient(
|
|
659
|
+
const http = captureHttpClient(createHttpClient(baseUrls.http), {
|
|
660
|
+
assertResolvableUrl: assertResolvableHttpUrl,
|
|
614
661
|
reserveOrder: reserveCaptureOrder,
|
|
615
662
|
reserveStreamOrdinal: () => {
|
|
616
663
|
nextStreamOrdinal += 1;
|
|
@@ -619,7 +666,7 @@ export function createCaptureContext(
|
|
|
619
666
|
onSensitiveParams: captureSensitiveParams,
|
|
620
667
|
onResponse: (order, response) => retainRawCapture(order, response.data),
|
|
621
668
|
onStreamResponse: (order, ordinal, requestUrl, method, response) => {
|
|
622
|
-
const resolvedRequestUrl = new URL(requestUrl,
|
|
669
|
+
const resolvedRequestUrl = new URL(requestUrl, baseUrls.http).toString();
|
|
623
670
|
const request = {
|
|
624
671
|
ordinal,
|
|
625
672
|
method,
|
|
@@ -642,18 +689,25 @@ export function createCaptureContext(
|
|
|
642
689
|
capturedSse = {
|
|
643
690
|
order,
|
|
644
691
|
method,
|
|
645
|
-
path: requestPathForFixture(new URL(requestUrl,
|
|
692
|
+
path: requestPathForFixture(new URL(requestUrl, baseUrls.http).toString()),
|
|
646
693
|
};
|
|
647
694
|
},
|
|
648
695
|
});
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
696
|
+
// Without a base `serve` binds no stealth transport either, so every call is refused.
|
|
697
|
+
const refuseStealth = (): never =>
|
|
698
|
+
refuse(
|
|
699
|
+
`ctx.stealth has no base URL: provider "${provider.id}" declares neither upstream.baseUrl nor allowedHosts. Declare allowedHosts on the provider (serve binds ctx.stealth to https://<allowedHosts[0]>) or upstream.baseUrl on the recorded operation.`,
|
|
700
|
+
);
|
|
701
|
+
const stealth: StealthClient = baseUrls.stealth
|
|
702
|
+
? proxyStealthClient(
|
|
703
|
+
createStealthClient(baseUrls.stealth, {
|
|
704
|
+
...(provider.stealth ? { stealth: provider.stealth } : {}),
|
|
705
|
+
}),
|
|
706
|
+
captureSensitiveParams,
|
|
707
|
+
(order, response) => retainRawCapture(order, normalizeCapturedStealthResponse(response)),
|
|
708
|
+
reserveCaptureOrder,
|
|
709
|
+
)
|
|
710
|
+
: { fetch: async () => refuseStealth(), createSession: refuseStealth };
|
|
657
711
|
|
|
658
712
|
const providerEnvironment = createProviderEnvironment(
|
|
659
713
|
process.env,
|
|
@@ -723,6 +777,9 @@ export function createCaptureContext(
|
|
|
723
777
|
return {
|
|
724
778
|
ctx,
|
|
725
779
|
resolverTelemetry,
|
|
780
|
+
throwIfRefused: () => {
|
|
781
|
+
if (refusal) throw refusal;
|
|
782
|
+
},
|
|
726
783
|
getCapturedRaw: async () => {
|
|
727
784
|
if (streamCaptures.length === 0) {
|
|
728
785
|
if (capturedSse) throw unsupportedSseCaptureError(capturedSse);
|
|
@@ -843,6 +900,8 @@ function snapshotRequestOptions<T extends SensitiveRequestOptions>(options: T):
|
|
|
843
900
|
}
|
|
844
901
|
|
|
845
902
|
type HttpCaptureCallbacks = {
|
|
903
|
+
/** Throws before the request when the recorder cannot resolve `url`. */
|
|
904
|
+
assertResolvableUrl(url: string): void;
|
|
846
905
|
reserveOrder(): number;
|
|
847
906
|
reserveStreamOrdinal(): number;
|
|
848
907
|
onSensitiveParams(url: string, options?: RequestOptions): void;
|
|
@@ -867,6 +926,7 @@ function captureHttpClient(client: HttpClient, callbacks: HttpCaptureCallbacks):
|
|
|
867
926
|
return response;
|
|
868
927
|
};
|
|
869
928
|
const captureRequestOptions = (method: PropertyKey, args: unknown[]) => {
|
|
929
|
+
callbacks.assertResolvableUrl(String(args[0]));
|
|
870
930
|
const invocation = parseHttpRequestInvocation(method, args);
|
|
871
931
|
const options = invocation ? requestOptionsFromHttpInvocation(invocation) : undefined;
|
|
872
932
|
if (!invocation || !options) return;
|
|
@@ -4076,18 +4076,37 @@ type MaskCommentsAndStringsOptions = {
|
|
|
4076
4076
|
blankPropertyKeys?: boolean;
|
|
4077
4077
|
};
|
|
4078
4078
|
|
|
4079
|
+
// Which literal bodies are blanked alongside comment bodies. "all" blanks
|
|
4080
|
+
// every string/template/regex body; "values" does the same but keeps quoted
|
|
4081
|
+
// property-key text (only its structural characters are blanked); "none"
|
|
4082
|
+
// leaves every literal and all code byte-identical, so only comments go.
|
|
4083
|
+
type LiteralMaskMode = "all" | "values" | "none";
|
|
4084
|
+
|
|
4079
4085
|
export function maskCommentsAndStrings(
|
|
4080
4086
|
source: string,
|
|
4081
4087
|
fileName = "provider.ts",
|
|
4082
4088
|
options: MaskCommentsAndStringsOptions = {},
|
|
4083
4089
|
): string {
|
|
4084
|
-
|
|
4085
|
-
|
|
4090
|
+
return maskSource(source, fileName, options.blankPropertyKeys === true ? "all" : "values");
|
|
4091
|
+
}
|
|
4092
|
+
|
|
4093
|
+
// Blanks comment bodies only, from the same TypeScript-lexer comment ranges
|
|
4094
|
+
// maskCommentsAndStrings uses: a `//` inside a string, template, or regex is
|
|
4095
|
+
// not a comment, and a block comment spanning lines is blanked whole. Line
|
|
4096
|
+
// terminators survive, so line N of the result is line N of the source.
|
|
4097
|
+
// Throws like maskCommentsAndStrings when the source cannot be parsed.
|
|
4098
|
+
export function maskComments(source: string, fileName = "provider.ts"): string {
|
|
4099
|
+
return maskSource(source, fileName, "none");
|
|
4100
|
+
}
|
|
4101
|
+
|
|
4102
|
+
function maskSource(source: string, fileName: string, literals: LiteralMaskMode): string {
|
|
4103
|
+
const { scriptKind } = maskScriptKind(fileName);
|
|
4104
|
+
const cacheKey = `${literals}\0${scriptKind}\0${source}`;
|
|
4086
4105
|
const cached = maskCache.get(cacheKey);
|
|
4087
4106
|
if (cached !== undefined) {
|
|
4088
4107
|
return cached;
|
|
4089
4108
|
}
|
|
4090
|
-
const masked = computeMaskedSource(source, fileName,
|
|
4109
|
+
const masked = computeMaskedSource(source, fileName, literals);
|
|
4091
4110
|
if (maskCache.size >= MASK_CACHE_LIMIT) {
|
|
4092
4111
|
const oldest = maskCache.keys().next();
|
|
4093
4112
|
if (!oldest.done) {
|
|
@@ -4098,13 +4117,34 @@ export function maskCommentsAndStrings(
|
|
|
4098
4117
|
return masked;
|
|
4099
4118
|
}
|
|
4100
4119
|
|
|
4101
|
-
|
|
4120
|
+
// The script kind a source is lexed as, from its file name, and the synthetic
|
|
4121
|
+
// implementation file name transpileModule parses it under. Declaration files
|
|
4122
|
+
// trigger an internal TypeScript Debug Failure when passed to transpileModule,
|
|
4123
|
+
// so `.d.ts` (like every other non-JS/JSX/TSX name) parses as `provider.ts`.
|
|
4124
|
+
function maskScriptKind(fileName: string): {
|
|
4125
|
+
scriptKind: TS.ScriptKind;
|
|
4126
|
+
syntheticFileName: string;
|
|
4127
|
+
} {
|
|
4128
|
+
if (fileName.endsWith(".tsx")) {
|
|
4129
|
+
return { scriptKind: ts.ScriptKind.TSX, syntheticFileName: "provider.tsx" };
|
|
4130
|
+
}
|
|
4131
|
+
if (fileName.endsWith(".jsx")) {
|
|
4132
|
+
return { scriptKind: ts.ScriptKind.JSX, syntheticFileName: "provider.jsx" };
|
|
4133
|
+
}
|
|
4134
|
+
if (/\.[cm]?js$/.test(fileName)) {
|
|
4135
|
+
return { scriptKind: ts.ScriptKind.JS, syntheticFileName: "provider.js" };
|
|
4136
|
+
}
|
|
4137
|
+
return { scriptKind: ts.ScriptKind.TS, syntheticFileName: "provider.ts" };
|
|
4138
|
+
}
|
|
4139
|
+
|
|
4140
|
+
function computeMaskedSource(source: string, fileName: string, literals: LiteralMaskMode): string {
|
|
4141
|
+
const blankPropertyKeys = literals === "all";
|
|
4142
|
+
const { scriptKind, syntheticFileName } = maskScriptKind(fileName);
|
|
4102
4143
|
const transpiled = ts.transpileModule(source, {
|
|
4103
|
-
//
|
|
4104
|
-
//
|
|
4105
|
-
//
|
|
4106
|
-
|
|
4107
|
-
fileName: "provider.ts",
|
|
4144
|
+
// Parsing is all we need here: parse under the synthetic implementation
|
|
4145
|
+
// filename while retaining the real filename for source mapping and
|
|
4146
|
+
// sanitized diagnostics below.
|
|
4147
|
+
fileName: syntheticFileName,
|
|
4108
4148
|
reportDiagnostics: true,
|
|
4109
4149
|
compilerOptions: { target: ts.ScriptTarget.Latest },
|
|
4110
4150
|
});
|
|
@@ -4113,7 +4153,7 @@ function computeMaskedSource(source: string, fileName: string, blankPropertyKeys
|
|
|
4113
4153
|
source,
|
|
4114
4154
|
ts.ScriptTarget.Latest,
|
|
4115
4155
|
true,
|
|
4116
|
-
|
|
4156
|
+
scriptKind,
|
|
4117
4157
|
);
|
|
4118
4158
|
const parseDiagnostic = transpiled.diagnostics?.[0];
|
|
4119
4159
|
if (parseDiagnostic) {
|
|
@@ -4144,10 +4184,7 @@ function computeMaskedSource(source: string, fileName: string, blankPropertyKeys
|
|
|
4144
4184
|
}
|
|
4145
4185
|
};
|
|
4146
4186
|
|
|
4147
|
-
const
|
|
4148
|
-
addCommentRanges(ts.getLeadingCommentRanges(source, node.getFullStart()));
|
|
4149
|
-
addCommentRanges(ts.getTrailingCommentRanges(source, node.getEnd()));
|
|
4150
|
-
|
|
4187
|
+
const maskLiteralBody = (node: TS.Node): void => {
|
|
4151
4188
|
const start = node.getStart(sourceFile);
|
|
4152
4189
|
if (ts.isStringLiteral(node)) {
|
|
4153
4190
|
// Preserve quoted property keys ("response": ...) by default so range/key
|
|
@@ -4184,7 +4221,12 @@ function computeMaskedSource(source: string, fileName: string, blankPropertyKeys
|
|
|
4184
4221
|
} else if (node.kind === ts.SyntaxKind.TemplateTail) {
|
|
4185
4222
|
maskRange(start + 1, node.end - 1);
|
|
4186
4223
|
}
|
|
4224
|
+
};
|
|
4187
4225
|
|
|
4226
|
+
const visit = (node: TS.Node): void => {
|
|
4227
|
+
addCommentRanges(ts.getLeadingCommentRanges(source, node.getFullStart()));
|
|
4228
|
+
addCommentRanges(ts.getTrailingCommentRanges(source, node.getEnd()));
|
|
4229
|
+
if (literals !== "none") maskLiteralBody(node);
|
|
4188
4230
|
for (const child of node.getChildren(sourceFile)) visit(child);
|
|
4189
4231
|
};
|
|
4190
4232
|
|
|
@@ -5599,18 +5641,18 @@ function findEntropySecretFindings(providerRoot: string, providerId: string): Se
|
|
|
5599
5641
|
if (isEntropySecretExcludedPath(relativePath)) continue;
|
|
5600
5642
|
const content = readFileSync(filePath, "utf8");
|
|
5601
5643
|
const lines = content.split(/\r?\n/);
|
|
5602
|
-
for (
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
|
|
5607
|
-
|
|
5608
|
-
|
|
5609
|
-
|
|
5610
|
-
|
|
5611
|
-
|
|
5612
|
-
|
|
5613
|
-
|
|
5644
|
+
for (const candidate of extractStringLiteralCandidates(content, relativePath)) {
|
|
5645
|
+
// Context is the whole source line, comment tail included: a
|
|
5646
|
+
// comment that names the value is evidence about it, even though
|
|
5647
|
+
// comment text itself is never a candidate.
|
|
5648
|
+
const finding = classifyEntropyCandidate({
|
|
5649
|
+
value: candidate.value,
|
|
5650
|
+
line: lines[candidate.lineIndex] ?? "",
|
|
5651
|
+
file: relativePath,
|
|
5652
|
+
lineNumber: candidate.lineIndex + 1,
|
|
5653
|
+
providerId,
|
|
5654
|
+
});
|
|
5655
|
+
if (finding) findings.push(finding);
|
|
5614
5656
|
}
|
|
5615
5657
|
}
|
|
5616
5658
|
return findings;
|
|
@@ -5626,7 +5668,62 @@ function isEntropySecretExcludedPath(relativePath: string): boolean {
|
|
|
5626
5668
|
);
|
|
5627
5669
|
}
|
|
5628
5670
|
|
|
5629
|
-
export
|
|
5671
|
+
export type StringLiteralCandidate = {
|
|
5672
|
+
value: string;
|
|
5673
|
+
// 0-based index of the source line the literal sits on.
|
|
5674
|
+
lineIndex: number;
|
|
5675
|
+
};
|
|
5676
|
+
|
|
5677
|
+
// Every string-literal candidate (a quoted body of at least
|
|
5678
|
+
// ENTROPY_CANDIDATE_MIN_LENGTH characters) in one provider source file.
|
|
5679
|
+
// Candidates come from code only. Comments are prose, not values, so each
|
|
5680
|
+
// file's own comment syntax is removed before a quote is read:
|
|
5681
|
+
// - JS/TS sources: comment trivia is blanked with the TypeScript lexer
|
|
5682
|
+
// (maskComments, the comment ranges the other source lints skip), so `//`
|
|
5683
|
+
// inside a string or regex is not a comment and a `/* … */` block spanning
|
|
5684
|
+
// lines is skipped whole. A backtick in code opens a template literal — a
|
|
5685
|
+
// string value — so it stays a quote form.
|
|
5686
|
+
// - Shell sources (.sh/.bash, Dockerfile, entrypoint): `#` starts a comment
|
|
5687
|
+
// when it begins a word outside quotes, and a backtick is command
|
|
5688
|
+
// substitution (code), not a string delimiter.
|
|
5689
|
+
// Identifiers are therefore never candidates by construction: a bare name is
|
|
5690
|
+
// not a string literal, and a backticked name in a comment is comment text.
|
|
5691
|
+
// A JS/TS source the lexer cannot parse keeps the unmasked scan (every quote
|
|
5692
|
+
// pair is a candidate): when comments cannot be located the scan may
|
|
5693
|
+
// over-report, never under-report.
|
|
5694
|
+
export function extractStringLiteralCandidates(
|
|
5695
|
+
content: string,
|
|
5696
|
+
relativePath: string,
|
|
5697
|
+
): StringLiteralCandidate[] {
|
|
5698
|
+
const shell = !isJavaScriptFamilySource(relativePath);
|
|
5699
|
+
const codeLines = (shell ? content : maskCommentsForSecretScan(content, relativePath)).split(
|
|
5700
|
+
/\r?\n/,
|
|
5701
|
+
);
|
|
5702
|
+
const candidates: StringLiteralCandidate[] = [];
|
|
5703
|
+
for (let lineIndex = 0; lineIndex < codeLines.length; lineIndex += 1) {
|
|
5704
|
+
const line = codeLines[lineIndex] ?? "";
|
|
5705
|
+
const values = shell ? extractShellLineStrings(line) : extractQuotedLineStrings(line);
|
|
5706
|
+
for (const value of values) candidates.push({ value, lineIndex });
|
|
5707
|
+
}
|
|
5708
|
+
return candidates;
|
|
5709
|
+
}
|
|
5710
|
+
|
|
5711
|
+
function isJavaScriptFamilySource(relativePath: string): boolean {
|
|
5712
|
+
return /\.(?:ts|tsx|js|jsx|mjs|cjs)$/.test(relativePath);
|
|
5713
|
+
}
|
|
5714
|
+
|
|
5715
|
+
function maskCommentsForSecretScan(content: string, relativePath: string): string {
|
|
5716
|
+
try {
|
|
5717
|
+
return maskComments(content, relativePath);
|
|
5718
|
+
} catch {
|
|
5719
|
+
// Unparseable: scan the source unmasked rather than skip it.
|
|
5720
|
+
return content;
|
|
5721
|
+
}
|
|
5722
|
+
}
|
|
5723
|
+
|
|
5724
|
+
// Quoted bodies on one comment-free JS/TS line (`"`, `'`, and template
|
|
5725
|
+
// backticks), with backslash escapes.
|
|
5726
|
+
function extractQuotedLineStrings(line: string): string[] {
|
|
5630
5727
|
const candidates: string[] = [];
|
|
5631
5728
|
for (let index = 0; index < line.length; index += 1) {
|
|
5632
5729
|
const quote = line[index];
|
|
@@ -5653,6 +5750,51 @@ export function extractStringLiteralCandidates(line: string): string[] {
|
|
|
5653
5750
|
return candidates;
|
|
5654
5751
|
}
|
|
5655
5752
|
|
|
5753
|
+
// Characters after which a `#` begins a new shell word (blanks and the
|
|
5754
|
+
// metacharacters `;&|()<>`).
|
|
5755
|
+
const SHELL_WORD_BOUNDARY = /[\s;&|()<>]/;
|
|
5756
|
+
|
|
5757
|
+
// Quoted bodies on one shell line. Single quotes take no escapes; double
|
|
5758
|
+
// quotes and unquoted text honor backslash escapes. A `#` that begins a word
|
|
5759
|
+
// outside quotes ends the line's code (the rest is a comment). A word begins
|
|
5760
|
+
// at the line start or after an UNESCAPED boundary character, so `${#var}`,
|
|
5761
|
+
// `$#`, `"a"#b`, and `word\ #` (escaped blank) keep `#` inside the word.
|
|
5762
|
+
// Backticks are command substitution, so their text is lexed as code. A
|
|
5763
|
+
// quote left open on the line is read as a literal character, as the
|
|
5764
|
+
// line-based scan always has.
|
|
5765
|
+
function extractShellLineStrings(line: string): string[] {
|
|
5766
|
+
const candidates: string[] = [];
|
|
5767
|
+
let index = 0;
|
|
5768
|
+
let atWordStart = true;
|
|
5769
|
+
while (index < line.length) {
|
|
5770
|
+
const char = line[index];
|
|
5771
|
+
if (char === "\\") {
|
|
5772
|
+
index += 2;
|
|
5773
|
+
atWordStart = false;
|
|
5774
|
+
continue;
|
|
5775
|
+
}
|
|
5776
|
+
if (char === "#" && atWordStart) break;
|
|
5777
|
+
if (char === '"' || char === "'") {
|
|
5778
|
+
const contentStart = index + 1;
|
|
5779
|
+
let cursor = contentStart;
|
|
5780
|
+
while (cursor < line.length && line[cursor] !== char) {
|
|
5781
|
+
cursor += char === '"' && line[cursor] === "\\" ? 2 : 1;
|
|
5782
|
+
}
|
|
5783
|
+
if (cursor < line.length) {
|
|
5784
|
+
if (cursor - contentStart >= ENTROPY_CANDIDATE_MIN_LENGTH) {
|
|
5785
|
+
candidates.push(line.slice(contentStart, cursor));
|
|
5786
|
+
}
|
|
5787
|
+
index = cursor + 1;
|
|
5788
|
+
atWordStart = false;
|
|
5789
|
+
continue;
|
|
5790
|
+
}
|
|
5791
|
+
}
|
|
5792
|
+
atWordStart = SHELL_WORD_BOUNDARY.test(char ?? "");
|
|
5793
|
+
index += 1;
|
|
5794
|
+
}
|
|
5795
|
+
return candidates;
|
|
5796
|
+
}
|
|
5797
|
+
|
|
5656
5798
|
function classifyEntropyCandidate(input: {
|
|
5657
5799
|
value: string;
|
|
5658
5800
|
line: string;
|
|
@@ -5757,7 +5899,7 @@ function literalContainerKey(literal: LineStringLiteral): string {
|
|
|
5757
5899
|
// other literal is a VALUE: ternary arms (preceded by "?" or ":"), array
|
|
5758
5900
|
// elements, call arguments, and assignment right-hand sides, even when a
|
|
5759
5901
|
// ternary's ":" happens to follow them. Uses the same quote/escape walking as
|
|
5760
|
-
//
|
|
5902
|
+
// extractQuotedLineStrings.
|
|
5761
5903
|
function tokenizeLineStringLiterals(line: string): LineStringLiteral[] {
|
|
5762
5904
|
const literals: LineStringLiteral[] = [];
|
|
5763
5905
|
const bracketStack: Array<{ bracket: "[" | "(" | "{"; index: number }> = [];
|