@specific.dev/spectest 0.95.0 → 0.97.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/components/aws.js +13 -3
- package/dist/components/emulate/entry.mjs +37 -2
- package/dist/components/emulate/service.js +1 -0
- package/dist/components/gcp.d.ts +2 -0
- package/dist/components/gcp.js +2 -0
- package/dist/components/google.js +3 -1
- package/dist/components/index.d.ts +1 -1
- package/dist/components/pubsub-push.py +83 -0
- package/dist/components/pubsub.d.ts +24 -1
- package/dist/components/pubsub.js +42 -6
- package/package.json +6 -1
- package/src/components/aws.test.ts +49 -0
- package/src/components/aws.ts +14 -3
- package/src/components/emulate/entry.mjs +37 -2
- package/src/components/emulate/service.ts +1 -0
- package/src/components/gcp.ts +15 -0
- package/src/components/google.ts +3 -1
- package/src/components/index.ts +1 -1
- package/src/components/pubsub-push.py +83 -0
- package/src/components/pubsub.test.ts +17 -2
- package/src/components/pubsub.ts +66 -7
package/README.md
CHANGED
|
@@ -17,11 +17,14 @@ and run `spectest docs /installation` for project setup and authentication.
|
|
|
17
17
|
|
|
18
18
|
## Google Cloud Pub/Sub
|
|
19
19
|
|
|
20
|
+
Google Cloud components are available from `@specific.dev/spectest/components/gcp`.
|
|
21
|
+
The existing Pub/Sub exports from `/components` remain supported.
|
|
22
|
+
|
|
20
23
|
`pubsub()` runs Google's local emulator and creates topics/subscriptions before
|
|
21
24
|
dependent services start:
|
|
22
25
|
|
|
23
26
|
```ts
|
|
24
|
-
import { pubsub } from "@specific.dev/spectest/components";
|
|
27
|
+
import { pubsub } from "@specific.dev/spectest/components/gcp";
|
|
25
28
|
|
|
26
29
|
const services = {
|
|
27
30
|
messages: pubsub({
|
|
@@ -40,6 +43,12 @@ the emulator's live state. See `examples/google-pubsub` and
|
|
|
40
43
|
`spectest docs /components/pubsub` for test credential setup and options.
|
|
41
44
|
Set `tls: false` to use explicit plaintext emulator mode instead.
|
|
42
45
|
|
|
46
|
+
Push subscriptions support HTTPS, `pushConfig.oidcToken`, `deadLetterPolicy`,
|
|
47
|
+
and `retryPolicy`. Add the `google()` component for locally signed service-account
|
|
48
|
+
ID tokens verified through Google’s normal certificate endpoints. Declare
|
|
49
|
+
authenticated subscriptions in `subscriptions` or through the component’s
|
|
50
|
+
`createSubscription` helper; raw emulator API calls do not add authentication.
|
|
51
|
+
|
|
43
52
|
## HTTP assertions
|
|
44
53
|
|
|
45
54
|
`ctx.fetch()` returns a wrapped response. Status fields, header lookups, and
|
package/dist/components/aws.js
CHANGED
|
@@ -94,13 +94,23 @@ const AWS_REGIONS = [
|
|
|
94
94
|
const AWS_CN_REGIONS = ["cn-north-1", "cn-northwest-1"];
|
|
95
95
|
/** Every hostname pattern the emulator answers to: the two partitions' global
|
|
96
96
|
* endpoints (`sts.amazonaws.com`, `iam.amazonaws.com`) plus one per region
|
|
97
|
-
* for the regional ones (`states.us-east-1.amazonaws.com`).
|
|
97
|
+
* for the regional ones (`states.us-east-1.amazonaws.com`). ECR adds
|
|
98
|
+
* labels for its API and account registries, each needing its own SAN. */
|
|
98
99
|
const AWS_HOST_PATTERNS = [
|
|
99
100
|
"*.amazonaws.com",
|
|
100
|
-
...AWS_REGIONS.
|
|
101
|
+
...AWS_REGIONS.flatMap((r) => regionalHostPatterns(r, "amazonaws.com")),
|
|
101
102
|
"*.amazonaws.com.cn",
|
|
102
|
-
...AWS_CN_REGIONS.
|
|
103
|
+
...AWS_CN_REGIONS.flatMap((r) => regionalHostPatterns(r, "amazonaws.com.cn")),
|
|
103
104
|
];
|
|
105
|
+
function regionalHostPatterns(region, suffix) {
|
|
106
|
+
return [
|
|
107
|
+
`*.${region}.${suffix}`,
|
|
108
|
+
`*.ecr.${region}.${suffix}`,
|
|
109
|
+
`*.dkr.ecr.${region}.${suffix}`,
|
|
110
|
+
`*.ecr-fips.${region}.${suffix}`,
|
|
111
|
+
`*.dkr.ecr-fips.${region}.${suffix}`,
|
|
112
|
+
];
|
|
113
|
+
}
|
|
104
114
|
/** The VM's own docker socket, bind-mounted so the emulator can start the
|
|
105
115
|
* official AWS runtime containers. This is the VM's dockerd, inside the
|
|
106
116
|
* hermetic environment — the same daemon that runs the services. */
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
// the whole OAuth exchange is readable on the case page with no helper API.
|
|
31
31
|
|
|
32
32
|
import { createServer } from "@emulators/core";
|
|
33
|
-
import { SignJWT, decodeJwt, exportJWK } from "jose";
|
|
33
|
+
import { SignJWT, decodeJwt, exportJWK, exportPKCS8 } from "jose";
|
|
34
34
|
|
|
35
35
|
const CONFIG = JSON.parse(process.env.SPECTEST_AUTH_CONFIG ?? "{}");
|
|
36
36
|
const PORT = Number(CONFIG.port ?? 4000);
|
|
@@ -60,6 +60,37 @@ const JWKS = {
|
|
|
60
60
|
keys: [{ ...(await exportJWK(KEY_PAIR.publicKey)), kid: KID, use: "sig", alg: "RS256" }],
|
|
61
61
|
};
|
|
62
62
|
|
|
63
|
+
// Google's Node/Python clients fetch PEM certificates from /oauth2/v1/certs,
|
|
64
|
+
// while browser clients use the JWK endpoint. Both must identify the same key.
|
|
65
|
+
let GOOGLE_CERT;
|
|
66
|
+
if ((CONFIG.providers ?? []).some(provider => provider.name === "google")) {
|
|
67
|
+
const child = Bun.spawn(["openssl", "req", "-new", "-x509", "-key", "/dev/stdin", "-days", "3650", "-subj", "/CN=spectest-google-token-signing"], {
|
|
68
|
+
stdin: new Blob([await exportPKCS8(KEY_PAIR.privateKey)]), stdout: "pipe", stderr: "pipe",
|
|
69
|
+
});
|
|
70
|
+
const [cert, error, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]);
|
|
71
|
+
if (code !== 0) throw new Error(`google(): signing certificate setup failed: ${error}`);
|
|
72
|
+
GOOGLE_CERT = cert;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async function googleIdToken(req, path) {
|
|
76
|
+
const match = /^\/v1\/projects\/-\/serviceAccounts\/([^/]+):generateIdToken$/.exec(path);
|
|
77
|
+
if (!match || req.method !== "POST") return json({ error: "Unsupported IAM operation" }, 404);
|
|
78
|
+
let options, email;
|
|
79
|
+
try {
|
|
80
|
+
options = await req.json();
|
|
81
|
+
email = decodeURIComponent(match[1]);
|
|
82
|
+
} catch { return json({ error: "Invalid token request" }, 400); }
|
|
83
|
+
if (!options || !/^[^@\s]+@[^@\s]+$/.test(email) || typeof options.audience !== "string" || !options.audience) return json({ error: "serviceAccountEmail and audience are required" }, 400);
|
|
84
|
+
// Stable service-account subject; isolated test identities require no cloud credentials.
|
|
85
|
+
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(email)));
|
|
86
|
+
const subject = BigInt('0x' + Buffer.from(digest.slice(0, 8)).toString('hex')).toString();
|
|
87
|
+
const token = await new SignJWT(options.includeEmail ? { email, email_verified: true } : {})
|
|
88
|
+
.setProtectedHeader({ alg: "RS256", kid: KID, typ: "JWT" })
|
|
89
|
+
.setIssuer("https://accounts.google.com").setSubject(subject).setAudience(options.audience)
|
|
90
|
+
.setIssuedAt().setExpirationTime("1h").sign(KEY_PAIR.privateKey);
|
|
91
|
+
return json({ token }, 200);
|
|
92
|
+
}
|
|
93
|
+
|
|
63
94
|
// ──────────────────────────────────────────────────────────────────────────
|
|
64
95
|
// Providers
|
|
65
96
|
// ──────────────────────────────────────────────────────────────────────────
|
|
@@ -205,7 +236,11 @@ Bun.serve({
|
|
|
205
236
|
}
|
|
206
237
|
|
|
207
238
|
let res;
|
|
208
|
-
if (provider.
|
|
239
|
+
if (provider.name === "google" && host === "iamcredentials.googleapis.com") {
|
|
240
|
+
res = await googleIdToken(req, path);
|
|
241
|
+
} else if (provider.name === "google" && path === "/oauth2/v1/certs") {
|
|
242
|
+
res = json({ [KID]: GOOGLE_CERT }, 200);
|
|
243
|
+
} else if (provider.oidc && provider.jwksPath?.test(path)) {
|
|
209
244
|
// Our key, never the emulator's — we re-sign every id_token.
|
|
210
245
|
res = json(JWKS, 200);
|
|
211
246
|
} else {
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
/** Google Cloud components for local Spectest environments. */
|
|
2
|
+
export { pubsub, pubsubClient, type PubSubOptions, type PubSubOidcToken, type PubSubPushConfig, type PubSubDeadLetterPolicy, type PubSubRetryPolicy, type PubSubClient, type PubSubHelpers, type PubSubSubscriptionOptions, type PubSubMessage, type PubSubReceivedMessage, } from "./pubsub.js";
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// `google()` — Google, emulated inside the environment, answering at its
|
|
2
2
|
// real endpoints.
|
|
3
3
|
//
|
|
4
|
-
//
|
|
4
|
+
// Supports service-account ID tokens (IAM generateIdToken) and Sign in
|
|
5
|
+
// with Google: OpenID Connect discovery, the account
|
|
5
6
|
// picker, PKCE, the token exchange, refresh, and userinfo. The component is
|
|
6
7
|
// named for the provider rather than for the feature, because the backing
|
|
7
8
|
// emulator also carries Gmail, Calendar and Drive — claiming
|
|
@@ -16,6 +17,7 @@ function spec(users, opts) {
|
|
|
16
17
|
baseUrl: "https://accounts.google.com",
|
|
17
18
|
hosts: [
|
|
18
19
|
"accounts.google.com",
|
|
20
|
+
"iamcredentials.googleapis.com",
|
|
19
21
|
"oauth2.googleapis.com",
|
|
20
22
|
"www.googleapis.com",
|
|
21
23
|
"openidconnect.googleapis.com",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { pubsub, pubsubClient, type PubSubOptions, type PubSubClient, type PubSubHelpers, type PubSubSubscriptionOptions, type PubSubMessage, type PubSubReceivedMessage } from "./pubsub.js";
|
|
1
|
+
export { pubsub, pubsubClient, type PubSubOptions, type PubSubOidcToken, type PubSubPushConfig, type PubSubDeadLetterPolicy, type PubSubRetryPolicy, type PubSubClient, type PubSubHelpers, type PubSubSubscriptionOptions, type PubSubMessage, type PubSubReceivedMessage } from "./pubsub.js";
|
|
2
2
|
export { temporal, temporalClient, type TemporalOptions, type TemporalClient, type TemporalNamespace, type TemporalWorkerOptions, type TemporalPoller, type TemporalSearchAttributeType, type TemporalTaskType } from "./temporal.js";
|
|
3
3
|
export { postgres, type PostgresOptions, type PostgresHelpers, } from "./postgres.js";
|
|
4
4
|
export { s3, type S3Options, type S3Helpers, } from "./s3.js";
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Authenticated push adapter; the emulator still owns scheduling and retries.
|
|
2
|
+
|
|
3
|
+
Runs only on loopback in the emulator container. Tokens come from the local
|
|
4
|
+
Google IAM fake, so recipients verify them with the same Google JWKS as usual.
|
|
5
|
+
Never follows redirects or acknowledges a failed token request/delivery.
|
|
6
|
+
"""
|
|
7
|
+
import base64
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import ssl
|
|
11
|
+
import sys
|
|
12
|
+
import urllib.error
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import urllib.request
|
|
15
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
19
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
20
|
+
return None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def opener():
|
|
24
|
+
context = ssl.create_default_context(cafile=os.environ.get("SSL_CERT_FILE"))
|
|
25
|
+
return urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Handler(BaseHTTPRequestHandler):
|
|
29
|
+
def do_POST(self):
|
|
30
|
+
phase = "configuration"
|
|
31
|
+
try:
|
|
32
|
+
if not self.path.startswith('/push/'):
|
|
33
|
+
self.send_error(404)
|
|
34
|
+
return
|
|
35
|
+
encoded = self.path[len('/push/'):]
|
|
36
|
+
config = json.loads(base64.urlsafe_b64decode(encoded + '=' * (-len(encoded) % 4)))
|
|
37
|
+
endpoint = config['endpoint']
|
|
38
|
+
if urllib.parse.urlsplit(endpoint).scheme not in ('http', 'https'):
|
|
39
|
+
raise ValueError('Invalid push endpoint')
|
|
40
|
+
size = int(self.headers.get('Content-Length', '0'))
|
|
41
|
+
if not 0 <= size <= 16 * 1024 * 1024:
|
|
42
|
+
self.send_error(413)
|
|
43
|
+
return
|
|
44
|
+
body = self.rfile.read(size)
|
|
45
|
+
account = urllib.parse.quote(config['serviceAccountEmail'], safe='')
|
|
46
|
+
token_url = 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + account + ':generateIdToken'
|
|
47
|
+
token_body = json.dumps({'audience': config.get('audience') or endpoint, 'includeEmail': True}).encode()
|
|
48
|
+
http = opener()
|
|
49
|
+
phase = "token issuance"
|
|
50
|
+
with http.open(urllib.request.Request(token_url, data=token_body, headers={'Content-Type': 'application/json'}), timeout=10) as response:
|
|
51
|
+
token = json.load(response)['token']
|
|
52
|
+
if not isinstance(token, str) or not token or '\r' in token or '\n' in token:
|
|
53
|
+
raise ValueError('Invalid ID token from Google IAM fake')
|
|
54
|
+
request = urllib.request.Request(endpoint, data=body, headers={
|
|
55
|
+
'Content-Type': self.headers.get('Content-Type', 'application/json'),
|
|
56
|
+
'Authorization': 'Bearer ' + token,
|
|
57
|
+
})
|
|
58
|
+
phase = "delivery"
|
|
59
|
+
with http.open(request, timeout=30) as response:
|
|
60
|
+
status = response.status
|
|
61
|
+
self.send_response(status)
|
|
62
|
+
self.end_headers()
|
|
63
|
+
except urllib.error.HTTPError as error:
|
|
64
|
+
# An unsuccessful token request or destination response must retry.
|
|
65
|
+
print(f"pubsub push {phase} returned HTTP {error.code}", flush=True)
|
|
66
|
+
self.send_response(error.code if error.code >= 300 else 502)
|
|
67
|
+
self.end_headers()
|
|
68
|
+
error.close()
|
|
69
|
+
except Exception as error:
|
|
70
|
+
# Do not log tokens, request bodies, or the encoded config URL.
|
|
71
|
+
print('pubsub push adapter failed: ' + type(error).__name__, flush=True)
|
|
72
|
+
self.send_response(502)
|
|
73
|
+
self.end_headers()
|
|
74
|
+
|
|
75
|
+
def log_message(self, format, *args):
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
if __name__ == '__main__':
|
|
80
|
+
server = ThreadingHTTPServer(('127.0.0.1', int(sys.argv[1])), Handler)
|
|
81
|
+
server.daemon_threads = True
|
|
82
|
+
print('pubsub authenticated push adapter ready', flush=True)
|
|
83
|
+
server.serve_forever()
|
|
@@ -1,10 +1,33 @@
|
|
|
1
1
|
import type { ServiceSetupContext, ServicesMap } from "../index.js";
|
|
2
|
+
export interface PubSubOidcToken {
|
|
3
|
+
serviceAccountEmail: string;
|
|
4
|
+
/** Defaults to the original push endpoint URL. */
|
|
5
|
+
audience?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface PubSubPushConfig {
|
|
8
|
+
pushEndpoint: string;
|
|
9
|
+
oidcToken?: PubSubOidcToken;
|
|
10
|
+
attributes?: Record<string, string>;
|
|
11
|
+
}
|
|
12
|
+
export interface PubSubDeadLetterPolicy {
|
|
13
|
+
/** Fully qualified projects/{project}/topics/{topic} name. Create this topic separately. */
|
|
14
|
+
deadLetterTopic: string;
|
|
15
|
+
maxDeliveryAttempts?: number;
|
|
16
|
+
}
|
|
17
|
+
export interface PubSubRetryPolicy {
|
|
18
|
+
/** Google duration strings, for example "1s" and "60s". */
|
|
19
|
+
minimumBackoff?: string;
|
|
20
|
+
maximumBackoff?: string;
|
|
21
|
+
}
|
|
2
22
|
export interface PubSubSubscriptionOptions {
|
|
3
23
|
/** Topic ID in this component's project. */
|
|
4
24
|
topic: string;
|
|
5
25
|
ackDeadlineSeconds?: number;
|
|
6
|
-
/**
|
|
26
|
+
/** Shorthand for pushConfig.pushEndpoint. HTTP and HTTPS are supported. */
|
|
7
27
|
pushEndpoint?: string;
|
|
28
|
+
pushConfig?: PubSubPushConfig;
|
|
29
|
+
deadLetterPolicy?: PubSubDeadLetterPolicy;
|
|
30
|
+
retryPolicy?: PubSubRetryPolicy;
|
|
8
31
|
}
|
|
9
32
|
export interface PubSubOptions {
|
|
10
33
|
/** Local project ID; no Google Cloud account is needed. Default "spectest". */
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
1
2
|
import { serviceGroup, certificate } from "../index.js";
|
|
2
3
|
import { rawFetch } from "../harness/raw-fetch.js";
|
|
3
4
|
import { readRaw } from "../inspect.js";
|
|
5
|
+
const PUSH_ADAPTER = readFileSync(new URL("./pubsub-push.py", import.meta.url), "utf8");
|
|
4
6
|
function resourceId(id) {
|
|
5
7
|
if (!/^[A-Za-z][A-Za-z0-9._~+%-]{2,254}$/.test(id) || id.startsWith("goog")) {
|
|
6
8
|
throw new Error(`pubsub(): invalid topic or subscription ID ${JSON.stringify(id)}; use a 3–255 character ID starting with a letter, not "goog"`);
|
|
@@ -17,8 +19,15 @@ function subscriptionOptions(options) {
|
|
|
17
19
|
const seconds = options.ackDeadlineSeconds ?? 10;
|
|
18
20
|
if (!Number.isInteger(seconds) || seconds < 10 || seconds > 600)
|
|
19
21
|
throw new Error("pubsub(): ackDeadlineSeconds must be an integer between 10 and 600");
|
|
20
|
-
if (options.pushEndpoint &&
|
|
21
|
-
throw new Error("pubsub(): pushEndpoint
|
|
22
|
+
if (options.pushEndpoint && options.pushConfig && options.pushEndpoint !== options.pushConfig.pushEndpoint)
|
|
23
|
+
throw new Error("pubsub(): pushEndpoint and pushConfig.pushEndpoint disagree");
|
|
24
|
+
const push = options.pushConfig ?? (options.pushEndpoint ? { pushEndpoint: options.pushEndpoint } : undefined);
|
|
25
|
+
if (push) {
|
|
26
|
+
if (!push.pushEndpoint || !/^https?:$/.test(new URL(push.pushEndpoint).protocol))
|
|
27
|
+
throw new Error("pubsub(): pushEndpoint must use HTTP or HTTPS");
|
|
28
|
+
if (push.oidcToken && (!/^[^@\s]+@[^@\s]+$/.test(push.oidcToken.serviceAccountEmail) || (push.oidcToken.audience !== undefined && !push.oidcToken.audience)))
|
|
29
|
+
throw new Error("pubsub(): oidcToken requires a serviceAccountEmail and nonempty audience");
|
|
30
|
+
}
|
|
22
31
|
return seconds;
|
|
23
32
|
}
|
|
24
33
|
/** Stateless emulator REST client. No credentials or gRPC channel to preserve across forks.
|
|
@@ -53,7 +62,9 @@ export function pubsubClient(endpoint, projectId = "spectest", fetcher = rawFetc
|
|
|
53
62
|
const ackDeadlineSeconds = subscriptionOptions(options);
|
|
54
63
|
return request(path("subscriptions", id), "PUT", {
|
|
55
64
|
topic: resource("topics", options.topic), ackDeadlineSeconds,
|
|
56
|
-
...(options.pushEndpoint ? { pushConfig: { pushEndpoint: options.pushEndpoint } } : {}),
|
|
65
|
+
...(options.pushConfig ? { pushConfig: options.pushConfig } : options.pushEndpoint ? { pushConfig: { pushEndpoint: options.pushEndpoint } } : {}),
|
|
66
|
+
...(options.deadLetterPolicy ? { deadLetterPolicy: options.deadLetterPolicy } : {}),
|
|
67
|
+
...(options.retryPolicy ? { retryPolicy: options.retryPolicy } : {}),
|
|
57
68
|
});
|
|
58
69
|
},
|
|
59
70
|
async publish(topic, data, attributes) {
|
|
@@ -77,6 +88,22 @@ export function pubsubClient(endpoint, projectId = "spectest", fetcher = rawFetc
|
|
|
77
88
|
},
|
|
78
89
|
};
|
|
79
90
|
}
|
|
91
|
+
// The official emulator ignores OIDC. Keep its push/retry/dead-letter engine,
|
|
92
|
+
// but route authenticated deliveries through a local adapter in its container.
|
|
93
|
+
function authenticatedPush(options, pushPort) {
|
|
94
|
+
subscriptionOptions(options);
|
|
95
|
+
const push = options.pushConfig;
|
|
96
|
+
if (!push?.oidcToken)
|
|
97
|
+
return options;
|
|
98
|
+
const config = Buffer.from(JSON.stringify({ endpoint: push.pushEndpoint, ...push.oidcToken })).toString("base64url");
|
|
99
|
+
return {
|
|
100
|
+
...options, pushEndpoint: undefined,
|
|
101
|
+
pushConfig: { ...push, pushEndpoint: `http://127.0.0.1:${pushPort}/push/${config}` },
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
function withPushAuth(client, pushPort) {
|
|
105
|
+
return { ...client, createSubscription: (id, options) => client.createSubscription(id, authenticatedPush(options, pushPort)) };
|
|
106
|
+
}
|
|
80
107
|
/** Google Cloud's real local Pub/Sub emulator, ready for Google client libraries.
|
|
81
108
|
* Mount directly in services: { pubsub: pubsub({ topics: ["orders"] }) }.
|
|
82
109
|
* Applications use their default pubsub.googleapis.com:443 endpoint with
|
|
@@ -86,6 +113,7 @@ export function pubsubClient(endpoint, projectId = "spectest", fetcher = rawFetc
|
|
|
86
113
|
export function pubsub(opts = {}) {
|
|
87
114
|
const projectId = projectName(opts.projectId ?? "spectest");
|
|
88
115
|
const port = opts.port ?? 8085;
|
|
116
|
+
const pushPort = port === 8086 ? 8087 : 8086;
|
|
89
117
|
const hostname = opts.tls === false ? undefined : opts.tls?.hostname ?? "pubsub.googleapis.com";
|
|
90
118
|
if (hostname !== undefined) {
|
|
91
119
|
certificate([hostname]);
|
|
@@ -107,12 +135,20 @@ export function pubsub(opts = {}) {
|
|
|
107
135
|
services: () => {
|
|
108
136
|
const emulator = {
|
|
109
137
|
image: { type: "registry", reference: opts.image ?? "gcr.io/google.com/cloudsdktool/google-cloud-cli:573.0.0-emulators" },
|
|
110
|
-
args: ["
|
|
138
|
+
args: ["sh", "-ec", `
|
|
139
|
+
java_home=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")
|
|
140
|
+
cp "$java_home/lib/security/cacerts" /tmp/spectest-pubsub-cacerts
|
|
141
|
+
keytool -importcert -noprompt -alias spectest-environment -keystore /tmp/spectest-pubsub-cacerts -storepass changeit -file "$NODE_EXTRA_CA_CERTS"
|
|
142
|
+
export JAVA_TOOL_OPTIONS="-Djavax.net.ssl.trustStore=/tmp/spectest-pubsub-cacerts -Djavax.net.ssl.trustStorePassword=changeit \${JAVA_TOOL_OPTIONS:-}"
|
|
143
|
+
python3 /opt/spectest-pubsub-push.py ${pushPort} &
|
|
144
|
+
exec gcloud beta emulators pubsub start --project=${projectId} --host-port=0.0.0.0:${port} --quiet
|
|
145
|
+
`],
|
|
146
|
+
files: [{ path: "/opt/spectest-pubsub-push.py", content: PUSH_ADAPTER }],
|
|
111
147
|
ports: [port],
|
|
112
148
|
...(hostname ? { tls: [{ hostname, port, http2: true }] } : {}),
|
|
113
149
|
readyCheck: { type: "http", port, path: `/v1/projects/${projectId}/topics`, timeoutSecs: 60 },
|
|
114
150
|
setup: async ({ name }) => {
|
|
115
|
-
const client = pubsubClient(`http://${name}:${port}`, projectId);
|
|
151
|
+
const client = withPushAuth(pubsubClient(`http://${name}:${port}`, projectId), pushPort);
|
|
116
152
|
for (const topic of topics)
|
|
117
153
|
await client.createTopic(topic);
|
|
118
154
|
for (const [id, options] of subscriptions)
|
|
@@ -122,7 +158,7 @@ export function pubsub(opts = {}) {
|
|
|
122
158
|
return { emulator };
|
|
123
159
|
},
|
|
124
160
|
helpers: (ctx) => ({
|
|
125
|
-
...pubsubClient(hostname ? `https://${hostname}` : `http://${ctx.name}:${port}`, projectId, async (input, init) => readRaw(await ctx.fetch(String(input), init))),
|
|
161
|
+
...withPushAuth(pubsubClient(hostname ? `https://${hostname}` : `http://${ctx.name}:${port}`, projectId, async (input, init) => readRaw(await ctx.fetch(String(input), init))), pushPort),
|
|
126
162
|
emulatorHost: `${ctx.name}:${port}`,
|
|
127
163
|
}),
|
|
128
164
|
setup: async (ctx) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@specific.dev/spectest",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.97.0",
|
|
4
4
|
"description": "Spectest SDK for defining test environments in TypeScript.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -16,6 +16,11 @@
|
|
|
16
16
|
"bun": "./src/components/index.ts",
|
|
17
17
|
"default": "./dist/components/index.js"
|
|
18
18
|
},
|
|
19
|
+
"./components/gcp": {
|
|
20
|
+
"types": "./dist/components/gcp.d.ts",
|
|
21
|
+
"bun": "./src/components/gcp.ts",
|
|
22
|
+
"default": "./dist/components/gcp.js"
|
|
23
|
+
},
|
|
19
24
|
"./coverage": {
|
|
20
25
|
"types": "./dist/coverage.d.ts",
|
|
21
26
|
"bun": "./src/coverage.ts",
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { X509Certificate } from "node:crypto";
|
|
3
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { defineEnvironment } from "../index.js";
|
|
7
|
+
import { lowerIngress } from "../ingress.js";
|
|
8
|
+
import { generateHostCert } from "../harness/certificate.js";
|
|
9
|
+
import { aws } from "./aws.js";
|
|
10
|
+
|
|
11
|
+
test("AWS certificate verifies nested ECR endpoints in every partition", async () => {
|
|
12
|
+
const env = defineEnvironment({ name: "test", services: { cloud: aws() } });
|
|
13
|
+
const ingress = lowerIngress(env.project([]));
|
|
14
|
+
expect(ingress.certificates).toHaveLength(1);
|
|
15
|
+
const names = ingress.certificates[0].hostnames;
|
|
16
|
+
const dir = await mkdtemp(join(tmpdir(), "spectest-aws-tls-"));
|
|
17
|
+
const ca = join(dir, "ca.crt");
|
|
18
|
+
const key = join(dir, "ca.key");
|
|
19
|
+
try {
|
|
20
|
+
const root = Bun.spawnSync(["openssl", "req", "-x509", "-newkey", "rsa:2048",
|
|
21
|
+
"-nodes", "-days", "1", "-subj", "/CN=test-ca", "-keyout", key, "-out", ca]);
|
|
22
|
+
expect(root.exitCode, root.stderr.toString()).toBe(0);
|
|
23
|
+
const { cert } = await generateHostCert("aws", names, ca, key);
|
|
24
|
+
const leaf = new X509Certificate(cert);
|
|
25
|
+
for (const host of [
|
|
26
|
+
"api.ecr.eu-central-1.amazonaws.com",
|
|
27
|
+
"123456789012.dkr.ecr.eu-central-1.amazonaws.com",
|
|
28
|
+
"api.ecr.us-gov-west-1.amazonaws.com",
|
|
29
|
+
"api.ecr-fips.us-gov-west-1.amazonaws.com",
|
|
30
|
+
"123456789012.dkr.ecr-fips.us-gov-west-1.amazonaws.com",
|
|
31
|
+
"api.ecr.cn-north-1.amazonaws.com.cn",
|
|
32
|
+
"123456789012.dkr.ecr.cn-northwest-1.amazonaws.com.cn",
|
|
33
|
+
"states.us-east-1.amazonaws.com",
|
|
34
|
+
"sts.amazonaws.com",
|
|
35
|
+
]) {
|
|
36
|
+
const pattern = leaf.checkHost(host);
|
|
37
|
+
expect(pattern, host).toBeDefined();
|
|
38
|
+
expect(ingress.proxies).toContainEqual({ hostname: pattern, service: "cloud", port: 4566 });
|
|
39
|
+
}
|
|
40
|
+
expect(leaf.checkHost("api.ecr.eu-central-1.amazonaws.com.attacker.test")).toBeUndefined();
|
|
41
|
+
const file = join(dir, "leaf.crt");
|
|
42
|
+
await writeFile(file, cert);
|
|
43
|
+
const verified = Bun.spawnSync(["openssl", "verify", "-CAfile", ca,
|
|
44
|
+
"-verify_hostname", "api.ecr.eu-central-1.amazonaws.com", file]);
|
|
45
|
+
expect(verified.exitCode, verified.stderr.toString()).toBe(0);
|
|
46
|
+
} finally {
|
|
47
|
+
await rm(dir, { recursive: true, force: true });
|
|
48
|
+
}
|
|
49
|
+
}, 10_000);
|
package/src/components/aws.ts
CHANGED
|
@@ -101,14 +101,25 @@ const AWS_CN_REGIONS = ["cn-north-1", "cn-northwest-1"];
|
|
|
101
101
|
|
|
102
102
|
/** Every hostname pattern the emulator answers to: the two partitions' global
|
|
103
103
|
* endpoints (`sts.amazonaws.com`, `iam.amazonaws.com`) plus one per region
|
|
104
|
-
* for the regional ones (`states.us-east-1.amazonaws.com`).
|
|
104
|
+
* for the regional ones (`states.us-east-1.amazonaws.com`). ECR adds
|
|
105
|
+
* labels for its API and account registries, each needing its own SAN. */
|
|
105
106
|
const AWS_HOST_PATTERNS = [
|
|
106
107
|
"*.amazonaws.com",
|
|
107
|
-
...AWS_REGIONS.
|
|
108
|
+
...AWS_REGIONS.flatMap((r) => regionalHostPatterns(r, "amazonaws.com")),
|
|
108
109
|
"*.amazonaws.com.cn",
|
|
109
|
-
...AWS_CN_REGIONS.
|
|
110
|
+
...AWS_CN_REGIONS.flatMap((r) => regionalHostPatterns(r, "amazonaws.com.cn")),
|
|
110
111
|
];
|
|
111
112
|
|
|
113
|
+
function regionalHostPatterns(region: string, suffix: string): string[] {
|
|
114
|
+
return [
|
|
115
|
+
`*.${region}.${suffix}`,
|
|
116
|
+
`*.ecr.${region}.${suffix}`,
|
|
117
|
+
`*.dkr.ecr.${region}.${suffix}`,
|
|
118
|
+
`*.ecr-fips.${region}.${suffix}`,
|
|
119
|
+
`*.dkr.ecr-fips.${region}.${suffix}`,
|
|
120
|
+
];
|
|
121
|
+
}
|
|
122
|
+
|
|
112
123
|
/** The VM's own docker socket, bind-mounted so the emulator can start the
|
|
113
124
|
* official AWS runtime containers. This is the VM's dockerd, inside the
|
|
114
125
|
* hermetic environment — the same daemon that runs the services. */
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
// the whole OAuth exchange is readable on the case page with no helper API.
|
|
31
31
|
|
|
32
32
|
import { createServer } from "@emulators/core";
|
|
33
|
-
import { SignJWT, decodeJwt, exportJWK } from "jose";
|
|
33
|
+
import { SignJWT, decodeJwt, exportJWK, exportPKCS8 } from "jose";
|
|
34
34
|
|
|
35
35
|
const CONFIG = JSON.parse(process.env.SPECTEST_AUTH_CONFIG ?? "{}");
|
|
36
36
|
const PORT = Number(CONFIG.port ?? 4000);
|
|
@@ -60,6 +60,37 @@ const JWKS = {
|
|
|
60
60
|
keys: [{ ...(await exportJWK(KEY_PAIR.publicKey)), kid: KID, use: "sig", alg: "RS256" }],
|
|
61
61
|
};
|
|
62
62
|
|
|
63
|
+
// Google's Node/Python clients fetch PEM certificates from /oauth2/v1/certs,
|
|
64
|
+
// while browser clients use the JWK endpoint. Both must identify the same key.
|
|
65
|
+
let GOOGLE_CERT;
|
|
66
|
+
if ((CONFIG.providers ?? []).some(provider => provider.name === "google")) {
|
|
67
|
+
const child = Bun.spawn(["openssl", "req", "-new", "-x509", "-key", "/dev/stdin", "-days", "3650", "-subj", "/CN=spectest-google-token-signing"], {
|
|
68
|
+
stdin: new Blob([await exportPKCS8(KEY_PAIR.privateKey)]), stdout: "pipe", stderr: "pipe",
|
|
69
|
+
});
|
|
70
|
+
const [cert, error, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]);
|
|
71
|
+
if (code !== 0) throw new Error(`google(): signing certificate setup failed: ${error}`);
|
|
72
|
+
GOOGLE_CERT = cert;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async function googleIdToken(req, path) {
|
|
76
|
+
const match = /^\/v1\/projects\/-\/serviceAccounts\/([^/]+):generateIdToken$/.exec(path);
|
|
77
|
+
if (!match || req.method !== "POST") return json({ error: "Unsupported IAM operation" }, 404);
|
|
78
|
+
let options, email;
|
|
79
|
+
try {
|
|
80
|
+
options = await req.json();
|
|
81
|
+
email = decodeURIComponent(match[1]);
|
|
82
|
+
} catch { return json({ error: "Invalid token request" }, 400); }
|
|
83
|
+
if (!options || !/^[^@\s]+@[^@\s]+$/.test(email) || typeof options.audience !== "string" || !options.audience) return json({ error: "serviceAccountEmail and audience are required" }, 400);
|
|
84
|
+
// Stable service-account subject; isolated test identities require no cloud credentials.
|
|
85
|
+
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(email)));
|
|
86
|
+
const subject = BigInt('0x' + Buffer.from(digest.slice(0, 8)).toString('hex')).toString();
|
|
87
|
+
const token = await new SignJWT(options.includeEmail ? { email, email_verified: true } : {})
|
|
88
|
+
.setProtectedHeader({ alg: "RS256", kid: KID, typ: "JWT" })
|
|
89
|
+
.setIssuer("https://accounts.google.com").setSubject(subject).setAudience(options.audience)
|
|
90
|
+
.setIssuedAt().setExpirationTime("1h").sign(KEY_PAIR.privateKey);
|
|
91
|
+
return json({ token }, 200);
|
|
92
|
+
}
|
|
93
|
+
|
|
63
94
|
// ──────────────────────────────────────────────────────────────────────────
|
|
64
95
|
// Providers
|
|
65
96
|
// ──────────────────────────────────────────────────────────────────────────
|
|
@@ -205,7 +236,11 @@ Bun.serve({
|
|
|
205
236
|
}
|
|
206
237
|
|
|
207
238
|
let res;
|
|
208
|
-
if (provider.
|
|
239
|
+
if (provider.name === "google" && host === "iamcredentials.googleapis.com") {
|
|
240
|
+
res = await googleIdToken(req, path);
|
|
241
|
+
} else if (provider.name === "google" && path === "/oauth2/v1/certs") {
|
|
242
|
+
res = json({ [KID]: GOOGLE_CERT }, 200);
|
|
243
|
+
} else if (provider.oidc && provider.jwksPath?.test(path)) {
|
|
209
244
|
// Our key, never the emulator's — we re-sign every id_token.
|
|
210
245
|
res = json(JWKS, 200);
|
|
211
246
|
} else {
|
|
@@ -155,6 +155,7 @@ const ENTRY_SCRIPT: string = (() => {
|
|
|
155
155
|
const DOCKERFILE = `
|
|
156
156
|
FROM ${IMAGE_BASE}
|
|
157
157
|
WORKDIR /srv
|
|
158
|
+
RUN apk add --no-cache openssl
|
|
158
159
|
RUN echo '{"name":"spectest-provider","private":true}' > package.json \\
|
|
159
160
|
&& bun add \\
|
|
160
161
|
@emulators/core@${EMULATE_VERSION} \\
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Google Cloud components for local Spectest environments. */
|
|
2
|
+
export {
|
|
3
|
+
pubsub,
|
|
4
|
+
pubsubClient,
|
|
5
|
+
type PubSubOptions,
|
|
6
|
+
type PubSubOidcToken,
|
|
7
|
+
type PubSubPushConfig,
|
|
8
|
+
type PubSubDeadLetterPolicy,
|
|
9
|
+
type PubSubRetryPolicy,
|
|
10
|
+
type PubSubClient,
|
|
11
|
+
type PubSubHelpers,
|
|
12
|
+
type PubSubSubscriptionOptions,
|
|
13
|
+
type PubSubMessage,
|
|
14
|
+
type PubSubReceivedMessage,
|
|
15
|
+
} from "./pubsub.js";
|
package/src/components/google.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// `google()` — Google, emulated inside the environment, answering at its
|
|
2
2
|
// real endpoints.
|
|
3
3
|
//
|
|
4
|
-
//
|
|
4
|
+
// Supports service-account ID tokens (IAM generateIdToken) and Sign in
|
|
5
|
+
// with Google: OpenID Connect discovery, the account
|
|
5
6
|
// picker, PKCE, the token exchange, refresh, and userinfo. The component is
|
|
6
7
|
// named for the provider rather than for the feature, because the backing
|
|
7
8
|
// emulator also carries Gmail, Calendar and Drive — claiming
|
|
@@ -27,6 +28,7 @@ function spec(users: ProviderUser[], opts: GoogleOptions): ProviderSpec {
|
|
|
27
28
|
baseUrl: "https://accounts.google.com",
|
|
28
29
|
hosts: [
|
|
29
30
|
"accounts.google.com",
|
|
31
|
+
"iamcredentials.googleapis.com",
|
|
30
32
|
"oauth2.googleapis.com",
|
|
31
33
|
"www.googleapis.com",
|
|
32
34
|
"openidconnect.googleapis.com",
|
package/src/components/index.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Reusable, pre-built service definitions for common dependencies.
|
|
2
|
-
export { pubsub, pubsubClient, type PubSubOptions, type PubSubClient, type PubSubHelpers,
|
|
2
|
+
export { pubsub, pubsubClient, type PubSubOptions, type PubSubOidcToken, type PubSubPushConfig, type PubSubDeadLetterPolicy, type PubSubRetryPolicy, type PubSubClient, type PubSubHelpers,
|
|
3
3
|
type PubSubSubscriptionOptions, type PubSubMessage, type PubSubReceivedMessage } from "./pubsub.js";
|
|
4
4
|
export { temporal, temporalClient, type TemporalOptions, type TemporalClient,
|
|
5
5
|
type TemporalNamespace, type TemporalWorkerOptions, type TemporalPoller,
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Authenticated push adapter; the emulator still owns scheduling and retries.
|
|
2
|
+
|
|
3
|
+
Runs only on loopback in the emulator container. Tokens come from the local
|
|
4
|
+
Google IAM fake, so recipients verify them with the same Google JWKS as usual.
|
|
5
|
+
Never follows redirects or acknowledges a failed token request/delivery.
|
|
6
|
+
"""
|
|
7
|
+
import base64
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import ssl
|
|
11
|
+
import sys
|
|
12
|
+
import urllib.error
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import urllib.request
|
|
15
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
19
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
20
|
+
return None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def opener():
|
|
24
|
+
context = ssl.create_default_context(cafile=os.environ.get("SSL_CERT_FILE"))
|
|
25
|
+
return urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Handler(BaseHTTPRequestHandler):
|
|
29
|
+
def do_POST(self):
|
|
30
|
+
phase = "configuration"
|
|
31
|
+
try:
|
|
32
|
+
if not self.path.startswith('/push/'):
|
|
33
|
+
self.send_error(404)
|
|
34
|
+
return
|
|
35
|
+
encoded = self.path[len('/push/'):]
|
|
36
|
+
config = json.loads(base64.urlsafe_b64decode(encoded + '=' * (-len(encoded) % 4)))
|
|
37
|
+
endpoint = config['endpoint']
|
|
38
|
+
if urllib.parse.urlsplit(endpoint).scheme not in ('http', 'https'):
|
|
39
|
+
raise ValueError('Invalid push endpoint')
|
|
40
|
+
size = int(self.headers.get('Content-Length', '0'))
|
|
41
|
+
if not 0 <= size <= 16 * 1024 * 1024:
|
|
42
|
+
self.send_error(413)
|
|
43
|
+
return
|
|
44
|
+
body = self.rfile.read(size)
|
|
45
|
+
account = urllib.parse.quote(config['serviceAccountEmail'], safe='')
|
|
46
|
+
token_url = 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + account + ':generateIdToken'
|
|
47
|
+
token_body = json.dumps({'audience': config.get('audience') or endpoint, 'includeEmail': True}).encode()
|
|
48
|
+
http = opener()
|
|
49
|
+
phase = "token issuance"
|
|
50
|
+
with http.open(urllib.request.Request(token_url, data=token_body, headers={'Content-Type': 'application/json'}), timeout=10) as response:
|
|
51
|
+
token = json.load(response)['token']
|
|
52
|
+
if not isinstance(token, str) or not token or '\r' in token or '\n' in token:
|
|
53
|
+
raise ValueError('Invalid ID token from Google IAM fake')
|
|
54
|
+
request = urllib.request.Request(endpoint, data=body, headers={
|
|
55
|
+
'Content-Type': self.headers.get('Content-Type', 'application/json'),
|
|
56
|
+
'Authorization': 'Bearer ' + token,
|
|
57
|
+
})
|
|
58
|
+
phase = "delivery"
|
|
59
|
+
with http.open(request, timeout=30) as response:
|
|
60
|
+
status = response.status
|
|
61
|
+
self.send_response(status)
|
|
62
|
+
self.end_headers()
|
|
63
|
+
except urllib.error.HTTPError as error:
|
|
64
|
+
# An unsuccessful token request or destination response must retry.
|
|
65
|
+
print(f"pubsub push {phase} returned HTTP {error.code}", flush=True)
|
|
66
|
+
self.send_response(error.code if error.code >= 300 else 502)
|
|
67
|
+
self.end_headers()
|
|
68
|
+
error.close()
|
|
69
|
+
except Exception as error:
|
|
70
|
+
# Do not log tokens, request bodies, or the encoded config URL.
|
|
71
|
+
print('pubsub push adapter failed: ' + type(error).__name__, flush=True)
|
|
72
|
+
self.send_response(502)
|
|
73
|
+
self.end_headers()
|
|
74
|
+
|
|
75
|
+
def log_message(self, format, *args):
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
if __name__ == '__main__':
|
|
80
|
+
server = ThreadingHTTPServer(('127.0.0.1', int(sys.argv[1])), Handler)
|
|
81
|
+
server.daemon_threads = True
|
|
82
|
+
print('pubsub authenticated push adapter ready', flush=True)
|
|
83
|
+
server.serve_forever()
|
|
@@ -6,8 +6,8 @@ describe("Pub/Sub", () => {
|
|
|
6
6
|
test("custom port and project configure both the emulator and API readiness", () => {
|
|
7
7
|
const env = defineEnvironment({ name: "test", services: { messaging: pubsub({ projectId: "local-project", port: 9095 }) } });
|
|
8
8
|
const service = env.config.services.messaging;
|
|
9
|
-
expect(service.args).toContain("--host-port=0.0.0.0:9095");
|
|
10
|
-
expect(service.args).toContain("--project=local-project");
|
|
9
|
+
expect(service.args?.join("\n")).toContain("--host-port=0.0.0.0:9095");
|
|
10
|
+
expect(service.args?.join("\n")).toContain("--project=local-project");
|
|
11
11
|
expect(service.tls).toEqual([{ hostname: "pubsub.googleapis.com", port: 9095, http2: true }]);
|
|
12
12
|
expect(Object.keys(env.config.services)).toEqual(["messaging"]);
|
|
13
13
|
const direct = defineEnvironment({ name: "test", services: { messaging: pubsub({ tls: false }) } });
|
|
@@ -86,3 +86,18 @@ describe("Pub/Sub", () => {
|
|
|
86
86
|
await expect(client.pull("worker", { timeoutMs: 10 })).rejects.toThrow();
|
|
87
87
|
});
|
|
88
88
|
});
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
test("subscription push authentication and delivery policies are passed through", async () => {
|
|
92
|
+
let body: any;
|
|
93
|
+
const client = pubsubClient("http://emulator:8085", "spectest", async (_, init) => {
|
|
94
|
+
body = JSON.parse(init!.body as string); return Response.json({});
|
|
95
|
+
});
|
|
96
|
+
const pushConfig = { pushEndpoint: "https://worker.run.app/push", oidcToken: { serviceAccountEmail: "push@spectest.iam.gserviceaccount.com", audience: "https://worker.run.app" }, attributes: { "x-goog-version": "v1" } };
|
|
97
|
+
const deadLetterPolicy = { deadLetterTopic: "projects/spectest/topics/dead-letters", maxDeliveryAttempts: 5 };
|
|
98
|
+
const retryPolicy = { minimumBackoff: "1s", maximumBackoff: "60s" };
|
|
99
|
+
await client.createSubscription("worker", { topic: "orders", pushConfig, deadLetterPolicy, retryPolicy });
|
|
100
|
+
expect(body).toEqual({ topic: "projects/spectest/topics/orders", ackDeadlineSeconds: 10, pushConfig, deadLetterPolicy, retryPolicy });
|
|
101
|
+
expect(() => client.createSubscription("worker", { topic: "orders", pushEndpoint: "https://other.run.app", pushConfig })).toThrow("disagree");
|
|
102
|
+
expect(() => client.createSubscription("worker", { topic: "orders", pushConfig: { ...pushConfig, oidcToken: { serviceAccountEmail: "invalid" } } })).toThrow("oidcToken");
|
|
103
|
+
});
|
package/src/components/pubsub.ts
CHANGED
|
@@ -1,14 +1,40 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
1
2
|
import type { ServiceDefinition, ServiceHelpersContext, ServiceSetupContext, ServicesMap } from "../index.js";
|
|
2
3
|
import { serviceGroup, certificate } from "../index.js";
|
|
3
4
|
import { rawFetch } from "../harness/raw-fetch.js";
|
|
4
5
|
import { readRaw } from "../inspect.js";
|
|
5
6
|
|
|
7
|
+
const PUSH_ADAPTER = readFileSync(new URL("./pubsub-push.py", import.meta.url), "utf8");
|
|
8
|
+
|
|
9
|
+
export interface PubSubOidcToken {
|
|
10
|
+
serviceAccountEmail: string;
|
|
11
|
+
/** Defaults to the original push endpoint URL. */
|
|
12
|
+
audience?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface PubSubPushConfig {
|
|
15
|
+
pushEndpoint: string;
|
|
16
|
+
oidcToken?: PubSubOidcToken;
|
|
17
|
+
attributes?: Record<string, string>;
|
|
18
|
+
}
|
|
19
|
+
export interface PubSubDeadLetterPolicy {
|
|
20
|
+
/** Fully qualified projects/{project}/topics/{topic} name. Create this topic separately. */
|
|
21
|
+
deadLetterTopic: string;
|
|
22
|
+
maxDeliveryAttempts?: number;
|
|
23
|
+
}
|
|
24
|
+
export interface PubSubRetryPolicy {
|
|
25
|
+
/** Google duration strings, for example "1s" and "60s". */
|
|
26
|
+
minimumBackoff?: string;
|
|
27
|
+
maximumBackoff?: string;
|
|
28
|
+
}
|
|
6
29
|
export interface PubSubSubscriptionOptions {
|
|
7
30
|
/** Topic ID in this component's project. */
|
|
8
31
|
topic: string;
|
|
9
32
|
ackDeadlineSeconds?: number;
|
|
10
|
-
/**
|
|
33
|
+
/** Shorthand for pushConfig.pushEndpoint. HTTP and HTTPS are supported. */
|
|
11
34
|
pushEndpoint?: string;
|
|
35
|
+
pushConfig?: PubSubPushConfig;
|
|
36
|
+
deadLetterPolicy?: PubSubDeadLetterPolicy;
|
|
37
|
+
retryPolicy?: PubSubRetryPolicy;
|
|
12
38
|
}
|
|
13
39
|
|
|
14
40
|
export interface PubSubOptions {
|
|
@@ -59,7 +85,12 @@ function subscriptionOptions(options: PubSubSubscriptionOptions) {
|
|
|
59
85
|
resourceId(options.topic);
|
|
60
86
|
const seconds = options.ackDeadlineSeconds ?? 10;
|
|
61
87
|
if (!Number.isInteger(seconds) || seconds < 10 || seconds > 600) throw new Error("pubsub(): ackDeadlineSeconds must be an integer between 10 and 600");
|
|
62
|
-
if (options.pushEndpoint &&
|
|
88
|
+
if (options.pushEndpoint && options.pushConfig && options.pushEndpoint !== options.pushConfig.pushEndpoint) throw new Error("pubsub(): pushEndpoint and pushConfig.pushEndpoint disagree");
|
|
89
|
+
const push = options.pushConfig ?? (options.pushEndpoint ? { pushEndpoint: options.pushEndpoint } : undefined);
|
|
90
|
+
if (push) {
|
|
91
|
+
if (!push.pushEndpoint || !/^https?:$/.test(new URL(push.pushEndpoint).protocol)) throw new Error("pubsub(): pushEndpoint must use HTTP or HTTPS");
|
|
92
|
+
if (push.oidcToken && (!/^[^@\s]+@[^@\s]+$/.test(push.oidcToken.serviceAccountEmail) || (push.oidcToken.audience !== undefined && !push.oidcToken.audience))) throw new Error("pubsub(): oidcToken requires a serviceAccountEmail and nonempty audience");
|
|
93
|
+
}
|
|
63
94
|
return seconds;
|
|
64
95
|
}
|
|
65
96
|
|
|
@@ -97,7 +128,9 @@ export function pubsubClient(
|
|
|
97
128
|
const ackDeadlineSeconds = subscriptionOptions(options);
|
|
98
129
|
return request<{ name: string; topic: string; ackDeadlineSeconds: number }>(path("subscriptions", id), "PUT", {
|
|
99
130
|
topic: resource("topics", options.topic), ackDeadlineSeconds,
|
|
100
|
-
...(options.pushEndpoint ? { pushConfig: { pushEndpoint: options.pushEndpoint } } : {}),
|
|
131
|
+
...(options.pushConfig ? { pushConfig: options.pushConfig } : options.pushEndpoint ? { pushConfig: { pushEndpoint: options.pushEndpoint } } : {}),
|
|
132
|
+
...(options.deadLetterPolicy ? { deadLetterPolicy: options.deadLetterPolicy } : {}),
|
|
133
|
+
...(options.retryPolicy ? { retryPolicy: options.retryPolicy } : {}),
|
|
101
134
|
});
|
|
102
135
|
},
|
|
103
136
|
async publish(topic: string, data: string | Uint8Array, attributes?: Record<string, string>): Promise<string> {
|
|
@@ -128,6 +161,23 @@ export interface PubSubHelpers extends PubSubClient {
|
|
|
128
161
|
emulatorHost: string;
|
|
129
162
|
}
|
|
130
163
|
|
|
164
|
+
// The official emulator ignores OIDC. Keep its push/retry/dead-letter engine,
|
|
165
|
+
// but route authenticated deliveries through a local adapter in its container.
|
|
166
|
+
function authenticatedPush(options: PubSubSubscriptionOptions, pushPort: number): PubSubSubscriptionOptions {
|
|
167
|
+
subscriptionOptions(options);
|
|
168
|
+
const push = options.pushConfig;
|
|
169
|
+
if (!push?.oidcToken) return options;
|
|
170
|
+
const config = Buffer.from(JSON.stringify({ endpoint: push.pushEndpoint, ...push.oidcToken })).toString("base64url");
|
|
171
|
+
return {
|
|
172
|
+
...options, pushEndpoint: undefined,
|
|
173
|
+
pushConfig: { ...push, pushEndpoint: `http://127.0.0.1:${pushPort}/push/${config}` },
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function withPushAuth(client: PubSubClient, pushPort: number): PubSubClient {
|
|
178
|
+
return { ...client, createSubscription: (id, options) => client.createSubscription(id, authenticatedPush(options, pushPort)) };
|
|
179
|
+
}
|
|
180
|
+
|
|
131
181
|
/** Google Cloud's real local Pub/Sub emulator, ready for Google client libraries.
|
|
132
182
|
* Mount directly in services: { pubsub: pubsub({ topics: ["orders"] }) }.
|
|
133
183
|
* Applications use their default pubsub.googleapis.com:443 endpoint with
|
|
@@ -137,6 +187,7 @@ export interface PubSubHelpers extends PubSubClient {
|
|
|
137
187
|
export function pubsub(opts: PubSubOptions = {}) {
|
|
138
188
|
const projectId = projectName(opts.projectId ?? "spectest");
|
|
139
189
|
const port = opts.port ?? 8085;
|
|
190
|
+
const pushPort = port === 8086 ? 8087 : 8086;
|
|
140
191
|
const hostname = opts.tls === false ? undefined : opts.tls?.hostname ?? "pubsub.googleapis.com";
|
|
141
192
|
if (hostname !== undefined) {
|
|
142
193
|
certificate([hostname]);
|
|
@@ -152,12 +203,20 @@ export function pubsub(opts: PubSubOptions = {}) {
|
|
|
152
203
|
services: (): ServicesMap => {
|
|
153
204
|
const emulator: ServiceDefinition = {
|
|
154
205
|
image: { type: "registry", reference: opts.image ?? "gcr.io/google.com/cloudsdktool/google-cloud-cli:573.0.0-emulators" },
|
|
155
|
-
args: ["
|
|
206
|
+
args: ["sh", "-ec", `
|
|
207
|
+
java_home=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")
|
|
208
|
+
cp "$java_home/lib/security/cacerts" /tmp/spectest-pubsub-cacerts
|
|
209
|
+
keytool -importcert -noprompt -alias spectest-environment -keystore /tmp/spectest-pubsub-cacerts -storepass changeit -file "$NODE_EXTRA_CA_CERTS"
|
|
210
|
+
export JAVA_TOOL_OPTIONS="-Djavax.net.ssl.trustStore=/tmp/spectest-pubsub-cacerts -Djavax.net.ssl.trustStorePassword=changeit \${JAVA_TOOL_OPTIONS:-}"
|
|
211
|
+
python3 /opt/spectest-pubsub-push.py ${pushPort} &
|
|
212
|
+
exec gcloud beta emulators pubsub start --project=${projectId} --host-port=0.0.0.0:${port} --quiet
|
|
213
|
+
`],
|
|
214
|
+
files: [{ path: "/opt/spectest-pubsub-push.py", content: PUSH_ADAPTER }],
|
|
156
215
|
ports: [port],
|
|
157
216
|
...(hostname ? { tls: [{ hostname, port, http2: true }] } : {}),
|
|
158
217
|
readyCheck: { type: "http", port, path: `/v1/projects/${projectId}/topics`, timeoutSecs: 60 },
|
|
159
218
|
setup: async ({ name }) => {
|
|
160
|
-
const client = pubsubClient(`http://${name}:${port}`, projectId);
|
|
219
|
+
const client = withPushAuth(pubsubClient(`http://${name}:${port}`, projectId), pushPort);
|
|
161
220
|
for (const topic of topics) await client.createTopic(topic);
|
|
162
221
|
for (const [id, options] of subscriptions) await client.createSubscription(id, options);
|
|
163
222
|
},
|
|
@@ -165,8 +224,8 @@ export function pubsub(opts: PubSubOptions = {}) {
|
|
|
165
224
|
return { emulator };
|
|
166
225
|
},
|
|
167
226
|
helpers: (ctx: ServiceHelpersContext): PubSubHelpers => ({
|
|
168
|
-
...pubsubClient(hostname ? `https://${hostname}` : `http://${ctx.name}:${port}`, projectId,
|
|
169
|
-
async (input, init) => readRaw(await ctx.fetch(String(input), init)) as unknown as Response),
|
|
227
|
+
...withPushAuth(pubsubClient(hostname ? `https://${hostname}` : `http://${ctx.name}:${port}`, projectId,
|
|
228
|
+
async (input, init) => readRaw(await ctx.fetch(String(input), init)) as unknown as Response), pushPort),
|
|
170
229
|
emulatorHost: `${ctx.name}:${port}`,
|
|
171
230
|
}),
|
|
172
231
|
setup: async (ctx: ServiceSetupContext<PubSubHelpers>) => {
|