@specific.dev/spectest 0.97.0 → 0.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -34
- package/dist/components/index.d.ts +0 -1
- package/dist/components/index.js +0 -1
- package/dist/components/supabase.d.ts +5 -1
- package/dist/components/supabase.js +3 -1
- package/dist/daemon.js +16 -22
- package/dist/harness/http-proxy.d.ts +9 -9
- package/dist/harness/http-proxy.js +14 -9
- package/dist/harness/ingress-table.d.ts +2 -0
- package/dist/index.d.ts +8 -2
- package/dist/index.js +3 -0
- package/dist/ingress.d.ts +8 -0
- package/dist/ingress.js +9 -2
- package/package.json +1 -6
- package/src/components/index.ts +0 -2
- package/src/components/supabase.ts +8 -1
- package/src/daemon.ts +17 -25
- package/src/harness/http-proxy.test.ts +2 -1
- package/src/harness/http-proxy.ts +20 -9
- package/src/harness/ingress-cors.test.ts +184 -0
- package/src/harness/ingress-table.ts +2 -1
- package/src/index.ts +10 -1
- package/src/ingress.ts +18 -6
- package/dist/components/gcp.d.ts +0 -2
- package/dist/components/gcp.js +0 -2
- package/dist/components/pubsub-push.py +0 -83
- package/dist/components/pubsub.d.ts +0 -100
- package/dist/components/pubsub.js +0 -175
- package/src/components/gcp.ts +0 -15
- package/src/components/pubsub-push.py +0 -83
- package/src/components/pubsub.test.ts +0 -103
- package/src/components/pubsub.ts +0 -241
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
2
|
-
import { serviceGroup, certificate } from "../index.js";
|
|
3
|
-
import { rawFetch } from "../harness/raw-fetch.js";
|
|
4
|
-
import { readRaw } from "../inspect.js";
|
|
5
|
-
const PUSH_ADAPTER = readFileSync(new URL("./pubsub-push.py", import.meta.url), "utf8");
|
|
6
|
-
function resourceId(id) {
|
|
7
|
-
if (!/^[A-Za-z][A-Za-z0-9._~+%-]{2,254}$/.test(id) || id.startsWith("goog")) {
|
|
8
|
-
throw new Error(`pubsub(): invalid topic or subscription ID ${JSON.stringify(id)}; use a 3–255 character ID starting with a letter, not "goog"`);
|
|
9
|
-
}
|
|
10
|
-
return id;
|
|
11
|
-
}
|
|
12
|
-
function projectName(id) {
|
|
13
|
-
if (!/^[a-z][a-z0-9-]{4,28}[a-z0-9]$/.test(id))
|
|
14
|
-
throw new Error("pubsub(): projectId must be a 6–30 character Google Cloud project ID");
|
|
15
|
-
return id;
|
|
16
|
-
}
|
|
17
|
-
function subscriptionOptions(options) {
|
|
18
|
-
resourceId(options.topic);
|
|
19
|
-
const seconds = options.ackDeadlineSeconds ?? 10;
|
|
20
|
-
if (!Number.isInteger(seconds) || seconds < 10 || seconds > 600)
|
|
21
|
-
throw new Error("pubsub(): ackDeadlineSeconds must be an integer between 10 and 600");
|
|
22
|
-
if (options.pushEndpoint && options.pushConfig && options.pushEndpoint !== options.pushConfig.pushEndpoint)
|
|
23
|
-
throw new Error("pubsub(): pushEndpoint and pushConfig.pushEndpoint disagree");
|
|
24
|
-
const push = options.pushConfig ?? (options.pushEndpoint ? { pushEndpoint: options.pushEndpoint } : undefined);
|
|
25
|
-
if (push) {
|
|
26
|
-
if (!push.pushEndpoint || !/^https?:$/.test(new URL(push.pushEndpoint).protocol))
|
|
27
|
-
throw new Error("pubsub(): pushEndpoint must use HTTP or HTTPS");
|
|
28
|
-
if (push.oidcToken && (!/^[^@\s]+@[^@\s]+$/.test(push.oidcToken.serviceAccountEmail) || (push.oidcToken.audience !== undefined && !push.oidcToken.audience)))
|
|
29
|
-
throw new Error("pubsub(): oidcToken requires a serviceAccountEmail and nonempty audience");
|
|
30
|
-
}
|
|
31
|
-
return seconds;
|
|
32
|
-
}
|
|
33
|
-
/** Stateless emulator REST client. No credentials or gRPC channel to preserve across forks.
|
|
34
|
-
* Resource arguments are short IDs in projectId; publish accepts UTF-8 text or bytes.
|
|
35
|
-
*/
|
|
36
|
-
export function pubsubClient(endpoint, projectId = "spectest", fetcher = rawFetch) {
|
|
37
|
-
projectName(projectId);
|
|
38
|
-
const base = endpoint.replace(/\/$/, "");
|
|
39
|
-
const resource = (kind, id) => `projects/${projectId}/${kind}/${resourceId(id)}`;
|
|
40
|
-
const path = (kind, id) => `/v1/projects/${projectId}/${kind}/${encodeURIComponent(resourceId(id))}`;
|
|
41
|
-
async function request(url, method, body, timeoutMs = 10000) {
|
|
42
|
-
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
|
|
43
|
-
throw new Error("pubsub(): timeoutMs must be positive");
|
|
44
|
-
const response = await fetcher(`${base}${url}`, {
|
|
45
|
-
method, headers: { "content-type": "application/json" },
|
|
46
|
-
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
47
|
-
signal: AbortSignal.timeout(Math.ceil(timeoutMs)),
|
|
48
|
-
});
|
|
49
|
-
const text = await response.text();
|
|
50
|
-
if (!response.ok)
|
|
51
|
-
throw new Error(`Pub/Sub ${method} ${url}: HTTP ${response.status}: ${text.slice(0, 1000)}`);
|
|
52
|
-
return (text ? JSON.parse(text) : {});
|
|
53
|
-
}
|
|
54
|
-
return {
|
|
55
|
-
projectId, endpoint: base,
|
|
56
|
-
topicName: (id) => resource("topics", id),
|
|
57
|
-
subscriptionName: (id) => resource("subscriptions", id),
|
|
58
|
-
createTopic(id) {
|
|
59
|
-
return request(path("topics", id), "PUT", {});
|
|
60
|
-
},
|
|
61
|
-
createSubscription(id, options) {
|
|
62
|
-
const ackDeadlineSeconds = subscriptionOptions(options);
|
|
63
|
-
return request(path("subscriptions", id), "PUT", {
|
|
64
|
-
topic: resource("topics", options.topic), ackDeadlineSeconds,
|
|
65
|
-
...(options.pushConfig ? { pushConfig: options.pushConfig } : options.pushEndpoint ? { pushConfig: { pushEndpoint: options.pushEndpoint } } : {}),
|
|
66
|
-
...(options.deadLetterPolicy ? { deadLetterPolicy: options.deadLetterPolicy } : {}),
|
|
67
|
-
...(options.retryPolicy ? { retryPolicy: options.retryPolicy } : {}),
|
|
68
|
-
});
|
|
69
|
-
},
|
|
70
|
-
async publish(topic, data, attributes) {
|
|
71
|
-
const result = await request(`${path("topics", topic)}:publish`, "POST", {
|
|
72
|
-
messages: [{ data: Buffer.from(data).toString("base64"), ...(attributes ? { attributes } : {}) }],
|
|
73
|
-
});
|
|
74
|
-
return result.messageIds[0];
|
|
75
|
-
},
|
|
76
|
-
/** A pull leases messages; call acknowledge() to remove them. Empty is valid.
|
|
77
|
-
* Uses the emulator's immediate pull mode for bounded test polling.
|
|
78
|
-
*/
|
|
79
|
-
async pull(subscription, options = {}) {
|
|
80
|
-
const maxMessages = options.maxMessages ?? 10;
|
|
81
|
-
if (!Number.isInteger(maxMessages) || maxMessages < 1)
|
|
82
|
-
throw new Error("pubsub(): maxMessages must be a positive integer");
|
|
83
|
-
const result = await request(`${path("subscriptions", subscription)}:pull`, "POST", { maxMessages, returnImmediately: true }, options.timeoutMs);
|
|
84
|
-
return result.receivedMessages ?? [];
|
|
85
|
-
},
|
|
86
|
-
async acknowledge(subscription, ackIds) {
|
|
87
|
-
await request(`${path("subscriptions", subscription)}:acknowledge`, "POST", { ackIds });
|
|
88
|
-
},
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
// The official emulator ignores OIDC. Keep its push/retry/dead-letter engine,
|
|
92
|
-
// but route authenticated deliveries through a local adapter in its container.
|
|
93
|
-
function authenticatedPush(options, pushPort) {
|
|
94
|
-
subscriptionOptions(options);
|
|
95
|
-
const push = options.pushConfig;
|
|
96
|
-
if (!push?.oidcToken)
|
|
97
|
-
return options;
|
|
98
|
-
const config = Buffer.from(JSON.stringify({ endpoint: push.pushEndpoint, ...push.oidcToken })).toString("base64url");
|
|
99
|
-
return {
|
|
100
|
-
...options, pushEndpoint: undefined,
|
|
101
|
-
pushConfig: { ...push, pushEndpoint: `http://127.0.0.1:${pushPort}/push/${config}` },
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
function withPushAuth(client, pushPort) {
|
|
105
|
-
return { ...client, createSubscription: (id, options) => client.createSubscription(id, authenticatedPush(options, pushPort)) };
|
|
106
|
-
}
|
|
107
|
-
/** Google Cloud's real local Pub/Sub emulator, ready for Google client libraries.
|
|
108
|
-
* Mount directly in services: { pubsub: pubsub({ topics: ["orders"] }) }.
|
|
109
|
-
* Applications use their default pubsub.googleapis.com:443 endpoint with
|
|
110
|
-
* verified TLS and local test credentials. No emulator endpoint override.
|
|
111
|
-
* State survives VM snapshots/forks, but not an emulator process restart.
|
|
112
|
-
*/
|
|
113
|
-
export function pubsub(opts = {}) {
|
|
114
|
-
const projectId = projectName(opts.projectId ?? "spectest");
|
|
115
|
-
const port = opts.port ?? 8085;
|
|
116
|
-
const pushPort = port === 8086 ? 8087 : 8086;
|
|
117
|
-
const hostname = opts.tls === false ? undefined : opts.tls?.hostname ?? "pubsub.googleapis.com";
|
|
118
|
-
if (hostname !== undefined) {
|
|
119
|
-
certificate([hostname]);
|
|
120
|
-
if (hostname.startsWith("*."))
|
|
121
|
-
throw new Error("pubsub(): TLS hostname must be an exact hostname");
|
|
122
|
-
}
|
|
123
|
-
if (!Number.isInteger(port) || port < 1 || port > 65535)
|
|
124
|
-
throw new Error("pubsub(): port must be an integer between 1 and 65535");
|
|
125
|
-
const subscriptions = Object.entries(opts.subscriptions ?? {});
|
|
126
|
-
for (const [id, options] of subscriptions) {
|
|
127
|
-
resourceId(id);
|
|
128
|
-
subscriptionOptions(options);
|
|
129
|
-
}
|
|
130
|
-
const topics = [...new Set([...(opts.topics ?? []), ...subscriptions.map(([, options]) => options.topic)])].map(resourceId);
|
|
131
|
-
if (hostname && port === 443)
|
|
132
|
-
throw new Error("pubsub(): the plaintext emulator port must differ from TLS port 443");
|
|
133
|
-
return serviceGroup({
|
|
134
|
-
primary: "emulator",
|
|
135
|
-
services: () => {
|
|
136
|
-
const emulator = {
|
|
137
|
-
image: { type: "registry", reference: opts.image ?? "gcr.io/google.com/cloudsdktool/google-cloud-cli:573.0.0-emulators" },
|
|
138
|
-
args: ["sh", "-ec", `
|
|
139
|
-
java_home=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")
|
|
140
|
-
cp "$java_home/lib/security/cacerts" /tmp/spectest-pubsub-cacerts
|
|
141
|
-
keytool -importcert -noprompt -alias spectest-environment -keystore /tmp/spectest-pubsub-cacerts -storepass changeit -file "$NODE_EXTRA_CA_CERTS"
|
|
142
|
-
export JAVA_TOOL_OPTIONS="-Djavax.net.ssl.trustStore=/tmp/spectest-pubsub-cacerts -Djavax.net.ssl.trustStorePassword=changeit \${JAVA_TOOL_OPTIONS:-}"
|
|
143
|
-
python3 /opt/spectest-pubsub-push.py ${pushPort} &
|
|
144
|
-
exec gcloud beta emulators pubsub start --project=${projectId} --host-port=0.0.0.0:${port} --quiet
|
|
145
|
-
`],
|
|
146
|
-
files: [{ path: "/opt/spectest-pubsub-push.py", content: PUSH_ADAPTER }],
|
|
147
|
-
ports: [port],
|
|
148
|
-
...(hostname ? { tls: [{ hostname, port, http2: true }] } : {}),
|
|
149
|
-
readyCheck: { type: "http", port, path: `/v1/projects/${projectId}/topics`, timeoutSecs: 60 },
|
|
150
|
-
setup: async ({ name }) => {
|
|
151
|
-
const client = withPushAuth(pubsubClient(`http://${name}:${port}`, projectId), pushPort);
|
|
152
|
-
for (const topic of topics)
|
|
153
|
-
await client.createTopic(topic);
|
|
154
|
-
for (const [id, options] of subscriptions)
|
|
155
|
-
await client.createSubscription(id, options);
|
|
156
|
-
},
|
|
157
|
-
};
|
|
158
|
-
return { emulator };
|
|
159
|
-
},
|
|
160
|
-
helpers: (ctx) => ({
|
|
161
|
-
...withPushAuth(pubsubClient(hostname ? `https://${hostname}` : `http://${ctx.name}:${port}`, projectId, async (input, init) => readRaw(await ctx.fetch(String(input), init))), pushPort),
|
|
162
|
-
emulatorHost: `${ctx.name}:${port}`,
|
|
163
|
-
}),
|
|
164
|
-
setup: async (ctx) => {
|
|
165
|
-
// Prove the TLS endpoint and certificate are ready before dependents.
|
|
166
|
-
if (hostname) {
|
|
167
|
-
const response = await rawFetch(`https://${hostname}/v1/projects/${projectId}/topics`, { signal: AbortSignal.timeout(10000) });
|
|
168
|
-
if (!response.ok)
|
|
169
|
-
throw new Error(`pubsub(): TLS readiness failed: HTTP ${response.status}`);
|
|
170
|
-
await response.text();
|
|
171
|
-
}
|
|
172
|
-
await opts.setup?.(ctx);
|
|
173
|
-
},
|
|
174
|
-
});
|
|
175
|
-
}
|
package/src/components/gcp.ts
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
/** Google Cloud components for local Spectest environments. */
|
|
2
|
-
export {
|
|
3
|
-
pubsub,
|
|
4
|
-
pubsubClient,
|
|
5
|
-
type PubSubOptions,
|
|
6
|
-
type PubSubOidcToken,
|
|
7
|
-
type PubSubPushConfig,
|
|
8
|
-
type PubSubDeadLetterPolicy,
|
|
9
|
-
type PubSubRetryPolicy,
|
|
10
|
-
type PubSubClient,
|
|
11
|
-
type PubSubHelpers,
|
|
12
|
-
type PubSubSubscriptionOptions,
|
|
13
|
-
type PubSubMessage,
|
|
14
|
-
type PubSubReceivedMessage,
|
|
15
|
-
} from "./pubsub.js";
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
"""Authenticated push adapter; the emulator still owns scheduling and retries.
|
|
2
|
-
|
|
3
|
-
Runs only on loopback in the emulator container. Tokens come from the local
|
|
4
|
-
Google IAM fake, so recipients verify them with the same Google JWKS as usual.
|
|
5
|
-
Never follows redirects or acknowledges a failed token request/delivery.
|
|
6
|
-
"""
|
|
7
|
-
import base64
|
|
8
|
-
import json
|
|
9
|
-
import os
|
|
10
|
-
import ssl
|
|
11
|
-
import sys
|
|
12
|
-
import urllib.error
|
|
13
|
-
import urllib.parse
|
|
14
|
-
import urllib.request
|
|
15
|
-
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
19
|
-
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
20
|
-
return None
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def opener():
|
|
24
|
-
context = ssl.create_default_context(cafile=os.environ.get("SSL_CERT_FILE"))
|
|
25
|
-
return urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context))
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class Handler(BaseHTTPRequestHandler):
|
|
29
|
-
def do_POST(self):
|
|
30
|
-
phase = "configuration"
|
|
31
|
-
try:
|
|
32
|
-
if not self.path.startswith('/push/'):
|
|
33
|
-
self.send_error(404)
|
|
34
|
-
return
|
|
35
|
-
encoded = self.path[len('/push/'):]
|
|
36
|
-
config = json.loads(base64.urlsafe_b64decode(encoded + '=' * (-len(encoded) % 4)))
|
|
37
|
-
endpoint = config['endpoint']
|
|
38
|
-
if urllib.parse.urlsplit(endpoint).scheme not in ('http', 'https'):
|
|
39
|
-
raise ValueError('Invalid push endpoint')
|
|
40
|
-
size = int(self.headers.get('Content-Length', '0'))
|
|
41
|
-
if not 0 <= size <= 16 * 1024 * 1024:
|
|
42
|
-
self.send_error(413)
|
|
43
|
-
return
|
|
44
|
-
body = self.rfile.read(size)
|
|
45
|
-
account = urllib.parse.quote(config['serviceAccountEmail'], safe='')
|
|
46
|
-
token_url = 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/' + account + ':generateIdToken'
|
|
47
|
-
token_body = json.dumps({'audience': config.get('audience') or endpoint, 'includeEmail': True}).encode()
|
|
48
|
-
http = opener()
|
|
49
|
-
phase = "token issuance"
|
|
50
|
-
with http.open(urllib.request.Request(token_url, data=token_body, headers={'Content-Type': 'application/json'}), timeout=10) as response:
|
|
51
|
-
token = json.load(response)['token']
|
|
52
|
-
if not isinstance(token, str) or not token or '\r' in token or '\n' in token:
|
|
53
|
-
raise ValueError('Invalid ID token from Google IAM fake')
|
|
54
|
-
request = urllib.request.Request(endpoint, data=body, headers={
|
|
55
|
-
'Content-Type': self.headers.get('Content-Type', 'application/json'),
|
|
56
|
-
'Authorization': 'Bearer ' + token,
|
|
57
|
-
})
|
|
58
|
-
phase = "delivery"
|
|
59
|
-
with http.open(request, timeout=30) as response:
|
|
60
|
-
status = response.status
|
|
61
|
-
self.send_response(status)
|
|
62
|
-
self.end_headers()
|
|
63
|
-
except urllib.error.HTTPError as error:
|
|
64
|
-
# An unsuccessful token request or destination response must retry.
|
|
65
|
-
print(f"pubsub push {phase} returned HTTP {error.code}", flush=True)
|
|
66
|
-
self.send_response(error.code if error.code >= 300 else 502)
|
|
67
|
-
self.end_headers()
|
|
68
|
-
error.close()
|
|
69
|
-
except Exception as error:
|
|
70
|
-
# Do not log tokens, request bodies, or the encoded config URL.
|
|
71
|
-
print('pubsub push adapter failed: ' + type(error).__name__, flush=True)
|
|
72
|
-
self.send_response(502)
|
|
73
|
-
self.end_headers()
|
|
74
|
-
|
|
75
|
-
def log_message(self, format, *args):
|
|
76
|
-
pass
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
if __name__ == '__main__':
|
|
80
|
-
server = ThreadingHTTPServer(('127.0.0.1', int(sys.argv[1])), Handler)
|
|
81
|
-
server.daemon_threads = True
|
|
82
|
-
print('pubsub authenticated push adapter ready', flush=True)
|
|
83
|
-
server.serve_forever()
|
|
@@ -1,103 +0,0 @@
|
|
|
1
|
-
import { describe, expect, test } from "bun:test";
|
|
2
|
-
import { defineEnvironment } from "../index.js";
|
|
3
|
-
import { pubsub, pubsubClient } from "./pubsub.js";
|
|
4
|
-
|
|
5
|
-
describe("Pub/Sub", () => {
|
|
6
|
-
test("custom port and project configure both the emulator and API readiness", () => {
|
|
7
|
-
const env = defineEnvironment({ name: "test", services: { messaging: pubsub({ projectId: "local-project", port: 9095 }) } });
|
|
8
|
-
const service = env.config.services.messaging;
|
|
9
|
-
expect(service.args?.join("\n")).toContain("--host-port=0.0.0.0:9095");
|
|
10
|
-
expect(service.args?.join("\n")).toContain("--project=local-project");
|
|
11
|
-
expect(service.tls).toEqual([{ hostname: "pubsub.googleapis.com", port: 9095, http2: true }]);
|
|
12
|
-
expect(Object.keys(env.config.services)).toEqual(["messaging"]);
|
|
13
|
-
const direct = defineEnvironment({ name: "test", services: { messaging: pubsub({ tls: false }) } });
|
|
14
|
-
expect(Object.keys(direct.config.services)).toEqual(["messaging"]);
|
|
15
|
-
expect(direct.config.services.messaging.certificates).toBeUndefined();
|
|
16
|
-
const custom = defineEnvironment({ name: "test", services: { messaging: pubsub({ tls: { hostname: "other.pubsub.test" } }) } });
|
|
17
|
-
expect(custom.config.services.messaging.tls).toEqual([{ hostname: "other.pubsub.test", port: 8085, http2: true }]);
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
test("rejects invalid configuration before launching a container", () => {
|
|
21
|
-
for (const port of [0, 65536, 1.5, NaN]) expect(() => pubsub({ port })).toThrow("port");
|
|
22
|
-
expect(() => pubsub({ projectId: "../bad" })).toThrow("projectId");
|
|
23
|
-
expect(() => pubsub({ port: 443 })).toThrow("TLS port");
|
|
24
|
-
expect(() => pubsub({ tls: { hostname: "invalid/path" } })).toThrow("hostname");
|
|
25
|
-
expect(() => pubsub({ tls: { hostname: "" } })).toThrow("hostname");
|
|
26
|
-
expect(() => pubsub({ tls: { hostname: "*.pubsub.test" } })).toThrow("exact hostname");
|
|
27
|
-
for (const topic of ["ab", "goog-topic", "projects/another/topics/topic", "../topic"]) expect(() => pubsub({ topics: [topic] })).toThrow("ID");
|
|
28
|
-
expect(() => pubsub({ subscriptions: { worker: { topic: "orders", ackDeadlineSeconds: 601 } } })).toThrow("ackDeadlineSeconds");
|
|
29
|
-
expect(() => pubsub({ subscriptions: { worker: { topic: "orders", pushEndpoint: "ftp://app" } } })).toThrow("HTTP");
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
test("publish encodes UTF-8 and arbitrary bytes exactly once and preserves attributes", async () => {
|
|
33
|
-
const bodies: any[] = [];
|
|
34
|
-
const client = pubsubClient("http://messaging:8085", "spectest", async (_, init) => {
|
|
35
|
-
bodies.push(JSON.parse(init!.body as string));
|
|
36
|
-
return Response.json({ messageIds: ["123"] });
|
|
37
|
-
});
|
|
38
|
-
expect(await client.publish("orders", "café ☕", { source: "test" })).toBe("123");
|
|
39
|
-
expect(Buffer.from(bodies[0].messages[0].data, "base64").toString()).toBe("café ☕");
|
|
40
|
-
expect(bodies[0].messages[0].attributes).toEqual({ source: "test" });
|
|
41
|
-
await client.publish("orders", new Uint8Array([0, 128, 255]));
|
|
42
|
-
expect([...Buffer.from(bodies[1].messages[0].data, "base64")]).toEqual([0, 128, 255]);
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
test("pull returns ack IDs and leaves acknowledgement explicit", async () => {
|
|
46
|
-
const calls: { url: string; body: any }[] = [];
|
|
47
|
-
const client = pubsubClient("http://messaging:8085/", "spectest", async (url, init) => {
|
|
48
|
-
calls.push({ url: String(url), body: JSON.parse(init!.body as string) });
|
|
49
|
-
return Response.json(calls.length === 1 ? { receivedMessages: [{ ackId: "ack-1", message: { data: "aGk=", messageId: "1" } }] } : {});
|
|
50
|
-
});
|
|
51
|
-
const messages = await client.pull("worker", { maxMessages: 1 });
|
|
52
|
-
expect(messages[0].ackId).toBe("ack-1");
|
|
53
|
-
expect(calls).toHaveLength(1);
|
|
54
|
-
expect(calls[0].body).toEqual({ maxMessages: 1, returnImmediately: true });
|
|
55
|
-
await client.acknowledge("worker", [messages[0].ackId]);
|
|
56
|
-
expect(calls[1]).toEqual({ url: "http://messaging:8085/v1/projects/spectest/subscriptions/worker:acknowledge", body: { ackIds: ["ack-1"] } });
|
|
57
|
-
expect(await client.pull("worker")).toEqual([]);
|
|
58
|
-
await expect(client.pull("worker", { maxMessages: 0 })).rejects.toThrow("maxMessages");
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
test("API errors include operation and status and never retry a publish", async () => {
|
|
62
|
-
let calls = 0;
|
|
63
|
-
const client = pubsubClient("http://messaging:8085", "spectest", async () => {
|
|
64
|
-
calls++;
|
|
65
|
-
return Response.json({ error: { message: "Topic not found" } }, { status: 404 });
|
|
66
|
-
});
|
|
67
|
-
await expect(client.publish("missing", "data")).rejects.toThrow("HTTP 404");
|
|
68
|
-
expect(calls).toBe(1);
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
test("resource IDs are escaped in URLs but remain canonical in subscription bodies", async () => {
|
|
72
|
-
let requestUrl = "", body: any;
|
|
73
|
-
const client = pubsubClient("http://messaging:8085", "spectest", async (url, init) => {
|
|
74
|
-
requestUrl = String(url); body = JSON.parse(init!.body as string);
|
|
75
|
-
return Response.json({});
|
|
76
|
-
});
|
|
77
|
-
await client.createSubscription("worker%one", { topic: "topic+one", pushEndpoint: "http://app:3000/push" });
|
|
78
|
-
expect(requestUrl).toEndWith("/subscriptions/worker%25one");
|
|
79
|
-
expect(body).toEqual({ topic: "projects/spectest/topics/topic+one", ackDeadlineSeconds: 10, pushConfig: { pushEndpoint: "http://app:3000/push" } });
|
|
80
|
-
});
|
|
81
|
-
|
|
82
|
-
test("every request has a bounded timeout", async () => {
|
|
83
|
-
const client = pubsubClient("http://messaging:8085", "spectest", async (_, init) => {
|
|
84
|
-
return new Promise<Response>((_, reject) => init!.signal!.addEventListener("abort", () => reject(init!.signal!.reason), { once: true }));
|
|
85
|
-
});
|
|
86
|
-
await expect(client.pull("worker", { timeoutMs: 10 })).rejects.toThrow();
|
|
87
|
-
});
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
test("subscription push authentication and delivery policies are passed through", async () => {
|
|
92
|
-
let body: any;
|
|
93
|
-
const client = pubsubClient("http://emulator:8085", "spectest", async (_, init) => {
|
|
94
|
-
body = JSON.parse(init!.body as string); return Response.json({});
|
|
95
|
-
});
|
|
96
|
-
const pushConfig = { pushEndpoint: "https://worker.run.app/push", oidcToken: { serviceAccountEmail: "push@spectest.iam.gserviceaccount.com", audience: "https://worker.run.app" }, attributes: { "x-goog-version": "v1" } };
|
|
97
|
-
const deadLetterPolicy = { deadLetterTopic: "projects/spectest/topics/dead-letters", maxDeliveryAttempts: 5 };
|
|
98
|
-
const retryPolicy = { minimumBackoff: "1s", maximumBackoff: "60s" };
|
|
99
|
-
await client.createSubscription("worker", { topic: "orders", pushConfig, deadLetterPolicy, retryPolicy });
|
|
100
|
-
expect(body).toEqual({ topic: "projects/spectest/topics/orders", ackDeadlineSeconds: 10, pushConfig, deadLetterPolicy, retryPolicy });
|
|
101
|
-
expect(() => client.createSubscription("worker", { topic: "orders", pushEndpoint: "https://other.run.app", pushConfig })).toThrow("disagree");
|
|
102
|
-
expect(() => client.createSubscription("worker", { topic: "orders", pushConfig: { ...pushConfig, oidcToken: { serviceAccountEmail: "invalid" } } })).toThrow("oidcToken");
|
|
103
|
-
});
|
package/src/components/pubsub.ts
DELETED
|
@@ -1,241 +0,0 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
2
|
-
import type { ServiceDefinition, ServiceHelpersContext, ServiceSetupContext, ServicesMap } from "../index.js";
|
|
3
|
-
import { serviceGroup, certificate } from "../index.js";
|
|
4
|
-
import { rawFetch } from "../harness/raw-fetch.js";
|
|
5
|
-
import { readRaw } from "../inspect.js";
|
|
6
|
-
|
|
7
|
-
const PUSH_ADAPTER = readFileSync(new URL("./pubsub-push.py", import.meta.url), "utf8");
|
|
8
|
-
|
|
9
|
-
export interface PubSubOidcToken {
|
|
10
|
-
serviceAccountEmail: string;
|
|
11
|
-
/** Defaults to the original push endpoint URL. */
|
|
12
|
-
audience?: string;
|
|
13
|
-
}
|
|
14
|
-
export interface PubSubPushConfig {
|
|
15
|
-
pushEndpoint: string;
|
|
16
|
-
oidcToken?: PubSubOidcToken;
|
|
17
|
-
attributes?: Record<string, string>;
|
|
18
|
-
}
|
|
19
|
-
export interface PubSubDeadLetterPolicy {
|
|
20
|
-
/** Fully qualified projects/{project}/topics/{topic} name. Create this topic separately. */
|
|
21
|
-
deadLetterTopic: string;
|
|
22
|
-
maxDeliveryAttempts?: number;
|
|
23
|
-
}
|
|
24
|
-
export interface PubSubRetryPolicy {
|
|
25
|
-
/** Google duration strings, for example "1s" and "60s". */
|
|
26
|
-
minimumBackoff?: string;
|
|
27
|
-
maximumBackoff?: string;
|
|
28
|
-
}
|
|
29
|
-
export interface PubSubSubscriptionOptions {
|
|
30
|
-
/** Topic ID in this component's project. */
|
|
31
|
-
topic: string;
|
|
32
|
-
ackDeadlineSeconds?: number;
|
|
33
|
-
/** Shorthand for pushConfig.pushEndpoint. HTTP and HTTPS are supported. */
|
|
34
|
-
pushEndpoint?: string;
|
|
35
|
-
pushConfig?: PubSubPushConfig;
|
|
36
|
-
deadLetterPolicy?: PubSubDeadLetterPolicy;
|
|
37
|
-
retryPolicy?: PubSubRetryPolicy;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export interface PubSubOptions {
|
|
41
|
-
/** Local project ID; no Google Cloud account is needed. Default "spectest". */
|
|
42
|
-
projectId?: string;
|
|
43
|
-
/** Official Cloud CLI image with the Pub/Sub emulator installed. */
|
|
44
|
-
image?: string;
|
|
45
|
-
/** Emulator HTTP and gRPC port. Default 8085. */
|
|
46
|
-
port?: number;
|
|
47
|
-
/** TLS on pubsub.googleapis.com by default. Set false for direct emulator
|
|
48
|
-
* access, or choose a different hostname for a second isolated instance. */
|
|
49
|
-
tls?: false | { hostname: string };
|
|
50
|
-
topics?: string[];
|
|
51
|
-
/** Subscription ID → options. Referenced topics are created automatically. */
|
|
52
|
-
subscriptions?: Record<string, PubSubSubscriptionOptions>;
|
|
53
|
-
/** Runs after the declared topics and subscriptions have been created. */
|
|
54
|
-
setup?: (ctx: ServiceSetupContext<PubSubHelpers>) => void | Promise<void>;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export interface PubSubMessage {
|
|
58
|
-
/** Base64 encoded data, as returned by the Pub/Sub REST API. */
|
|
59
|
-
data?: string;
|
|
60
|
-
attributes?: Record<string, string>;
|
|
61
|
-
messageId: string;
|
|
62
|
-
publishTime: string;
|
|
63
|
-
orderingKey?: string;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
export interface PubSubReceivedMessage {
|
|
67
|
-
ackId: string;
|
|
68
|
-
message: PubSubMessage;
|
|
69
|
-
deliveryAttempt?: number;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function resourceId(id: string): string {
|
|
73
|
-
if (!/^[A-Za-z][A-Za-z0-9._~+%-]{2,254}$/.test(id) || id.startsWith("goog")) {
|
|
74
|
-
throw new Error(`pubsub(): invalid topic or subscription ID ${JSON.stringify(id)}; use a 3–255 character ID starting with a letter, not "goog"`);
|
|
75
|
-
}
|
|
76
|
-
return id;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function projectName(id: string): string {
|
|
80
|
-
if (!/^[a-z][a-z0-9-]{4,28}[a-z0-9]$/.test(id)) throw new Error("pubsub(): projectId must be a 6–30 character Google Cloud project ID");
|
|
81
|
-
return id;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
function subscriptionOptions(options: PubSubSubscriptionOptions) {
|
|
85
|
-
resourceId(options.topic);
|
|
86
|
-
const seconds = options.ackDeadlineSeconds ?? 10;
|
|
87
|
-
if (!Number.isInteger(seconds) || seconds < 10 || seconds > 600) throw new Error("pubsub(): ackDeadlineSeconds must be an integer between 10 and 600");
|
|
88
|
-
if (options.pushEndpoint && options.pushConfig && options.pushEndpoint !== options.pushConfig.pushEndpoint) throw new Error("pubsub(): pushEndpoint and pushConfig.pushEndpoint disagree");
|
|
89
|
-
const push = options.pushConfig ?? (options.pushEndpoint ? { pushEndpoint: options.pushEndpoint } : undefined);
|
|
90
|
-
if (push) {
|
|
91
|
-
if (!push.pushEndpoint || !/^https?:$/.test(new URL(push.pushEndpoint).protocol)) throw new Error("pubsub(): pushEndpoint must use HTTP or HTTPS");
|
|
92
|
-
if (push.oidcToken && (!/^[^@\s]+@[^@\s]+$/.test(push.oidcToken.serviceAccountEmail) || (push.oidcToken.audience !== undefined && !push.oidcToken.audience))) throw new Error("pubsub(): oidcToken requires a serviceAccountEmail and nonempty audience");
|
|
93
|
-
}
|
|
94
|
-
return seconds;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/** Stateless emulator REST client. No credentials or gRPC channel to preserve across forks.
|
|
98
|
-
* Resource arguments are short IDs in projectId; publish accepts UTF-8 text or bytes.
|
|
99
|
-
*/
|
|
100
|
-
export function pubsubClient(
|
|
101
|
-
endpoint: string,
|
|
102
|
-
projectId = "spectest",
|
|
103
|
-
fetcher: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = rawFetch,
|
|
104
|
-
) {
|
|
105
|
-
projectName(projectId);
|
|
106
|
-
const base = endpoint.replace(/\/$/, "");
|
|
107
|
-
const resource = (kind: "topics" | "subscriptions", id: string) => `projects/${projectId}/${kind}/${resourceId(id)}`;
|
|
108
|
-
const path = (kind: "topics" | "subscriptions", id: string) => `/v1/projects/${projectId}/${kind}/${encodeURIComponent(resourceId(id))}`;
|
|
109
|
-
async function request<T>(url: string, method: string, body?: unknown, timeoutMs = 10000): Promise<T> {
|
|
110
|
-
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error("pubsub(): timeoutMs must be positive");
|
|
111
|
-
const response = await fetcher(`${base}${url}`, {
|
|
112
|
-
method, headers: { "content-type": "application/json" },
|
|
113
|
-
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
114
|
-
signal: AbortSignal.timeout(Math.ceil(timeoutMs)),
|
|
115
|
-
});
|
|
116
|
-
const text = await response.text();
|
|
117
|
-
if (!response.ok) throw new Error(`Pub/Sub ${method} ${url}: HTTP ${response.status}: ${text.slice(0, 1000)}`);
|
|
118
|
-
return (text ? JSON.parse(text) : {}) as T;
|
|
119
|
-
}
|
|
120
|
-
return {
|
|
121
|
-
projectId, endpoint: base,
|
|
122
|
-
topicName: (id: string) => resource("topics", id),
|
|
123
|
-
subscriptionName: (id: string) => resource("subscriptions", id),
|
|
124
|
-
createTopic(id: string) {
|
|
125
|
-
return request<{ name: string }>(path("topics", id), "PUT", {});
|
|
126
|
-
},
|
|
127
|
-
createSubscription(id: string, options: PubSubSubscriptionOptions) {
|
|
128
|
-
const ackDeadlineSeconds = subscriptionOptions(options);
|
|
129
|
-
return request<{ name: string; topic: string; ackDeadlineSeconds: number }>(path("subscriptions", id), "PUT", {
|
|
130
|
-
topic: resource("topics", options.topic), ackDeadlineSeconds,
|
|
131
|
-
...(options.pushConfig ? { pushConfig: options.pushConfig } : options.pushEndpoint ? { pushConfig: { pushEndpoint: options.pushEndpoint } } : {}),
|
|
132
|
-
...(options.deadLetterPolicy ? { deadLetterPolicy: options.deadLetterPolicy } : {}),
|
|
133
|
-
...(options.retryPolicy ? { retryPolicy: options.retryPolicy } : {}),
|
|
134
|
-
});
|
|
135
|
-
},
|
|
136
|
-
async publish(topic: string, data: string | Uint8Array, attributes?: Record<string, string>): Promise<string> {
|
|
137
|
-
const result = await request<{ messageIds: string[] }>(`${path("topics", topic)}:publish`, "POST", {
|
|
138
|
-
messages: [{ data: Buffer.from(data).toString("base64"), ...(attributes ? { attributes } : {}) }],
|
|
139
|
-
});
|
|
140
|
-
return result.messageIds[0];
|
|
141
|
-
},
|
|
142
|
-
/** A pull leases messages; call acknowledge() to remove them. Empty is valid.
|
|
143
|
-
* Uses the emulator's immediate pull mode for bounded test polling.
|
|
144
|
-
*/
|
|
145
|
-
async pull(subscription: string, options: { maxMessages?: number; timeoutMs?: number } = {}): Promise<PubSubReceivedMessage[]> {
|
|
146
|
-
const maxMessages = options.maxMessages ?? 10;
|
|
147
|
-
if (!Number.isInteger(maxMessages) || maxMessages < 1) throw new Error("pubsub(): maxMessages must be a positive integer");
|
|
148
|
-
const result = await request<{ receivedMessages?: PubSubReceivedMessage[] }>(`${path("subscriptions", subscription)}:pull`, "POST",
|
|
149
|
-
{ maxMessages, returnImmediately: true }, options.timeoutMs);
|
|
150
|
-
return result.receivedMessages ?? [];
|
|
151
|
-
},
|
|
152
|
-
async acknowledge(subscription: string, ackIds: string[]): Promise<void> {
|
|
153
|
-
await request(`${path("subscriptions", subscription)}:acknowledge`, "POST", { ackIds });
|
|
154
|
-
},
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
export type PubSubClient = ReturnType<typeof pubsubClient>;
|
|
159
|
-
export interface PubSubHelpers extends PubSubClient {
|
|
160
|
-
/** Direct plaintext address for clients explicitly using emulator mode. */
|
|
161
|
-
emulatorHost: string;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
// The official emulator ignores OIDC. Keep its push/retry/dead-letter engine,
|
|
165
|
-
// but route authenticated deliveries through a local adapter in its container.
|
|
166
|
-
function authenticatedPush(options: PubSubSubscriptionOptions, pushPort: number): PubSubSubscriptionOptions {
|
|
167
|
-
subscriptionOptions(options);
|
|
168
|
-
const push = options.pushConfig;
|
|
169
|
-
if (!push?.oidcToken) return options;
|
|
170
|
-
const config = Buffer.from(JSON.stringify({ endpoint: push.pushEndpoint, ...push.oidcToken })).toString("base64url");
|
|
171
|
-
return {
|
|
172
|
-
...options, pushEndpoint: undefined,
|
|
173
|
-
pushConfig: { ...push, pushEndpoint: `http://127.0.0.1:${pushPort}/push/${config}` },
|
|
174
|
-
};
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
function withPushAuth(client: PubSubClient, pushPort: number): PubSubClient {
|
|
178
|
-
return { ...client, createSubscription: (id, options) => client.createSubscription(id, authenticatedPush(options, pushPort)) };
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
/** Google Cloud's real local Pub/Sub emulator, ready for Google client libraries.
|
|
182
|
-
* Mount directly in services: { pubsub: pubsub({ topics: ["orders"] }) }.
|
|
183
|
-
* Applications use their default pubsub.googleapis.com:443 endpoint with
|
|
184
|
-
* verified TLS and local test credentials. No emulator endpoint override.
|
|
185
|
-
* State survives VM snapshots/forks, but not an emulator process restart.
|
|
186
|
-
*/
|
|
187
|
-
export function pubsub(opts: PubSubOptions = {}) {
|
|
188
|
-
const projectId = projectName(opts.projectId ?? "spectest");
|
|
189
|
-
const port = opts.port ?? 8085;
|
|
190
|
-
const pushPort = port === 8086 ? 8087 : 8086;
|
|
191
|
-
const hostname = opts.tls === false ? undefined : opts.tls?.hostname ?? "pubsub.googleapis.com";
|
|
192
|
-
if (hostname !== undefined) {
|
|
193
|
-
certificate([hostname]);
|
|
194
|
-
if (hostname.startsWith("*.")) throw new Error("pubsub(): TLS hostname must be an exact hostname");
|
|
195
|
-
}
|
|
196
|
-
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("pubsub(): port must be an integer between 1 and 65535");
|
|
197
|
-
const subscriptions = Object.entries(opts.subscriptions ?? {});
|
|
198
|
-
for (const [id, options] of subscriptions) { resourceId(id); subscriptionOptions(options); }
|
|
199
|
-
const topics = [...new Set([...(opts.topics ?? []), ...subscriptions.map(([, options]) => options.topic)])].map(resourceId);
|
|
200
|
-
if (hostname && port === 443) throw new Error("pubsub(): the plaintext emulator port must differ from TLS port 443");
|
|
201
|
-
return serviceGroup({
|
|
202
|
-
primary: "emulator",
|
|
203
|
-
services: (): ServicesMap => {
|
|
204
|
-
const emulator: ServiceDefinition = {
|
|
205
|
-
image: { type: "registry", reference: opts.image ?? "gcr.io/google.com/cloudsdktool/google-cloud-cli:573.0.0-emulators" },
|
|
206
|
-
args: ["sh", "-ec", `
|
|
207
|
-
java_home=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")
|
|
208
|
-
cp "$java_home/lib/security/cacerts" /tmp/spectest-pubsub-cacerts
|
|
209
|
-
keytool -importcert -noprompt -alias spectest-environment -keystore /tmp/spectest-pubsub-cacerts -storepass changeit -file "$NODE_EXTRA_CA_CERTS"
|
|
210
|
-
export JAVA_TOOL_OPTIONS="-Djavax.net.ssl.trustStore=/tmp/spectest-pubsub-cacerts -Djavax.net.ssl.trustStorePassword=changeit \${JAVA_TOOL_OPTIONS:-}"
|
|
211
|
-
python3 /opt/spectest-pubsub-push.py ${pushPort} &
|
|
212
|
-
exec gcloud beta emulators pubsub start --project=${projectId} --host-port=0.0.0.0:${port} --quiet
|
|
213
|
-
`],
|
|
214
|
-
files: [{ path: "/opt/spectest-pubsub-push.py", content: PUSH_ADAPTER }],
|
|
215
|
-
ports: [port],
|
|
216
|
-
...(hostname ? { tls: [{ hostname, port, http2: true }] } : {}),
|
|
217
|
-
readyCheck: { type: "http", port, path: `/v1/projects/${projectId}/topics`, timeoutSecs: 60 },
|
|
218
|
-
setup: async ({ name }) => {
|
|
219
|
-
const client = withPushAuth(pubsubClient(`http://${name}:${port}`, projectId), pushPort);
|
|
220
|
-
for (const topic of topics) await client.createTopic(topic);
|
|
221
|
-
for (const [id, options] of subscriptions) await client.createSubscription(id, options);
|
|
222
|
-
},
|
|
223
|
-
};
|
|
224
|
-
return { emulator };
|
|
225
|
-
},
|
|
226
|
-
helpers: (ctx: ServiceHelpersContext): PubSubHelpers => ({
|
|
227
|
-
...withPushAuth(pubsubClient(hostname ? `https://${hostname}` : `http://${ctx.name}:${port}`, projectId,
|
|
228
|
-
async (input, init) => readRaw(await ctx.fetch(String(input), init)) as unknown as Response), pushPort),
|
|
229
|
-
emulatorHost: `${ctx.name}:${port}`,
|
|
230
|
-
}),
|
|
231
|
-
setup: async (ctx: ServiceSetupContext<PubSubHelpers>) => {
|
|
232
|
-
// Prove the TLS endpoint and certificate are ready before dependents.
|
|
233
|
-
if (hostname) {
|
|
234
|
-
const response = await rawFetch(`https://${hostname}/v1/projects/${projectId}/topics`, { signal: AbortSignal.timeout(10000) });
|
|
235
|
-
if (!response.ok) throw new Error(`pubsub(): TLS readiness failed: HTTP ${response.status}`);
|
|
236
|
-
await response.text();
|
|
237
|
-
}
|
|
238
|
-
await opts.setup?.(ctx);
|
|
239
|
-
},
|
|
240
|
-
});
|
|
241
|
-
}
|