@caelo-cms/shared 0.11.9 → 0.11.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/certs/aws-rds-global.pem +2888 -0
- package/certs/cloudsql-global.pem +1897 -0
- package/dist/body-cap.d.ts +35 -0
- package/dist/body-cap.d.ts.map +1 -0
- package/dist/body-cap.js +63 -0
- package/dist/body-cap.js.map +1 -0
- package/dist/client-address.d.ts +69 -0
- package/dist/client-address.d.ts.map +1 -0
- package/dist/client-address.js +99 -0
- package/dist/client-address.js.map +1 -0
- package/dist/context.d.ts +8 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/database-tls.d.ts +64 -0
- package/dist/database-tls.d.ts.map +1 -0
- package/dist/database-tls.js +95 -0
- package/dist/database-tls.js.map +1 -0
- package/dist/database-url.d.ts +2 -15
- package/dist/database-url.d.ts.map +1 -1
- package/dist/database-url.js +14 -1
- package/dist/database-url.js.map +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/interim-forwarded-host.d.ts +15 -0
- package/dist/interim-forwarded-host.d.ts.map +1 -0
- package/dist/interim-forwarded-host.js +16 -0
- package/dist/interim-forwarded-host.js.map +1 -0
- package/dist/json-security-headers.d.ts +23 -0
- package/dist/json-security-headers.d.ts.map +1 -0
- package/dist/json-security-headers.js +53 -0
- package/dist/json-security-headers.js.map +1 -0
- package/dist/pool-budget.d.ts +10 -0
- package/dist/pool-budget.d.ts.map +1 -1
- package/dist/pool-budget.js +15 -0
- package/dist/pool-budget.js.map +1 -1
- package/dist/redirect-emit-gcp.d.ts +16 -8
- package/dist/redirect-emit-gcp.d.ts.map +1 -1
- package/dist/redirect-emit-gcp.js +22 -11
- package/dist/redirect-emit-gcp.js.map +1 -1
- package/dist/release-manifest.d.ts +18 -2
- package/dist/release-manifest.d.ts.map +1 -1
- package/dist/release-manifest.js +32 -4
- package/dist/release-manifest.js.map +1 -1
- package/dist/session-cookie.d.ts +88 -0
- package/dist/session-cookie.d.ts.map +1 -0
- package/dist/session-cookie.js +151 -0
- package/dist/session-cookie.js.map +1 -0
- package/dist/staging-plugin-stub.d.ts +28 -0
- package/dist/staging-plugin-stub.d.ts.map +1 -0
- package/dist/staging-plugin-stub.js +34 -0
- package/dist/staging-plugin-stub.js.map +1 -0
- package/dist/version.d.ts +2 -2
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +2 -1
- package/src/body-cap.test.ts +44 -0
- package/src/body-cap.ts +83 -0
- package/src/client-address.test.ts +90 -0
- package/src/client-address.ts +111 -0
- package/src/context.ts +8 -0
- package/src/database-tls.test.ts +83 -0
- package/src/database-tls.ts +99 -0
- package/src/database-url.test.ts +49 -11
- package/src/database-url.ts +21 -1
- package/src/index.ts +7 -0
- package/src/interim-forwarded-host.ts +16 -0
- package/src/json-security-headers.test.ts +31 -0
- package/src/json-security-headers.ts +53 -0
- package/src/pool-budget.test.ts +17 -3
- package/src/pool-budget.ts +19 -0
- package/src/redirect-emit-gcp.ts +24 -11
- package/src/redirect-emit.test.ts +6 -5
- package/src/release-manifest.ts +33 -4
- package/src/session-cookie.test.ts +96 -0
- package/src/session-cookie.ts +163 -0
- package/src/staging-plugin-stub.test.ts +23 -0
- package/src/staging-plugin-stub.ts +37 -0
- package/src/version.ts +1 -1
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
import { describe, expect, it } from "bun:test";
|
|
4
|
+
import { X509Certificate } from "node:crypto";
|
|
5
|
+
import { resolve } from "node:path";
|
|
6
|
+
import {
|
|
7
|
+
AWS_RDS_CA_BUNDLE,
|
|
8
|
+
CLOUD_SQL_CA_BUNDLE,
|
|
9
|
+
DATABASE_CA_DIR,
|
|
10
|
+
databaseTlsProblem,
|
|
11
|
+
isIpLiteral,
|
|
12
|
+
verifyFullQuery,
|
|
13
|
+
} from "./database-tls.js";
|
|
14
|
+
|
|
15
|
+
const CERTS = resolve(import.meta.dir, "../certs");
|
|
16
|
+
|
|
17
|
+
/** The certificates of a PEM bundle in the repo. */
|
|
18
|
+
async function bundle(name: string): Promise<X509Certificate[]> {
|
|
19
|
+
const pem = await Bun.file(resolve(CERTS, name)).text();
|
|
20
|
+
const blocks = pem.match(/-----BEGIN CERTIFICATE-----[\s\S]+?-----END CERTIFICATE-----/g) ?? [];
|
|
21
|
+
return blocks.map((b) => new X509Certificate(b));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe("shipped database CA bundles", () => {
|
|
25
|
+
// The images copy packages/shared/certs/ to DATABASE_CA_DIR; the constants
|
|
26
|
+
// name files there, so each must exist in the repo under the same name.
|
|
27
|
+
for (const path of [CLOUD_SQL_CA_BUNDLE, AWS_RDS_CA_BUNDLE]) {
|
|
28
|
+
const name = path.slice(DATABASE_CA_DIR.length + 1);
|
|
29
|
+
it(`${name} parses, holds a root, and no certificate expires within two years`, async () => {
|
|
30
|
+
const certs = await bundle(name);
|
|
31
|
+
expect(certs.length).toBeGreaterThan(0);
|
|
32
|
+
expect(certs.some((c) => c.subject === c.issuer)).toBe(true);
|
|
33
|
+
// A bundle that is about to expire breaks every connection of every
|
|
34
|
+
// install on that day: refresh it from the provider (see the file
|
|
35
|
+
// comment of database-tls.ts for the URLs) well before.
|
|
36
|
+
const horizon = Date.now() + 2 * 365 * 24 * 3600 * 1000;
|
|
37
|
+
for (const c of certs) expect(Date.parse(c.validTo)).toBeGreaterThan(horizon);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
it("holds no private key", async () => {
|
|
42
|
+
for (const name of ["cloudsql-global.pem", "aws-rds-global.pem"]) {
|
|
43
|
+
expect(await Bun.file(resolve(CERTS, name)).text()).not.toMatch(/PRIVATE KEY/);
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
describe("verifyFullQuery", () => {
|
|
49
|
+
it("names the CA bundle, or relies on the public roots without one", () => {
|
|
50
|
+
expect(verifyFullQuery(CLOUD_SQL_CA_BUNDLE)).toBe(
|
|
51
|
+
"sslmode=verify-full&sslrootcert=/etc/caelo/db-ca/cloudsql-global.pem",
|
|
52
|
+
);
|
|
53
|
+
expect(verifyFullQuery()).toBe("sslmode=verify-full");
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
describe("databaseTlsProblem", () => {
|
|
58
|
+
it("accepts verify-full to a remote host", () => {
|
|
59
|
+
expect(databaseTlsProblem(`postgres://a@db.example:5432/d?${verifyFullQuery()}`)).toBeNull();
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it("refuses every weaker mode to a remote host", () => {
|
|
63
|
+
for (const q of ["", "?sslmode=require", "?sslmode=verify-ca", "?sslmode=prefer"]) {
|
|
64
|
+
expect(databaseTlsProblem(`postgres://a@10.0.0.3:5432/d${q}`, "X_URL")).toMatch(
|
|
65
|
+
/^X_URL reaches 10\.0\.0\.3 /,
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("refuses a URL that does not parse", () => {
|
|
71
|
+
expect(databaseTlsProblem("not a url", "X_URL")).toBe("X_URL is not a valid URL");
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
describe("isIpLiteral", () => {
|
|
76
|
+
it("tells IP literals from host names", () => {
|
|
77
|
+
expect(isIpLiteral("10.20.0.3")).toBe(true);
|
|
78
|
+
expect(isIpLiteral("[fd00::3]")).toBe(true);
|
|
79
|
+
expect(isIpLiteral("fd00::3")).toBe(true);
|
|
80
|
+
expect(isIpLiteral("i1.p.europe-west1.sql-psa.goog")).toBe(false);
|
|
81
|
+
expect(isIpLiteral("postgres")).toBe(false);
|
|
82
|
+
});
|
|
83
|
+
});
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* TLS of every database connection (#1338, gateway hardening L6).
|
|
5
|
+
*
|
|
6
|
+
* Every connection to a database on another machine uses
|
|
7
|
+
* `sslmode=verify-full`: the server's certificate must chain to a CA the
|
|
8
|
+
* install trusts AND name the host the client dialled. `sslmode=require`
|
|
9
|
+
* (what every provider used before) encrypts but checks neither, so anyone
|
|
10
|
+
* who can answer on the database's address can read the passwords and the
|
|
11
|
+
* data. There is no weaker mode for a remote host and no automatic downgrade
|
|
12
|
+
* (CLAUDE.md §2, no fallbacks): a URL that asks for less is refused with
|
|
13
|
+
* {@link databaseTlsProblem} wherever a process reads one from its
|
|
14
|
+
* environment (`databaseUrlFromEnv`).
|
|
15
|
+
*
|
|
16
|
+
* Which CA each provider's server presents, and so which bundle the client
|
|
17
|
+
* passes as `sslrootcert`:
|
|
18
|
+
*
|
|
19
|
+
* - gcp / gcp-firebase: Cloud SQL with the shared CA
|
|
20
|
+
* (`GOOGLE_MANAGED_CAS_CA`); its server certificate names the instance's
|
|
21
|
+
* DNS name, not its IP, so clients dial that name (resolved by a private
|
|
22
|
+
* Cloud DNS zone in the install's VPC) and trust Google's published root
|
|
23
|
+
* + regional bundle ({@link CLOUD_SQL_CA_BUNDLE}).
|
|
24
|
+
* - aws: RDS, the endpoint host name, Amazon's RDS bundle
|
|
25
|
+
* ({@link AWS_RDS_CA_BUNDLE}).
|
|
26
|
+
* - azure: Flexible Server, the server's FQDN; its certificate chains to
|
|
27
|
+
* public roots (DigiCert Global Root G2 / Microsoft RSA Root CA 2017)
|
|
28
|
+
* that Bun trusts out of the box, so no `sslrootcert`.
|
|
29
|
+
* - self-hosted: the compose Postgres serves a certificate for the host
|
|
30
|
+
* name `postgres`, signed by a CA the installer generates once per
|
|
31
|
+
* install ({@link SELF_HOSTED_CA_FILE}).
|
|
32
|
+
*
|
|
33
|
+
* The bundles ship in every image at {@link DATABASE_CA_DIR}
|
|
34
|
+
* (packages/shared/certs/ in the repo), so a URL built on the operator's
|
|
35
|
+
* machine names a path that exists in the container.
|
|
36
|
+
*
|
|
37
|
+
* Pure module — no I/O.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
/** Where every Caelo image carries the database CA bundles (copied from packages/shared/certs/). */
|
|
41
|
+
export const DATABASE_CA_DIR = "/etc/caelo/db-ca";
|
|
42
|
+
|
|
43
|
+
/** Google's Cloud SQL shared-CA bundle (all regions), `cloudsql-ca-bundles/global.pem`. */
|
|
44
|
+
export const CLOUD_SQL_CA_BUNDLE = `${DATABASE_CA_DIR}/cloudsql-global.pem`;
|
|
45
|
+
|
|
46
|
+
/** Amazon's RDS bundle (all regions), `truststore.pki.rds.amazonaws.com/global/global-bundle.pem`. */
|
|
47
|
+
export const AWS_RDS_CA_BUNDLE = `${DATABASE_CA_DIR}/aws-rds-global.pem`;
|
|
48
|
+
|
|
49
|
+
/** The self-hosted install's own CA, mounted read-only into every container that connects. */
|
|
50
|
+
export const SELF_HOSTED_CA_FILE = `${DATABASE_CA_DIR}/self-hosted-ca.pem`;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The query string of a verify-full connection, with `caFile` as the trust
|
|
54
|
+
* anchor (omit it for a server whose chain ends in a public root).
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* verifyFullQuery(CLOUD_SQL_CA_BUNDLE)
|
|
58
|
+
* // → "sslmode=verify-full&sslrootcert=/etc/caelo/db-ca/cloudsql-global.pem"
|
|
59
|
+
*/
|
|
60
|
+
export function verifyFullQuery(caFile?: string): string {
|
|
61
|
+
return caFile ? `sslmode=verify-full&sslrootcert=${caFile}` : "sslmode=verify-full";
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Whether `host` is an IPv4 or IPv6 literal (with or without URL brackets). */
|
|
65
|
+
export function isIpLiteral(host: string): boolean {
|
|
66
|
+
const h = host.replace(/^\[|\]$/g, "");
|
|
67
|
+
return /^\d{1,3}(\.\d{1,3}){3}$/.test(h) || (h.includes(":") && /^[0-9a-f:.]+$/i.test(h));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Hosts on this machine: a connection to them never crosses a network, so
|
|
72
|
+
* dev, CI and the test suites connect without TLS.
|
|
73
|
+
*/
|
|
74
|
+
function isLoopbackHost(host: string): boolean {
|
|
75
|
+
const h = host.replace(/^\[|\]$/g, "").toLowerCase();
|
|
76
|
+
return h === "localhost" || h === "::1" || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(h);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Why `url` may not be used to reach a database, or null when it may: a
|
|
81
|
+
* remote host needs `sslmode=verify-full`. Loopback hosts and unix sockets
|
|
82
|
+
* (`?host=/path`) are exempt. The message names the variable and the fix.
|
|
83
|
+
*
|
|
84
|
+
* @param label how the URL is known to the operator, e.g. `ADMIN_DATABASE_URL`
|
|
85
|
+
*/
|
|
86
|
+
export function databaseTlsProblem(url: string, label = "database URL"): string | null {
|
|
87
|
+
let parsed: URL;
|
|
88
|
+
try {
|
|
89
|
+
parsed = new URL(url);
|
|
90
|
+
} catch {
|
|
91
|
+
return `${label} is not a valid URL`;
|
|
92
|
+
}
|
|
93
|
+
const socket = parsed.searchParams.get("host");
|
|
94
|
+
if (socket?.startsWith("/")) return null;
|
|
95
|
+
if (parsed.hostname === "" || isLoopbackHost(parsed.hostname)) return null;
|
|
96
|
+
const mode = parsed.searchParams.get("sslmode");
|
|
97
|
+
if (mode === "verify-full") return null;
|
|
98
|
+
return `${label} reaches ${parsed.hostname} with sslmode=${mode ?? "(none)"}; every connection to a database on another machine must use sslmode=verify-full, which checks the server's certificate and host name (#1338). Cloud and self-hosted installs: run \`cms-provision upgrade\` (it sets the URLs and the database's certificate up). Local development: connect to localhost.`;
|
|
99
|
+
}
|
package/src/database-url.test.ts
CHANGED
|
@@ -72,34 +72,35 @@ describe("databaseUrlFromEnv", () => {
|
|
|
72
72
|
|
|
73
73
|
it("composes the URL from the plain URL var + its secret-mounted password var (cloud)", () => {
|
|
74
74
|
const env = {
|
|
75
|
-
ADMIN_DATABASE_URL:
|
|
75
|
+
ADMIN_DATABASE_URL:
|
|
76
|
+
"postgresql://admin_role@i1.p.europe-west1.sql-psa.goog:5432/cms_admin?sslmode=verify-full&sslrootcert=/etc/caelo/db-ca/cloudsql-global.pem",
|
|
76
77
|
ADMIN_DATABASE_PASSWORD: "s3cret",
|
|
77
78
|
};
|
|
78
79
|
expect(databaseUrlFromEnv(["ADMIN_DATABASE_URL"], env)).toBe(
|
|
79
|
-
"postgresql://admin_role:s3cret@
|
|
80
|
+
"postgresql://admin_role:s3cret@i1.p.europe-west1.sql-psa.goog:5432/cms_admin?sslmode=verify-full&sslrootcert=/etc/caelo/db-ca/cloudsql-global.pem",
|
|
80
81
|
);
|
|
81
82
|
});
|
|
82
83
|
|
|
83
84
|
it("uses the password var that belongs to the URL var it picked", () => {
|
|
84
85
|
const env = {
|
|
85
|
-
PUBLIC_DATABASE_URL: "postgresql://public_role@h:5432/cms_public",
|
|
86
|
+
PUBLIC_DATABASE_URL: "postgresql://public_role@h:5432/cms_public?sslmode=verify-full",
|
|
86
87
|
PUBLIC_DATABASE_PASSWORD: "pub",
|
|
87
88
|
// Not consulted: it pairs with PUBLIC_ADMIN_DATABASE_URL, which is unset.
|
|
88
89
|
PUBLIC_ADMIN_DATABASE_PASSWORD: "adm",
|
|
89
90
|
};
|
|
90
91
|
expect(databaseUrlFromEnv(["PUBLIC_ADMIN_DATABASE_URL", "PUBLIC_DATABASE_URL"], env)).toBe(
|
|
91
|
-
"postgresql://public_role:pub@h:5432/cms_public",
|
|
92
|
+
"postgresql://public_role:pub@h:5432/cms_public?sslmode=verify-full",
|
|
92
93
|
);
|
|
93
94
|
});
|
|
94
95
|
|
|
95
96
|
it("prefers the first URL var that is set", () => {
|
|
96
97
|
const env = {
|
|
97
|
-
PUBLIC_ADMIN_DATABASE_URL: "postgresql://admin_role@h:5432/cms_public",
|
|
98
|
+
PUBLIC_ADMIN_DATABASE_URL: "postgresql://admin_role@h:5432/cms_public?sslmode=verify-full",
|
|
98
99
|
PUBLIC_ADMIN_DATABASE_PASSWORD: "adm",
|
|
99
|
-
PUBLIC_DATABASE_URL: "postgresql://public_role@h:5432/cms_public",
|
|
100
|
+
PUBLIC_DATABASE_URL: "postgresql://public_role@h:5432/cms_public?sslmode=verify-full",
|
|
100
101
|
};
|
|
101
102
|
expect(databaseUrlFromEnv(["PUBLIC_ADMIN_DATABASE_URL", "PUBLIC_DATABASE_URL"], env)).toBe(
|
|
102
|
-
"postgresql://admin_role:adm@h:5432/cms_public",
|
|
103
|
+
"postgresql://admin_role:adm@h:5432/cms_public?sslmode=verify-full",
|
|
103
104
|
);
|
|
104
105
|
});
|
|
105
106
|
|
|
@@ -111,7 +112,7 @@ describe("databaseUrlFromEnv", () => {
|
|
|
111
112
|
|
|
112
113
|
it("fails loudly when both the URL and the password var carry a password", () => {
|
|
113
114
|
const env = {
|
|
114
|
-
ADMIN_DATABASE_URL: "postgres://admin_role:inline@h/cms_admin",
|
|
115
|
+
ADMIN_DATABASE_URL: "postgres://admin_role:inline@h/cms_admin?sslmode=verify-full",
|
|
115
116
|
ADMIN_DATABASE_PASSWORD: "mounted",
|
|
116
117
|
};
|
|
117
118
|
expect(() => databaseUrlFromEnv(["ADMIN_DATABASE_URL"], env)).toThrow(
|
|
@@ -122,10 +123,47 @@ describe("databaseUrlFromEnv", () => {
|
|
|
122
123
|
|
|
123
124
|
describe("cloudSqlIamDatabaseUrls", () => {
|
|
124
125
|
it("logs in as the service account's IAM database user with the token as password", () => {
|
|
125
|
-
expect(
|
|
126
|
+
expect(
|
|
127
|
+
cloudSqlIamDatabaseUrls(
|
|
128
|
+
"i1.p.europe-west1.sql-psa.goog",
|
|
129
|
+
"job@p.iam.gserviceaccount.com",
|
|
130
|
+
"ya29/x",
|
|
131
|
+
),
|
|
132
|
+
).toEqual({
|
|
126
133
|
user: "job@p.iam",
|
|
127
|
-
admin:
|
|
128
|
-
|
|
134
|
+
admin:
|
|
135
|
+
"postgresql://job%40p.iam:ya29%2Fx@i1.p.europe-west1.sql-psa.goog:5432/cms_admin?sslmode=verify-full&sslrootcert=/etc/caelo/db-ca/cloudsql-global.pem",
|
|
136
|
+
public:
|
|
137
|
+
"postgresql://job%40p.iam:ya29%2Fx@i1.p.europe-west1.sql-psa.goog:5432/cms_public?sslmode=verify-full&sslrootcert=/etc/caelo/db-ca/cloudsql-global.pem",
|
|
129
138
|
});
|
|
130
139
|
});
|
|
140
|
+
|
|
141
|
+
it("refuses an IP host: the shared-CA certificate names only the instance's DNS name", () => {
|
|
142
|
+
expect(() => cloudSqlIamDatabaseUrls("10.0.0.3", "job@p.iam.gserviceaccount.com", "t")).toThrow(
|
|
143
|
+
/names only its DNS name.*cms-provision upgrade/,
|
|
144
|
+
);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
describe("databaseUrlFromEnv TLS (#1338)", () => {
|
|
149
|
+
it("refuses a remote host without verify-full and names the variable", () => {
|
|
150
|
+
for (const mode of ["", "?sslmode=require", "?sslmode=verify-ca", "?sslmode=disable"]) {
|
|
151
|
+
expect(() =>
|
|
152
|
+
databaseUrlFromEnv(["PUBLIC_DATABASE_URL"], {
|
|
153
|
+
PUBLIC_DATABASE_URL: `postgres://public_role:x@postgres:5432/cms_public${mode}`,
|
|
154
|
+
}),
|
|
155
|
+
).toThrow(/PUBLIC_DATABASE_URL reaches postgres .*verify-full/);
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("lets loopback hosts and unix sockets connect without TLS (dev, CI, tests)", () => {
|
|
160
|
+
for (const url of [
|
|
161
|
+
"postgres://a:x@localhost:5432/d",
|
|
162
|
+
"postgres://a:x@127.0.0.1:5541/d",
|
|
163
|
+
"postgres://a:x@[::1]:5432/d",
|
|
164
|
+
"postgres://a:x@sockethost/d?host=/var/run/postgresql",
|
|
165
|
+
]) {
|
|
166
|
+
expect(databaseUrlFromEnv(["U_URL"], { U_URL: url })).toBe(url);
|
|
167
|
+
}
|
|
168
|
+
});
|
|
131
169
|
});
|
package/src/database-url.ts
CHANGED
|
@@ -13,9 +13,20 @@
|
|
|
13
13
|
* password is a Secret Manager reference, so the password never shows up in
|
|
14
14
|
* the service's configuration or its revision history.
|
|
15
15
|
*
|
|
16
|
+
* A URL that reaches a database on another machine must ask for
|
|
17
|
+
* `sslmode=verify-full` (database-tls.ts, #1338); {@link databaseUrlFromEnv}
|
|
18
|
+
* refuses one that does not.
|
|
19
|
+
*
|
|
16
20
|
* Pure module — no I/O.
|
|
17
21
|
*/
|
|
18
22
|
|
|
23
|
+
import {
|
|
24
|
+
CLOUD_SQL_CA_BUNDLE,
|
|
25
|
+
databaseTlsProblem,
|
|
26
|
+
isIpLiteral,
|
|
27
|
+
verifyFullQuery,
|
|
28
|
+
} from "./database-tls.js";
|
|
29
|
+
|
|
19
30
|
/** The `<NAME>_PASSWORD` var that pairs with a `<NAME>_URL` var. */
|
|
20
31
|
export function databasePasswordVar(urlVar: string): string {
|
|
21
32
|
if (!urlVar.endsWith("_URL")) {
|
|
@@ -75,6 +86,8 @@ export function databaseUrlFromEnv(
|
|
|
75
86
|
for (const urlVar of urlVars) {
|
|
76
87
|
const url = env[urlVar];
|
|
77
88
|
if (!url) continue;
|
|
89
|
+
const tls = databaseTlsProblem(url, urlVar);
|
|
90
|
+
if (tls) throw new Error(tls);
|
|
78
91
|
const passwordVar = databasePasswordVar(urlVar);
|
|
79
92
|
const password = env[passwordVar];
|
|
80
93
|
return password ? withDatabasePassword(url, password, `${urlVar} (with ${passwordVar})`) : url;
|
|
@@ -87,6 +100,8 @@ export function databaseUrlFromEnv(
|
|
|
87
100
|
* how the in-cloud jobs (the operator-access sync job, the upgrader job)
|
|
88
101
|
* connect: the user is the service account email without
|
|
89
102
|
* `.gserviceaccount.com`, the password a short-lived OAuth access token.
|
|
103
|
+
* TLS is verify-full against Google's Cloud SQL shared-CA bundle, so `host`
|
|
104
|
+
* must be the instance's DNS name (its certificate names no IP).
|
|
90
105
|
*
|
|
91
106
|
* @example
|
|
92
107
|
* cloudSqlIamDatabaseUrls("10.0.0.3", "job@p.iam.gserviceaccount.com", "ya29.x").user
|
|
@@ -97,8 +112,13 @@ export function cloudSqlIamDatabaseUrls(
|
|
|
97
112
|
serviceAccountEmail: string,
|
|
98
113
|
accessToken: string,
|
|
99
114
|
): { admin: string; public: string; user: string } {
|
|
115
|
+
if (isIpLiteral(host)) {
|
|
116
|
+
throw new Error(
|
|
117
|
+
`the Cloud SQL host is the IP address ${host}, but the instance's certificate names only its DNS name, so a verify-full connection to it cannot succeed (#1338). Run the converge (\`cms-provision upgrade\`, or Install update for a cloud-setup release) — it moves the services onto the DNS name — then retry.`,
|
|
118
|
+
);
|
|
119
|
+
}
|
|
100
120
|
const user = serviceAccountEmail.replace(/\.gserviceaccount\.com$/, "");
|
|
101
121
|
const url = (db: string) =>
|
|
102
|
-
`postgresql://${encodeURIComponent(user)}:${encodeURIComponent(accessToken)}@${host}:5432/${db}
|
|
122
|
+
`postgresql://${encodeURIComponent(user)}:${encodeURIComponent(accessToken)}@${host}:5432/${db}?${verifyFullQuery(CLOUD_SQL_CA_BUNDLE)}`;
|
|
103
123
|
return { admin: url("cms_admin"), public: url("cms_public"), user };
|
|
104
124
|
}
|
package/src/index.ts
CHANGED
|
@@ -6,10 +6,12 @@ export * from "./ai-tools.js";
|
|
|
6
6
|
export * from "./auth-forms.js";
|
|
7
7
|
export * from "./auto-update.js";
|
|
8
8
|
export * from "./base-css.js";
|
|
9
|
+
export * from "./body-cap.js";
|
|
9
10
|
export * from "./bug-intake.js";
|
|
10
11
|
export * from "./build-page.js";
|
|
11
12
|
export * from "./cap-failures.js";
|
|
12
13
|
export * from "./change-journal.js";
|
|
14
|
+
export * from "./client-address.js";
|
|
13
15
|
export * from "./cloud-shell-install.js";
|
|
14
16
|
export * from "./content.js";
|
|
15
17
|
export * from "./context.js";
|
|
@@ -17,6 +19,7 @@ export * from "./css-gradient-scan.js";
|
|
|
17
19
|
export * from "./css-scope-scan.js";
|
|
18
20
|
export * from "./css-var-aliases.js";
|
|
19
21
|
export * from "./css-var-scan.js";
|
|
22
|
+
export * from "./database-tls.js";
|
|
20
23
|
export * from "./database-url.js";
|
|
21
24
|
export * from "./design-draft-shell.js";
|
|
22
25
|
export * from "./document-language.js";
|
|
@@ -33,6 +36,8 @@ export * from "./google-oauth.js";
|
|
|
33
36
|
export * from "./id-safe-truncate.js";
|
|
34
37
|
export * from "./image-capabilities.js";
|
|
35
38
|
export * from "./interactions.js";
|
|
39
|
+
export * from "./interim-forwarded-host.js";
|
|
40
|
+
export * from "./json-security-headers.js";
|
|
36
41
|
export * from "./link-canonicalise.js";
|
|
37
42
|
export * from "./link-refs.js";
|
|
38
43
|
export * from "./lists/index.js";
|
|
@@ -76,11 +81,13 @@ export * from "./run-batched.js";
|
|
|
76
81
|
export * from "./safe-keys.js";
|
|
77
82
|
export * from "./screenshot-capture.js";
|
|
78
83
|
export * from "./seo.js";
|
|
84
|
+
export * from "./session-cookie.js";
|
|
79
85
|
export * from "./skills.js";
|
|
80
86
|
export * from "./snapshots.js";
|
|
81
87
|
export * from "./source-text-transport.js";
|
|
82
88
|
export * from "./stage-status.js";
|
|
83
89
|
export * from "./staging-gate.js";
|
|
90
|
+
export * from "./staging-plugin-stub.js";
|
|
84
91
|
export * from "./static-cache-policy.js";
|
|
85
92
|
export * from "./static-security-headers.js";
|
|
86
93
|
export * from "./strip-cdata.js";
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* FIXME(issue #1331): the header the MCP-Auth-Proxy's interim forwarder
|
|
5
|
+
* (apps/mcp-auth-proxy server.ts) sets to the host a client asked for, and
|
|
6
|
+
* the visitor gateway's staging gate (apps/api-gateway staging-proxy.ts)
|
|
7
|
+
* reads. One name for both ends of the contract.
|
|
8
|
+
*
|
|
9
|
+
* On gcp-firebase the Cloud Run service `gateway` became the MCP-Auth-Proxy
|
|
10
|
+
* (#1328, E13 a), while an existing staging domain mapping (#650) and the
|
|
11
|
+
* live release's Firebase `/api/**` rewrite still point at it. The proxy
|
|
12
|
+
* streams that traffic to the data gateway's run.app URL; Cloud Run routes
|
|
13
|
+
* by Host, so the original host travels in this header. Removed with the
|
|
14
|
+
* staging cutover (P3b-3).
|
|
15
|
+
*/
|
|
16
|
+
export const INTERIM_FORWARDED_HOST_HEADER = "x-caelo-forwarded-host";
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
import { describe, expect, it } from "bun:test";
|
|
4
|
+
import { JSON_SECURITY_HEADERS, withJsonSecurityHeaders } from "./json-security-headers.js";
|
|
5
|
+
|
|
6
|
+
describe("withJsonSecurityHeaders", () => {
|
|
7
|
+
it("adds every header to a JSON answer and keeps one already set", () => {
|
|
8
|
+
const res = withJsonSecurityHeaders(
|
|
9
|
+
new Response("{}", {
|
|
10
|
+
headers: { "content-type": "application/json; charset=utf-8", "referrer-policy": "origin" },
|
|
11
|
+
}),
|
|
12
|
+
);
|
|
13
|
+
expect(res.headers.get("referrer-policy")).toBe("origin");
|
|
14
|
+
expect(res.headers.get("x-frame-options")).toBe(JSON_SECURITY_HEADERS["X-Frame-Options"] ?? "");
|
|
15
|
+
expect(res.headers.get("x-content-type-options")).toBe("nosniff");
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it("leaves a non-JSON answer as it is", () => {
|
|
19
|
+
const res = withJsonSecurityHeaders(
|
|
20
|
+
new Response("data: x\n\n", { headers: { "content-type": "text/event-stream" } }),
|
|
21
|
+
);
|
|
22
|
+
expect(res.headers.get("x-frame-options")).toBeNull();
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it("rebuilds an answer with immutable headers around the same body", async () => {
|
|
26
|
+
const upstream = Response.json({ ok: true });
|
|
27
|
+
const res = withJsonSecurityHeaders(upstream);
|
|
28
|
+
expect(res.headers.get("content-security-policy")).toContain("default-src 'none'");
|
|
29
|
+
expect(await res.json()).toEqual({ ok: true });
|
|
30
|
+
});
|
|
31
|
+
});
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Security headers of a public service's JSON answers (#1326, #763 L2): the
|
|
5
|
+
* visitor gateway's and the MCP-Auth-Proxy's (#1328).
|
|
6
|
+
*
|
|
7
|
+
* A JSON answer is data, never a document: it must not be sniffed into
|
|
8
|
+
* HTML or script, framed, or handed to another origin's page. The headers
|
|
9
|
+
* say so to the browser on every JSON response, whichever route produced
|
|
10
|
+
* it. Pure Web APIs, so both services share the one definition.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** The headers every JSON response carries. */
|
|
14
|
+
export const JSON_SECURITY_HEADERS: Readonly<Record<string, string>> = {
|
|
15
|
+
"X-Content-Type-Options": "nosniff",
|
|
16
|
+
// Nothing in a JSON answer may load or run anything, nor be framed.
|
|
17
|
+
"Content-Security-Policy": "default-src 'none'; frame-ancestors 'none'",
|
|
18
|
+
"X-Frame-Options": "DENY",
|
|
19
|
+
"Referrer-Policy": "no-referrer",
|
|
20
|
+
"Cross-Origin-Resource-Policy": "same-origin",
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/** Whether `res` is a JSON answer (any `application/json` or `+json` type). */
|
|
24
|
+
export function isJsonResponse(res: Response): boolean {
|
|
25
|
+
const type = (res.headers.get("content-type") ?? "").split(";")[0]?.trim().toLowerCase() ?? "";
|
|
26
|
+
return type === "application/json" || type.endsWith("+json");
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Set the {@link JSON_SECURITY_HEADERS} on `headers`, keeping a value already set. */
|
|
30
|
+
export function addJsonSecurityHeaders(headers: Headers): void {
|
|
31
|
+
for (const [name, value] of Object.entries(JSON_SECURITY_HEADERS)) {
|
|
32
|
+
if (!headers.has(name)) headers.set(name, value);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* `res` with the JSON security headers when it is JSON, else `res` as it
|
|
38
|
+
* is. A response whose headers are immutable (one passed through from a
|
|
39
|
+
* `fetch`) is rebuilt around the same body stream, so a streamed answer
|
|
40
|
+
* stays streamed.
|
|
41
|
+
*/
|
|
42
|
+
export function withJsonSecurityHeaders(res: Response): Response {
|
|
43
|
+
if (!isJsonResponse(res)) return res;
|
|
44
|
+
if (Object.keys(JSON_SECURITY_HEADERS).every((h) => res.headers.has(h))) return res;
|
|
45
|
+
try {
|
|
46
|
+
addJsonSecurityHeaders(res.headers);
|
|
47
|
+
return res;
|
|
48
|
+
} catch {
|
|
49
|
+
const headers = new Headers(res.headers);
|
|
50
|
+
addJsonSecurityHeaders(headers);
|
|
51
|
+
return new Response(res.body, { status: res.status, statusText: res.statusText, headers });
|
|
52
|
+
}
|
|
53
|
+
}
|
package/src/pool-budget.test.ts
CHANGED
|
@@ -10,8 +10,10 @@ import {
|
|
|
10
10
|
DB_POOL_MAX_ENV,
|
|
11
11
|
DB_PUBLIC_POOL_MAX_ENV,
|
|
12
12
|
GATEWAY_POOL_MAX,
|
|
13
|
+
GATEWAY_ROLE_CONNECTION_LIMIT,
|
|
13
14
|
GENERATOR_POOL_MAX,
|
|
14
15
|
LISTEN_CONNECTIONS_PER_ADMIN_INSTANCE,
|
|
16
|
+
LISTEN_CONNECTIONS_PER_GATEWAY_INSTANCE,
|
|
15
17
|
type PoolBudgetInput,
|
|
16
18
|
poolBudgetEnv,
|
|
17
19
|
} from "./pool-budget.js";
|
|
@@ -67,15 +69,27 @@ describe("computePoolBudget", () => {
|
|
|
67
69
|
});
|
|
68
70
|
|
|
69
71
|
it("refuses a budget below the admin floor, naming the connections needed", () => {
|
|
70
|
-
//
|
|
71
|
-
expect(() => computePoolBudget({ ...GCP_DEFAULT,
|
|
72
|
-
/Raise maxConnections to at least
|
|
72
|
+
// Ten gateway instances on 60 slots: 30 for the gateway leave too little.
|
|
73
|
+
expect(() => computePoolBudget({ ...GCP_DEFAULT, maxConnections: 60 })).toThrow(
|
|
74
|
+
/Raise maxConnections to at least 94/,
|
|
73
75
|
);
|
|
74
76
|
expect(() => computePoolBudget({ ...GCP_DEFAULT, maxConnections: 50 })).toThrow(
|
|
75
77
|
new RegExp(`minimum of ${ADMIN_POOL_FLOOR}`),
|
|
76
78
|
);
|
|
77
79
|
});
|
|
78
80
|
|
|
81
|
+
it("refuses a gateway scale-out beyond public_role's CONNECTION LIMIT (#1338)", () => {
|
|
82
|
+
expect(GATEWAY_ROLE_CONNECTION_LIMIT).toBe(
|
|
83
|
+
10 * (GATEWAY_POOL_MAX + LISTEN_CONNECTIONS_PER_GATEWAY_INSTANCE),
|
|
84
|
+
);
|
|
85
|
+
expect(() =>
|
|
86
|
+
computePoolBudget({ ...GCP_DEFAULT, maxConnections: 1000, gatewayMaxInstances: 11 }),
|
|
87
|
+
).toThrow(
|
|
88
|
+
"11 gateway instances need 33 connections, but the database lets the gateway's login (public_role) hold at most 30",
|
|
89
|
+
);
|
|
90
|
+
expect(() => computePoolBudget({ ...GCP_DEFAULT, gatewayMaxInstances: 10 })).not.toThrow();
|
|
91
|
+
});
|
|
92
|
+
|
|
79
93
|
it("accepts exactly the floor", () => {
|
|
80
94
|
// reserves 74 + 5 × 4 = 94.
|
|
81
95
|
expect(computePoolBudget({ ...GCP_DEFAULT, maxConnections: 94 }).adminPoolMax).toBe(4);
|
package/src/pool-budget.ts
CHANGED
|
@@ -78,6 +78,17 @@ export const LISTEN_CONNECTIONS_PER_ADMIN_INSTANCE = 1;
|
|
|
78
78
|
*/
|
|
79
79
|
export const LISTEN_CONNECTIONS_PER_GATEWAY_INSTANCE = 1;
|
|
80
80
|
|
|
81
|
+
/**
|
|
82
|
+
* #1338 — public_role's `CONNECTION LIMIT` (cms_admin migration 0500,
|
|
83
|
+
* bootstrap.sh; the role audit's `PUBLIC_ROLE_LIMITS` pins the same value):
|
|
84
|
+
* the gateway's share at ten instances, `10 × (GATEWAY_POOL_MAX +
|
|
85
|
+
* LISTEN_CONNECTIONS_PER_GATEWAY_INSTANCE)`. The database refuses the
|
|
86
|
+
* gateway's connections beyond it, so {@link computePoolBudget} refuses a
|
|
87
|
+
* scale-out that would need more instead of letting instances fail to
|
|
88
|
+
* connect under load.
|
|
89
|
+
*/
|
|
90
|
+
export const GATEWAY_ROLE_CONNECTION_LIMIT = 30;
|
|
91
|
+
|
|
81
92
|
/** What the budget is computed from: the server and the scale-out limits. */
|
|
82
93
|
export interface PoolBudgetInput {
|
|
83
94
|
/** The server's `max_connections`, as provisioning sets it. */
|
|
@@ -173,6 +184,14 @@ export function computePoolBudget(input: PoolBudgetInput): PoolBudget {
|
|
|
173
184
|
positiveInteger(input.maintenanceReserve, "maintenanceReserve", 0);
|
|
174
185
|
positiveInteger(input.migrationsReserve, "migrationsReserve", 0);
|
|
175
186
|
positiveInteger(input.jobsReserve, "jobsReserve", 0);
|
|
187
|
+
const gatewayConnections =
|
|
188
|
+
input.gatewayMaxInstances * (GATEWAY_POOL_MAX + LISTEN_CONNECTIONS_PER_GATEWAY_INSTANCE);
|
|
189
|
+
if (gatewayConnections > GATEWAY_ROLE_CONNECTION_LIMIT) {
|
|
190
|
+
throw new Error(
|
|
191
|
+
`pool budget: ${input.gatewayMaxInstances} gateway instances need ${gatewayConnections} connections, but the database lets the gateway's login (public_role) hold at most ${GATEWAY_ROLE_CONNECTION_LIMIT} (CONNECTION LIMIT, #1338). ` +
|
|
192
|
+
`Lower gatewayMaxInstances to at most ${Math.floor(GATEWAY_ROLE_CONNECTION_LIMIT / (GATEWAY_POOL_MAX + LISTEN_CONNECTIONS_PER_GATEWAY_INSTANCE))}.`,
|
|
193
|
+
);
|
|
194
|
+
}
|
|
176
195
|
|
|
177
196
|
const reserves: PoolBudgetReserve[] = [
|
|
178
197
|
{ name: "maintenance", connections: input.maintenanceReserve },
|
package/src/redirect-emit-gcp.ts
CHANGED
|
@@ -161,19 +161,32 @@ export function assertGcpRedirectBudget(
|
|
|
161
161
|
withGcpRedirects({ name: "budget-check", pathRules: baseRules }, plan);
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
+
/** The paths the `gcp` load balancer sends to the visitor gateway: its whole API. */
|
|
165
|
+
export const GCP_GATEWAY_PATHS: readonly string[] = ["/api/*"];
|
|
166
|
+
|
|
164
167
|
/**
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
* `openid-configuration` prefix: the broker
|
|
173
|
-
* serves no such document (#899), so the site
|
|
168
|
+
* #1328 — the paths the `gcp` load balancer sends to the MCP-Auth-Proxy, the
|
|
169
|
+
* remote-MCP broker's own service (#806 put it inside the gateway). The
|
|
170
|
+
* endpoints and the broker's OAuth routes under `/api/mcp/*` — the more
|
|
171
|
+
* specific prefix wins over `/api/*` in one path matcher — and the two
|
|
172
|
+
* `/.well-known/oauth-*` prefixes, the broker's discovery documents, which
|
|
173
|
+
* MCP clients look up at the host root (RFC 9728 protected-resource
|
|
174
|
+
* metadata, RFC 8414 authorization-server metadata); they must reach the
|
|
175
|
+
* proxy, not 404 in the bucket. No `openid-configuration` prefix: the broker
|
|
176
|
+
* is not an OpenID provider and serves no such document (#899), so the site
|
|
177
|
+
* keeps that path.
|
|
174
178
|
*/
|
|
175
|
-
export const
|
|
176
|
-
"/api/*",
|
|
179
|
+
export const GCP_MCP_AUTH_PROXY_PATHS: readonly string[] = [
|
|
180
|
+
"/api/mcp/*",
|
|
177
181
|
"/.well-known/oauth-protected-resource/*",
|
|
178
182
|
"/.well-known/oauth-authorization-server/*",
|
|
179
183
|
];
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* The site's own routing in the `public` path matcher (stack + budget): the
|
|
187
|
+
* paths the load balancer sends to a service instead of the static bucket.
|
|
188
|
+
*/
|
|
189
|
+
export const GCP_PUBLIC_BASE_PATHS: readonly string[] = [
|
|
190
|
+
...GCP_GATEWAY_PATHS,
|
|
191
|
+
...GCP_MCP_AUTH_PROXY_PATHS,
|
|
192
|
+
];
|
|
@@ -153,8 +153,9 @@ describe("GCP URL-map rules", () => {
|
|
|
153
153
|
});
|
|
154
154
|
|
|
155
155
|
it("refuses a plan over the 1,000-predicate quota with the counts, before any change", () => {
|
|
156
|
-
// The base
|
|
157
|
-
|
|
156
|
+
// The base rules route /api/* to the gateway and /api/mcp/* plus the
|
|
157
|
+
// remote-MCP discovery paths to the MCP-Auth-Proxy (#806, #1328).
|
|
158
|
+
expect(GCP_PUBLIC_BASE_PATHS).toHaveLength(4);
|
|
158
159
|
const base = [{ paths: [...GCP_PUBLIC_BASE_PATHS] }];
|
|
159
160
|
const rows = (n: number) => Array.from({ length: n }, (_, i) => row(`/p${i}`, "/x"));
|
|
160
161
|
expect(() => assertGcpRedirectBudget(planOf(rows(498)), base)).not.toThrow();
|
|
@@ -163,12 +164,12 @@ describe("GCP URL-map rules", () => {
|
|
|
163
164
|
throw new Error("expected GcpUrlMapBudgetError");
|
|
164
165
|
} catch (e) {
|
|
165
166
|
expect(e).toBeInstanceOf(GcpUrlMapBudgetError);
|
|
166
|
-
expect((e as GcpUrlMapBudgetError).predicatesNeeded).toBe(
|
|
167
|
-
expect((e as Error).message).toContain("499 redirect(s) need
|
|
167
|
+
expect((e as GcpUrlMapBudgetError).predicatesNeeded).toBe(1002);
|
|
168
|
+
expect((e as Error).message).toContain("499 redirect(s) need 1002 paths");
|
|
168
169
|
expect((e as Error).message).toContain("bulk_delete_redirects");
|
|
169
170
|
}
|
|
170
171
|
// File-like paths cost one predicate each.
|
|
171
|
-
const files = Array.from({ length:
|
|
172
|
+
const files = Array.from({ length: 996 }, (_, i) => row(`/p${i}.html`, "/x"));
|
|
172
173
|
expect(() => assertGcpRedirectBudget(planOf(files), base)).not.toThrow();
|
|
173
174
|
});
|
|
174
175
|
|