@sanoma-ai/workflows 0.11.12 → 0.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLOUD-IDENTITY.md +12 -4
- package/KMS-SECRETS.md +3 -1
- package/README.md +4 -2
- package/dist/bundled.d.ts +2 -2
- package/dist/chats.d.ts +3 -3
- package/dist/chats.js +34 -20
- package/dist/chats.js.map +1 -1
- package/dist/client.d.ts +17 -4
- package/dist/client.js +74 -63
- package/dist/client.js.map +1 -1
- package/dist/cloud-identity.js +14 -12
- package/dist/cloud-identity.js.map +1 -1
- package/dist/collection.d.ts +2 -2
- package/dist/collection.js +53 -42
- package/dist/collection.js.map +1 -1
- package/dist/config.d.ts +11 -6
- package/dist/config.js +15 -13
- package/dist/config.js.map +1 -1
- package/dist/connection.d.ts +2 -0
- package/dist/connection.js +112 -40
- package/dist/connection.js.map +1 -1
- package/dist/credentials.d.ts +1 -1
- package/dist/credentials.js +33 -29
- package/dist/credentials.js.map +1 -1
- package/dist/db/company.d.ts +1152 -0
- package/dist/db/company.js +118 -0
- package/dist/db/company.js.map +1 -0
- package/dist/db/dbos.d.ts +111 -0
- package/dist/db/dbos.js +15 -0
- package/dist/db/dbos.js.map +1 -0
- package/dist/db/identity.d.ts +367 -0
- package/dist/db/identity.js +48 -0
- package/dist/db/identity.js.map +1 -0
- package/dist/db/index.d.ts +6 -0
- package/dist/db/index.js +10 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/kms.d.ts +224 -0
- package/dist/db/kms.js +29 -0
- package/dist/db/kms.js.map +1 -0
- package/dist/db/migrate.d.ts +63 -0
- package/dist/db/migrate.js +41 -0
- package/dist/db/migrate.js.map +1 -0
- package/dist/db/wakeup.d.ts +271 -0
- package/dist/db/wakeup.js +34 -0
- package/dist/db/wakeup.js.map +1 -0
- package/dist/db.d.ts +42 -39
- package/dist/db.js +129 -169
- package/dist/db.js.map +1 -1
- package/dist/deadlines.js +8 -4
- package/dist/deadlines.js.map +1 -1
- package/dist/deployment-key.d.ts +4 -1
- package/dist/deployment-key.js +10 -3
- package/dist/deployment-key.js.map +1 -1
- package/dist/host-derive.d.ts +12 -3
- package/dist/host-derive.js +21 -8
- package/dist/host-derive.js.map +1 -1
- package/dist/identity-provisioning.d.ts +2 -2
- package/dist/identity-provisioning.js +75 -42
- package/dist/identity-provisioning.js.map +1 -1
- package/dist/identity-service.js +18 -15
- package/dist/identity-service.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/kms-secrets.d.ts +9 -0
- package/dist/kms-secrets.js +59 -27
- package/dist/kms-secrets.js.map +1 -1
- package/dist/ledger.d.ts +1 -1
- package/dist/ledger.js +22 -14
- package/dist/ledger.js.map +1 -1
- package/dist/op.d.ts +27 -6
- package/dist/op.js.map +1 -1
- package/dist/run-status.js +7 -3
- package/dist/run-status.js.map +1 -1
- package/dist/schedules.d.ts +1 -1
- package/dist/schedules.js +5 -7
- package/dist/schedules.js.map +1 -1
- package/dist/schema.d.ts +6 -5
- package/dist/schema.js +11 -7
- package/dist/schema.js.map +1 -1
- package/dist/shared.d.ts +6 -23
- package/dist/shared.js +6 -5
- package/dist/shared.js.map +1 -1
- package/dist/storage-provisioning.d.ts +2 -0
- package/dist/storage-provisioning.js +51 -50
- package/dist/storage-provisioning.js.map +1 -1
- package/dist/tokens.d.ts +2 -2
- package/dist/tokens.js +31 -20
- package/dist/tokens.js.map +1 -1
- package/dist/version.d.ts +4 -4
- package/dist/version.js +5 -5
- package/dist/version.js.map +1 -1
- package/dist/wakeup-service.d.ts +0 -4
- package/dist/wakeup-service.js +102 -93
- package/dist/wakeup-service.js.map +1 -1
- package/dist/worker.d.ts +7 -0
- package/dist/worker.js +26 -21
- package/dist/worker.js.map +1 -1
- package/drizzle/company/0000_tables.sql +104 -0
- package/drizzle/company/0001_wakeup_row.sql +2 -0
- package/drizzle/company/meta/0000_snapshot.json +652 -0
- package/drizzle/company/meta/0001_snapshot.json +652 -0
- package/drizzle/company/meta/_journal.json +20 -0
- package/drizzle/wakeup/0000_tables.sql +26 -0
- package/drizzle/wakeup/meta/0000_snapshot.json +200 -0
- package/drizzle/wakeup/meta/_journal.json +13 -0
- package/oxlint.json +2 -1
- package/package.json +11 -2
- package/src/bundled.ts +2 -2
- package/src/chats.ts +41 -42
- package/src/client.ts +81 -64
- package/src/cloud-identity.ts +23 -24
- package/src/collection.ts +58 -50
- package/src/config.ts +29 -19
- package/src/connection.ts +132 -102
- package/src/credentials.ts +37 -33
- package/src/db/company.ts +173 -0
- package/src/db/dbos.ts +17 -0
- package/src/db/identity.ts +61 -0
- package/src/db/index.ts +10 -0
- package/src/db/kms.ts +41 -0
- package/src/db/migrate.ts +70 -0
- package/src/db/wakeup.ts +41 -0
- package/src/db.ts +151 -182
- package/src/deadlines.ts +10 -4
- package/src/deployment-key.ts +11 -3
- package/src/host-derive.ts +26 -9
- package/src/identity-provisioning.ts +85 -57
- package/src/identity-service.ts +19 -19
- package/src/index.ts +3 -0
- package/src/kms-secrets.ts +64 -56
- package/src/ledger.ts +23 -18
- package/src/op.ts +28 -6
- package/src/run-status.ts +7 -6
- package/src/schedules.ts +6 -8
- package/src/schema.ts +11 -8
- package/src/shared.ts +6 -17
- package/src/storage-provisioning.ts +47 -52
- package/src/tokens.ts +43 -33
- package/src/version.ts +5 -5
- package/src/wakeup-service.ts +117 -131
- package/src/worker.ts +38 -21
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { and, desc, eq, sql } from "drizzle-orm";
|
|
2
|
+
import { Pool } from "pg";
|
|
2
3
|
import type { IdentityEnrollment } from "./cloud-identity.ts";
|
|
4
|
+
import { type Db, databaseOn, pgCode } from "./db.ts";
|
|
5
|
+
import { identityCredentials, identityDeployments } from "./db/identity.ts";
|
|
3
6
|
|
|
4
7
|
export type DeploymentIdentity = Omit<IdentityEnrollment, "generation">;
|
|
5
8
|
export interface EnrollIdentity extends IdentityEnrollment {
|
|
@@ -44,17 +47,18 @@ export function configuredIdentityProvisioning(
|
|
|
44
47
|
throw new IdentityProvisioningError("invalid");
|
|
45
48
|
}
|
|
46
49
|
const connectionString = url.href;
|
|
47
|
-
let
|
|
50
|
+
let db: (Db & { $client: Pool }) | undefined;
|
|
48
51
|
let closed = false;
|
|
49
52
|
let closing: Promise<void> | undefined;
|
|
50
53
|
const store = postgresIdentityProvisioning(async () => {
|
|
51
54
|
if (closed) throw new IdentityProvisioningError("unavailable");
|
|
52
|
-
if (!
|
|
53
|
-
pool = new Pool({ connectionString, max: 5, connectionTimeoutMillis: 10_000, query_timeout: 15_000 });
|
|
55
|
+
if (!db) {
|
|
56
|
+
const pool = new Pool({ connectionString, max: 5, connectionTimeoutMillis: 10_000, query_timeout: 15_000 });
|
|
54
57
|
pool.on("error", () => {});
|
|
58
|
+
db = databaseOn(pool);
|
|
55
59
|
}
|
|
56
60
|
// The control plane needs enrollment writes, never schema or database administration.
|
|
57
|
-
const { rows } = await
|
|
61
|
+
const { rows } = await db.execute<{ allowed: boolean }>(sql`SELECT
|
|
58
62
|
NOT EXISTS (SELECT FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreatedb OR rolcreaterole OR rolbypassrls))
|
|
59
63
|
AND has_schema_privilege('sanoma_identity','USAGE') AND NOT has_schema_privilege('sanoma_identity','CREATE')
|
|
60
64
|
AND has_table_privilege('sanoma_identity.deployments','SELECT')
|
|
@@ -66,13 +70,13 @@ export function configuredIdentityProvisioning(
|
|
|
66
70
|
AND NOT has_table_privilege('sanoma_identity.deployments','DELETE,TRUNCATE,REFERENCES,TRIGGER')
|
|
67
71
|
AND NOT has_table_privilege('sanoma_identity.credentials','DELETE,TRUNCATE,REFERENCES,TRIGGER') AS allowed`);
|
|
68
72
|
if (!rows[0]?.allowed) fail("invalid");
|
|
69
|
-
return
|
|
73
|
+
return db;
|
|
70
74
|
}, config);
|
|
71
75
|
return {
|
|
72
76
|
store,
|
|
73
77
|
close() {
|
|
74
78
|
closed = true;
|
|
75
|
-
return (closing ??=
|
|
79
|
+
return (closing ??= db?.$client.end() ?? Promise.resolve());
|
|
76
80
|
},
|
|
77
81
|
};
|
|
78
82
|
}
|
|
@@ -130,7 +134,7 @@ const deadlineOf = (value: string) => {
|
|
|
130
134
|
|
|
131
135
|
/** Caller owns a dedicated writer pool; never give this pool to the broker or company code. */
|
|
132
136
|
export function postgresIdentityProvisioning(
|
|
133
|
-
database: () => Promise<
|
|
137
|
+
database: () => Promise<Db>,
|
|
134
138
|
options: { maxValidityMs: number },
|
|
135
139
|
): IdentityProvisioning {
|
|
136
140
|
const maximum = options.maxValidityMs;
|
|
@@ -138,38 +142,46 @@ export function postgresIdentityProvisioning(
|
|
|
138
142
|
async function transaction(
|
|
139
143
|
input: DeploymentIdentity,
|
|
140
144
|
create: boolean,
|
|
141
|
-
run: (
|
|
145
|
+
run: (tx: Db, enabled: boolean, now: number) => Promise<void>,
|
|
142
146
|
) {
|
|
143
147
|
checkIdentity(input);
|
|
144
|
-
let db: PoolClient | undefined;
|
|
145
148
|
try {
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
149
|
+
await (
|
|
150
|
+
await database()
|
|
151
|
+
).transaction(async (tx) => {
|
|
152
|
+
if (create)
|
|
153
|
+
await tx
|
|
154
|
+
.insert(identityDeployments)
|
|
155
|
+
.values({
|
|
156
|
+
id: input.deployment,
|
|
157
|
+
company: input.company,
|
|
158
|
+
environment: input.environment,
|
|
159
|
+
serviceAccount: input.serviceAccount,
|
|
160
|
+
})
|
|
161
|
+
.onConflictDoNothing({ target: identityDeployments.id });
|
|
162
|
+
// Every writer takes the deployment row lock: enrollment cannot race revocation/retirement.
|
|
163
|
+
const [row] = await tx
|
|
164
|
+
.select({
|
|
165
|
+
deployment: identityDeployments.id,
|
|
166
|
+
company: identityDeployments.company,
|
|
167
|
+
environment: identityDeployments.environment,
|
|
168
|
+
serviceAccount: identityDeployments.serviceAccount,
|
|
169
|
+
enabled: identityDeployments.enabled,
|
|
170
|
+
})
|
|
171
|
+
.from(identityDeployments)
|
|
172
|
+
.where(eq(identityDeployments.id, input.deployment))
|
|
173
|
+
.for("update");
|
|
174
|
+
if (!row || !sameIdentity(row, input)) fail("conflict");
|
|
175
|
+
const clock = await tx.execute<{ now: number }>(
|
|
176
|
+
sql`SELECT (extract(epoch FROM clock_timestamp()) * 1000)::float8 AS now`,
|
|
153
177
|
);
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
`SELECT id AS deployment, company, environment, service_account AS "serviceAccount", enabled
|
|
157
|
-
FROM sanoma_identity.deployments WHERE id=$1 FOR UPDATE`,
|
|
158
|
-
[input.deployment],
|
|
159
|
-
);
|
|
160
|
-
const row = rows[0];
|
|
161
|
-
if (!row || !sameIdentity(row, input)) fail("conflict");
|
|
162
|
-
const clock = await db.query<{ now: Date }>("SELECT clock_timestamp() AS now");
|
|
163
|
-
await run(db, row.enabled, clock.rows[0]!.now.getTime());
|
|
164
|
-
await db.query("COMMIT");
|
|
178
|
+
await run(tx, row.enabled, clock.rows[0]!.now);
|
|
179
|
+
});
|
|
165
180
|
} catch (error) {
|
|
166
|
-
await db?.query("ROLLBACK").catch(() => {});
|
|
167
181
|
const code = identityProvisioningCode(error);
|
|
168
182
|
if (code) throw new IdentityProvisioningError(code);
|
|
169
|
-
if ((error
|
|
183
|
+
if (pgCode(error) === "23505") throw new IdentityProvisioningError("conflict");
|
|
170
184
|
throw new IdentityProvisioningError("unavailable");
|
|
171
|
-
} finally {
|
|
172
|
-
db?.release();
|
|
173
185
|
}
|
|
174
186
|
}
|
|
175
187
|
return {
|
|
@@ -177,48 +189,64 @@ export function postgresIdentityProvisioning(
|
|
|
177
189
|
generationOf(input.generation);
|
|
178
190
|
const until = deadlineOf(input.validUntil);
|
|
179
191
|
if (!/^[0-9a-f]{64}$/.test(input.hash)) fail("invalid");
|
|
180
|
-
await transaction(input, true, async (
|
|
192
|
+
await transaction(input, true, async (tx, enabled, now) => {
|
|
181
193
|
if (!enabled) fail("conflict");
|
|
182
194
|
if (until <= now || until - now > maximum) fail("invalid");
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
195
|
+
const rows = await tx
|
|
196
|
+
.select({
|
|
197
|
+
generation: identityCredentials.generation,
|
|
198
|
+
hash: identityCredentials.hash,
|
|
199
|
+
validUntil: identityCredentials.validUntil,
|
|
200
|
+
revoked: identityCredentials.revoked,
|
|
201
|
+
})
|
|
202
|
+
.from(identityCredentials)
|
|
203
|
+
.where(eq(identityCredentials.deployment, input.deployment))
|
|
204
|
+
.orderBy(desc(identityCredentials.generation));
|
|
205
|
+
const existing = rows.find((r) => r.generation === input.generation);
|
|
189
206
|
if (existing) {
|
|
190
207
|
// An exact retry is safe. Never extend a retired generation or revive a revoked one.
|
|
191
|
-
if (existing.hash !== input.hash || existing.revoked || existing.
|
|
208
|
+
if (existing.hash !== input.hash || existing.revoked || existing.validUntil.getTime() !== until)
|
|
192
209
|
fail("conflict");
|
|
193
210
|
return;
|
|
194
211
|
}
|
|
195
|
-
if (rows[0] &&
|
|
196
|
-
await
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
212
|
+
if (rows[0] && rows[0].generation >= input.generation) fail("conflict");
|
|
213
|
+
await tx.insert(identityCredentials).values({
|
|
214
|
+
hash: input.hash,
|
|
215
|
+
deployment: input.deployment,
|
|
216
|
+
generation: input.generation,
|
|
217
|
+
validUntil: new Date(until),
|
|
218
|
+
});
|
|
201
219
|
});
|
|
202
220
|
},
|
|
203
221
|
async retire(input) {
|
|
204
222
|
generationOf(input.generation);
|
|
205
223
|
const until = deadlineOf(input.validUntil);
|
|
206
|
-
await transaction(input, false, async (
|
|
224
|
+
await transaction(input, false, async (tx) => {
|
|
207
225
|
// Retirement only shortens access. The predicate protects the schema's finite interval.
|
|
208
|
-
const
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
226
|
+
const deadline = sql`${new Date(until).toISOString()}::timestamptz`;
|
|
227
|
+
const updated = await tx
|
|
228
|
+
.update(identityCredentials)
|
|
229
|
+
.set({
|
|
230
|
+
revoked: sql`${identityCredentials.revoked} OR ${deadline} <= clock_timestamp()`,
|
|
231
|
+
validUntil: sql`CASE WHEN ${deadline} > ${identityCredentials.validFrom} THEN LEAST(${identityCredentials.validUntil}, ${deadline}) ELSE ${identityCredentials.validUntil} END`,
|
|
232
|
+
})
|
|
233
|
+
.where(
|
|
234
|
+
and(
|
|
235
|
+
eq(identityCredentials.deployment, input.deployment),
|
|
236
|
+
eq(identityCredentials.generation, input.generation),
|
|
237
|
+
),
|
|
238
|
+
)
|
|
239
|
+
.returning({ hash: identityCredentials.hash });
|
|
240
|
+
if (!updated.length) fail("conflict");
|
|
216
241
|
});
|
|
217
242
|
},
|
|
218
243
|
async revoke(input) {
|
|
219
|
-
await transaction(input, false, async (
|
|
244
|
+
await transaction(input, false, async (tx) => {
|
|
220
245
|
// Terminal for this deployment id. A new enrollment cannot silently re-enable it.
|
|
221
|
-
await
|
|
246
|
+
await tx
|
|
247
|
+
.update(identityDeployments)
|
|
248
|
+
.set({ enabled: false })
|
|
249
|
+
.where(eq(identityDeployments.id, input.deployment));
|
|
222
250
|
});
|
|
223
251
|
},
|
|
224
252
|
};
|
package/src/identity-service.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { createServer } from "node:http";
|
|
2
2
|
import type { AddressInfo } from "node:net";
|
|
3
|
+
import { sql } from "drizzle-orm";
|
|
3
4
|
import { Pool } from "pg";
|
|
4
5
|
import { NodeRequest, sendNodeResponse } from "srvx/node";
|
|
6
|
+
import { type Db, databaseOn } from "./db.ts";
|
|
7
|
+
import { identityAuditEvents } from "./db/identity.ts";
|
|
5
8
|
import {
|
|
6
9
|
identityBroker,
|
|
7
10
|
postgresIdentityEnrollments,
|
|
@@ -54,8 +57,8 @@ const poolFor = (connectionString: string) => {
|
|
|
54
57
|
return pool;
|
|
55
58
|
};
|
|
56
59
|
|
|
57
|
-
async function checkRoles(reader:
|
|
58
|
-
const read = await reader.
|
|
60
|
+
async function checkRoles(reader: Db, audit: Db): Promise<void> {
|
|
61
|
+
const read = await reader.execute<{ allowed: boolean }>(sql`SELECT
|
|
59
62
|
has_schema_privilege('sanoma_identity', 'USAGE') AND
|
|
60
63
|
NOT has_schema_privilege('sanoma_identity', 'CREATE') AND
|
|
61
64
|
NOT EXISTS (SELECT FROM pg_roles WHERE rolname = current_user AND (rolsuper OR rolcreaterole OR rolcreatedb OR rolbypassrls)) AND
|
|
@@ -63,7 +66,7 @@ async function checkRoles(reader: Pool, audit: Pool): Promise<void> {
|
|
|
63
66
|
has_table_privilege('sanoma_identity.credentials', 'SELECT') AND
|
|
64
67
|
NOT has_table_privilege('sanoma_identity.deployments', 'INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER') AND
|
|
65
68
|
NOT has_table_privilege('sanoma_identity.credentials', 'INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER') AS allowed`);
|
|
66
|
-
const write = await audit.
|
|
69
|
+
const write = await audit.execute<{ allowed: boolean }>(sql`SELECT
|
|
67
70
|
has_schema_privilege('sanoma_identity', 'USAGE') AND
|
|
68
71
|
NOT has_schema_privilege('sanoma_identity', 'CREATE') AND
|
|
69
72
|
NOT EXISTS (SELECT FROM pg_roles WHERE rolname = current_user AND (rolsuper OR rolcreaterole OR rolcreatedb OR rolbypassrls)) AND
|
|
@@ -94,26 +97,23 @@ export async function startIdentityService(options: IdentityServiceOptions): Pro
|
|
|
94
97
|
let active = 0;
|
|
95
98
|
reader = poolFor(options.enrollmentDatabaseUrl);
|
|
96
99
|
audit = poolFor(options.auditDatabaseUrl);
|
|
97
|
-
|
|
100
|
+
const readerDb = databaseOn(reader);
|
|
101
|
+
const auditDb = databaseOn(audit);
|
|
102
|
+
await checkRoles(readerDb, auditDb);
|
|
98
103
|
const record = async (event: IdentityAudit) => {
|
|
99
104
|
const e = event.enrollment;
|
|
100
105
|
// Fixed columns only: no tokens, hashes, headers or arbitrary caller fields.
|
|
101
|
-
await
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
e?.serviceAccount ?? null,
|
|
111
|
-
e?.generation ?? null,
|
|
112
|
-
event.expiresAt === undefined ? null : new Date(event.expiresAt),
|
|
113
|
-
],
|
|
114
|
-
);
|
|
106
|
+
await auditDb.insert(identityAuditEvents).values({
|
|
107
|
+
result: event.result,
|
|
108
|
+
deployment: e?.deployment,
|
|
109
|
+
company: e?.company,
|
|
110
|
+
environment: e?.environment,
|
|
111
|
+
serviceAccount: e?.serviceAccount,
|
|
112
|
+
generation: e?.generation,
|
|
113
|
+
expiresAt: event.expiresAt === undefined ? null : new Date(event.expiresAt),
|
|
114
|
+
});
|
|
115
115
|
};
|
|
116
|
-
const enrollments = postgresIdentityEnrollments(async () =>
|
|
116
|
+
const enrollments = postgresIdentityEnrollments(async () => readerDb);
|
|
117
117
|
const handle =
|
|
118
118
|
options.mode.type === "runtime"
|
|
119
119
|
? identityBroker({ enrollments, issue: attachedGoogleTokenIssuer(), audit: record })
|
package/src/index.ts
CHANGED
|
@@ -6,6 +6,7 @@ export {
|
|
|
6
6
|
type Connector,
|
|
7
7
|
type ConnectorSetup,
|
|
8
8
|
type ConnectorVendor,
|
|
9
|
+
type DatabasePreparation,
|
|
9
10
|
type Driver,
|
|
10
11
|
type DriverEnv,
|
|
11
12
|
type DriverFn,
|
|
@@ -212,6 +213,8 @@ export {
|
|
|
212
213
|
deriveHosts,
|
|
213
214
|
type HostFacts,
|
|
214
215
|
hostFacts,
|
|
216
|
+
type HostRootFacts,
|
|
217
|
+
hostRootFacts,
|
|
215
218
|
DEPLOY_KEY_TITLE,
|
|
216
219
|
DEPLOY_KEY_TYPE,
|
|
217
220
|
GITHUB_APP,
|
package/src/kms-secrets.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { createCipheriv, createDecipheriv, randomBytes, randomUUID } from "node:crypto";
|
|
2
2
|
import type { protos } from "@google-cloud/kms";
|
|
3
3
|
import crc32c from "crc-32/crc32c.js";
|
|
4
|
+
import { and, eq, sql } from "drizzle-orm";
|
|
4
5
|
import type { Db } from "./db.ts";
|
|
6
|
+
import { kmsSecrets } from "./db/kms.ts";
|
|
5
7
|
import { cloudIdentityAuth } from "./cloud-identity.ts";
|
|
6
8
|
import {
|
|
7
9
|
SecretStoreError,
|
|
@@ -58,7 +60,7 @@ export interface KmsPostgresOptions {
|
|
|
58
60
|
database?: () => Promise<Db>;
|
|
59
61
|
}
|
|
60
62
|
|
|
61
|
-
interface Envelope {
|
|
63
|
+
export interface Envelope {
|
|
62
64
|
format: "sanoma-gcp-kms-v1";
|
|
63
65
|
key: string;
|
|
64
66
|
keyVersion: string;
|
|
@@ -67,12 +69,17 @@ interface Envelope {
|
|
|
67
69
|
ciphertext: string;
|
|
68
70
|
tag: string;
|
|
69
71
|
}
|
|
70
|
-
|
|
71
|
-
version:
|
|
72
|
-
envelope:
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
}
|
|
72
|
+
const columns = {
|
|
73
|
+
version: kmsSecrets.version,
|
|
74
|
+
envelope: kmsSecrets.envelope,
|
|
75
|
+
updatedBy: kmsSecrets.updatedBy,
|
|
76
|
+
updatedAt: kmsSecrets.updatedAt,
|
|
77
|
+
};
|
|
78
|
+
type Row = Pick<typeof kmsSecrets.$inferSelect, keyof typeof columns>;
|
|
79
|
+
const at = (company: string, address: SecretAddress) =>
|
|
80
|
+
and(eq(kmsSecrets.company, company), eq(kmsSecrets.namespace, address.namespace), eq(kmsSecrets.path, address.path));
|
|
81
|
+
/** The row of `version`, compared as text: an expected version that is no uuid matches nothing. */
|
|
82
|
+
const versionIs = (version: string) => sql`${kmsSecrets.version}::text = ${version}`;
|
|
76
83
|
const FORMAT = "sanoma-gcp-kms-v1";
|
|
77
84
|
const MAX_BYTES = 65_536;
|
|
78
85
|
const capabilities = Object.freeze({
|
|
@@ -245,7 +252,8 @@ export function kmsPostgresStore(options: KmsPostgresOptions): SecretStore | Sec
|
|
|
245
252
|
}
|
|
246
253
|
};
|
|
247
254
|
const open = async (address: SecretAddress, row: Row): Promise<SecretValue> => {
|
|
248
|
-
|
|
255
|
+
// A JSON `null` is a jsonb value the column takes: an envelope written by hand, not one of ours.
|
|
256
|
+
const e = row.envelope as Envelope | null;
|
|
249
257
|
if (!e || e.format !== FORMAT || !keys.has(e.key) || !versionOf(e.keyVersion, e.key))
|
|
250
258
|
throw new SecretStoreError("invalid_value");
|
|
251
259
|
const nonce = bytes(e.nonce, 12),
|
|
@@ -280,8 +288,8 @@ export function kmsPostgresStore(options: KmsPostgresOptions): SecretStore | Sec
|
|
|
280
288
|
return {
|
|
281
289
|
value: plaintext.toString("utf8"),
|
|
282
290
|
version: row.version,
|
|
283
|
-
updatedBy: row.
|
|
284
|
-
updatedAt: row.
|
|
291
|
+
updatedBy: row.updatedBy,
|
|
292
|
+
updatedAt: row.updatedAt.toISOString(),
|
|
285
293
|
};
|
|
286
294
|
} finally {
|
|
287
295
|
plaintext.fill(0);
|
|
@@ -300,33 +308,35 @@ export function kmsPostgresStore(options: KmsPostgresOptions): SecretStore | Sec
|
|
|
300
308
|
value = randomUUID();
|
|
301
309
|
const envelope = await seal(address, value, version);
|
|
302
310
|
const db = await database();
|
|
303
|
-
const
|
|
311
|
+
const exact = and(at(company, address), eq(kmsSecrets.version, version));
|
|
304
312
|
let removed = false;
|
|
305
313
|
try {
|
|
306
|
-
const created = await db
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
314
|
+
const created = await db
|
|
315
|
+
.insert(kmsSecrets)
|
|
316
|
+
.values({
|
|
317
|
+
company,
|
|
318
|
+
namespace: address.namespace,
|
|
319
|
+
path: address.path,
|
|
320
|
+
version,
|
|
321
|
+
envelope,
|
|
322
|
+
updatedBy: "validation",
|
|
323
|
+
})
|
|
324
|
+
.onConflictDoNothing()
|
|
325
|
+
.returning({ version: kmsSecrets.version });
|
|
326
|
+
if (created.length !== 1) throw new SecretStoreError("conflict");
|
|
327
|
+
const updated = await db
|
|
328
|
+
.update(kmsSecrets)
|
|
329
|
+
.set({ envelope, version, updatedBy: "validation" })
|
|
330
|
+
.where(exact)
|
|
331
|
+
.returning({ version: kmsSecrets.version });
|
|
332
|
+
if (updated.length !== 1) throw new SecretStoreError("unavailable");
|
|
333
|
+
const [stored] = await db.select(columns).from(kmsSecrets).where(exact);
|
|
334
|
+
if (!stored || (await open(address, stored)).value !== value) throw new SecretStoreError("invalid_value");
|
|
322
335
|
} finally {
|
|
323
336
|
// Also clean up a committed INSERT whose response was lost. The random address and
|
|
324
337
|
// exact version prevent one concurrent probe from deleting another probe or a secret.
|
|
325
|
-
const cleanup = await db.
|
|
326
|
-
|
|
327
|
-
scope,
|
|
328
|
-
);
|
|
329
|
-
removed = cleanup.rowCount === 1;
|
|
338
|
+
const cleanup = await db.delete(kmsSecrets).where(exact).returning({ version: kmsSecrets.version });
|
|
339
|
+
removed = cleanup.length === 1;
|
|
330
340
|
}
|
|
331
341
|
if (!removed) throw new SecretStoreError("unavailable");
|
|
332
342
|
// Own-key usability only. Cross-company authorization requires live IAM proof with another existing key.
|
|
@@ -334,13 +344,8 @@ export function kmsPostgresStore(options: KmsPostgresOptions): SecretStore | Sec
|
|
|
334
344
|
read: (address) =>
|
|
335
345
|
safe(async () => {
|
|
336
346
|
context(address, "read");
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
).query<Row>(
|
|
340
|
-
"SELECT version,envelope,updated_by,updated_at FROM sanoma_kms.secrets WHERE company=$1 AND namespace=$2 AND path=$3",
|
|
341
|
-
[company, address.namespace, address.path],
|
|
342
|
-
);
|
|
343
|
-
return rows[0] && open(address, rows[0]);
|
|
347
|
+
const [row] = await (await database()).select(columns).from(kmsSecrets).where(at(company, address));
|
|
348
|
+
return row && open(address, row);
|
|
344
349
|
}),
|
|
345
350
|
write: (address, value, { expectedVersion, by }) =>
|
|
346
351
|
safe(async () => {
|
|
@@ -348,30 +353,33 @@ export function kmsPostgresStore(options: KmsPostgresOptions): SecretStore | Sec
|
|
|
348
353
|
if (!nonempty(by)) throw new SecretStoreError("invalid_value");
|
|
349
354
|
const version = randomUUID(),
|
|
350
355
|
envelope = await seal(address, value, version);
|
|
351
|
-
const
|
|
352
|
-
const
|
|
356
|
+
const db = await database();
|
|
357
|
+
const [row] =
|
|
353
358
|
expectedVersion === null
|
|
354
|
-
?
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
359
|
+
? await db
|
|
360
|
+
.insert(kmsSecrets)
|
|
361
|
+
.values({ company, namespace: address.namespace, path: address.path, version, envelope, updatedBy: by })
|
|
362
|
+
.onConflictDoNothing()
|
|
363
|
+
.returning({ updatedAt: kmsSecrets.updatedAt })
|
|
364
|
+
: await db
|
|
365
|
+
.update(kmsSecrets)
|
|
366
|
+
.set({ version, envelope, updatedBy: by })
|
|
367
|
+
.where(and(at(company, address), versionIs(expectedVersion)))
|
|
368
|
+
.returning({ updatedAt: kmsSecrets.updatedAt });
|
|
369
|
+
if (!row) throw new SecretStoreError("conflict");
|
|
360
370
|
// Return the caller's value, never make a second decrypt after a successful commit.
|
|
361
|
-
return { value, version, updatedBy: by, updatedAt:
|
|
371
|
+
return { value, version, updatedBy: by, updatedAt: row.updatedAt.toISOString() };
|
|
362
372
|
}),
|
|
363
373
|
delete: (address, { expectedVersion }) =>
|
|
364
374
|
safe(async () => {
|
|
365
375
|
context(address, expectedVersion);
|
|
366
|
-
const
|
|
376
|
+
const deleted = await (
|
|
367
377
|
await database()
|
|
368
|
-
)
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
]);
|
|
374
|
-
if (!rowCount) throw new SecretStoreError("conflict");
|
|
378
|
+
)
|
|
379
|
+
.delete(kmsSecrets)
|
|
380
|
+
.where(and(at(company, address), versionIs(expectedVersion)))
|
|
381
|
+
.returning({ version: kmsSecrets.version });
|
|
382
|
+
if (!deleted.length) throw new SecretStoreError("conflict");
|
|
375
383
|
}),
|
|
376
384
|
};
|
|
377
385
|
}
|
package/src/ledger.ts
CHANGED
|
@@ -3,9 +3,11 @@ import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
4
|
import { appendFile, mkdir, readFile, truncate } from "node:fs/promises";
|
|
5
5
|
import { join, resolve } from "node:path";
|
|
6
|
+
import { eq, sql } from "drizzle-orm";
|
|
6
7
|
import { setTimeout as delay } from "node:timers/promises";
|
|
7
8
|
import { isDeepStrictEqual } from "node:util";
|
|
8
|
-
import { type Db,
|
|
9
|
+
import { type Db, lazyDb, pgCode } from "./db.ts";
|
|
10
|
+
import { ledger } from "./db/company.ts";
|
|
9
11
|
import type { Approver, Principal } from "./define.ts";
|
|
10
12
|
import { type ErrorInfo, isFinal, keepCode } from "./errors.ts";
|
|
11
13
|
import { shown, warn } from "./log.ts";
|
|
@@ -372,29 +374,31 @@ const DATA_EXCEPTION = /^22/;
|
|
|
372
374
|
* Keeps records in the `sanoma_ledger` table of the database at `databaseUrl`, beside DBOS's
|
|
373
375
|
* tables: the default store, which the worker, the app and the client share through Postgres.
|
|
374
376
|
* The database and the table are made on first use when they are not there (`prepareDatabase`).
|
|
375
|
-
*
|
|
377
|
+
* Its connections are opened on first use and ended by `close()`, after which the store refuses use.
|
|
376
378
|
*/
|
|
377
379
|
export function postgresLedger(databaseUrl: string): LedgerStore & { close(): Promise<void> } {
|
|
378
|
-
const
|
|
380
|
+
const store = lazyDb(databaseUrl, "ledger");
|
|
379
381
|
|
|
380
382
|
return {
|
|
381
383
|
async append(record) {
|
|
382
|
-
const
|
|
383
|
-
if (await insertRecord(
|
|
384
|
+
const db = await store.db();
|
|
385
|
+
if (await insertRecord(db, record)) return;
|
|
384
386
|
// Another write of the id was first: the record it stored stands.
|
|
385
|
-
const
|
|
386
|
-
|
|
387
|
-
]);
|
|
388
|
-
if (rows[0]) warnIfDifferent(rows[0].record, lineOf(record));
|
|
387
|
+
const [stored] = await db.select({ record: ledger.record }).from(ledger).where(eq(ledger.id, record.id));
|
|
388
|
+
if (stored) warnIfDifferent(stored.record, lineOf(record));
|
|
389
389
|
},
|
|
390
390
|
async read(runId) {
|
|
391
|
-
const
|
|
392
|
-
await
|
|
393
|
-
)
|
|
391
|
+
const rows = await (
|
|
392
|
+
await store.db()
|
|
393
|
+
)
|
|
394
|
+
.select({ record: ledger.record })
|
|
395
|
+
.from(ledger)
|
|
396
|
+
.where(eq(ledger.runId, runId))
|
|
397
|
+
.orderBy(ledger.seq, ledger.n);
|
|
394
398
|
// In seq order, as the other stores read; two records of one seq in write order.
|
|
395
399
|
return rows.map((r) => r.record);
|
|
396
400
|
},
|
|
397
|
-
close:
|
|
401
|
+
close: store.close,
|
|
398
402
|
};
|
|
399
403
|
}
|
|
400
404
|
|
|
@@ -404,18 +408,19 @@ export function postgresLedger(databaseUrl: string): LedgerStore & { close(): Pr
|
|
|
404
408
|
* (`SanomaClient.call`) inside the transaction that saves its record.
|
|
405
409
|
*/
|
|
406
410
|
export async function insertRecord(db: Db, record: LedgerRecord): Promise<boolean> {
|
|
411
|
+
// The record's JSON, written once, as `jsonlLedger` writes it: what can't be JSON is refused as it refuses it.
|
|
407
412
|
const inserted = await db
|
|
408
|
-
.
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
)
|
|
413
|
+
.insert(ledger)
|
|
414
|
+
.values({ id: record.id, runId: record.runId, seq: record.seq, record: sql`${serialize(record)}::jsonb` })
|
|
415
|
+
.onConflictDoNothing({ target: ledger.id })
|
|
416
|
+
.returning({ n: ledger.n })
|
|
412
417
|
.catch((err: unknown) => {
|
|
413
418
|
if (DATA_EXCEPTION.test(pgCode(err))) {
|
|
414
419
|
throw final(`Ledger record ${record.id} can't be stored: ${errorMessage(err)}`, err);
|
|
415
420
|
}
|
|
416
421
|
throw err;
|
|
417
422
|
});
|
|
418
|
-
return inserted.
|
|
423
|
+
return inserted.length > 0;
|
|
419
424
|
}
|
|
420
425
|
|
|
421
426
|
/** Keeps records in memory, for tests. */
|
package/src/op.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Pool } from "pg";
|
|
1
2
|
import type { Principal } from "./define.ts";
|
|
2
3
|
import type { Authentication, CallAuthentication } from "./authentication.ts";
|
|
3
4
|
import type { ConnectionSnapshot } from "./connection.ts";
|
|
@@ -336,9 +337,15 @@ export interface ConnectorSetup {
|
|
|
336
337
|
fake?: () => Fake<any, any>;
|
|
337
338
|
/** The connector's steps for Gherkin tests (`defineSteps`): a config's scenarios and the connector's own features can use them. */
|
|
338
339
|
steps?: Steps;
|
|
340
|
+
/**
|
|
341
|
+
* Makes the connector's tables on the company's database (a login's): by the worker at its
|
|
342
|
+
* start, after the runtime's own tables, on the database's direct URL (`DatabasePreparation`);
|
|
343
|
+
* idempotent. The app never makes them: a login opened before they are there refuses.
|
|
344
|
+
*/
|
|
345
|
+
prepare?: DatabasePreparation;
|
|
339
346
|
/**
|
|
340
347
|
* Makes the connector's login half for one public site: once per site, by the app, when it
|
|
341
|
-
* starts
|
|
348
|
+
* starts, on the tables `prepare` made.
|
|
342
349
|
*/
|
|
343
350
|
login?: (context: LoginContext) => LoginHalf | Promise<LoginHalf>;
|
|
344
351
|
/**
|
|
@@ -370,6 +377,14 @@ export interface LoginIdentity {
|
|
|
370
377
|
github?: string;
|
|
371
378
|
}
|
|
372
379
|
|
|
380
|
+
/**
|
|
381
|
+
* Makes what a connector keeps on the company's database (`ConnectorSetup.prepare`): run by the
|
|
382
|
+
* worker at its start, once the runtime's own tables are there, with the database's direct URL,
|
|
383
|
+
* never through a pooler (an advisory lock is one connection's); idempotent, and safe for two
|
|
384
|
+
* workers at once.
|
|
385
|
+
*/
|
|
386
|
+
export type DatabasePreparation = (databaseUrl: string) => Promise<void>;
|
|
387
|
+
|
|
373
388
|
/** How a login half lets a person in: an email code, GitHub, an invite link, an email and a password. */
|
|
374
389
|
export type LoginMethod = "code" | "github" | "invite" | "password";
|
|
375
390
|
|
|
@@ -386,8 +401,14 @@ export interface SocialClients {
|
|
|
386
401
|
* public site, once per site; for the staff login, once, with `staff`.
|
|
387
402
|
*/
|
|
388
403
|
export interface LoginContext {
|
|
389
|
-
/**
|
|
390
|
-
|
|
404
|
+
/**
|
|
405
|
+
* The company's database, as the app's client reaches it (`SanomaClient.pool`): the pool a login
|
|
406
|
+
* like Better Auth keeps its users on, once the database is prepared. On Node it is the app's
|
|
407
|
+
* one pool for as long as the app runs; on a Worker it is the request's own, ended with the
|
|
408
|
+
* response. A login keeps one instance per pool and nothing else across requests, since the
|
|
409
|
+
* next request may bring another pool.
|
|
410
|
+
*/
|
|
411
|
+
database(): Promise<Pool>;
|
|
391
412
|
/**
|
|
392
413
|
* A 32-byte key derived from the deployment's `SANOMA_SIGNING_KEY` for this login (HKDF-SHA256,
|
|
393
414
|
* purpose `login <connector>`), hex; a login signs its cookies with it and never asks a person for a secret.
|
|
@@ -454,8 +475,10 @@ export function sameOrigin(request: Request): boolean {
|
|
|
454
475
|
/**
|
|
455
476
|
* A connector's login half: how a public site tells who a visitor is, and, made with
|
|
456
477
|
* `LoginContext.staff`, how the app's staff sign in. The app makes one with `login(context)` once
|
|
457
|
-
* per public site (and once for staff), when it starts
|
|
458
|
-
*
|
|
478
|
+
* per public site (and once for staff), when it starts, which opens nothing: a login reaches the
|
|
479
|
+
* database through `context.database` as requests come, and a login that keeps tables there has
|
|
480
|
+
* its connector's `prepare` make them, which the worker runs, and only checks for them. It
|
|
481
|
+
* answers the provider's subject and
|
|
459
482
|
* what it verified; the app makes the principal from it (`visitor:<connector>:<subject>`, in
|
|
460
483
|
* `visitors`, or the declared person the identity matches), so a connector never sets groups. A
|
|
461
484
|
* visitor's address, for rate limits, is the `CLIENT_IP_HEADER` header.
|
|
@@ -484,7 +507,6 @@ export interface LoginHalf {
|
|
|
484
507
|
* signs them in with it, using it up.
|
|
485
508
|
*/
|
|
486
509
|
invitation?(token: string): Promise<{ email: string; path: string } | undefined>;
|
|
487
|
-
close?(): Promise<void>;
|
|
488
510
|
}
|
|
489
511
|
|
|
490
512
|
/** How long an invite lasts: 24 hours. */
|