@segmentstream-cloud/ontology 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyze.js +16 -1
- package/dist/blob.d.ts +39 -0
- package/dist/blob.js +49 -0
- package/dist/entity.d.ts +5 -3
- package/dist/index.d.ts +4 -1
- package/dist/index.js +2 -0
- package/dist/manifest.d.ts +23 -7
- package/dist/manifest.js +9 -1
- package/dist/migrate/model.d.ts +7 -2
- package/dist/migrate/plan.js +26 -2
- package/dist/migrate/storage.d.ts +2 -0
- package/dist/migrate/storage.js +13 -0
- package/dist/property.d.ts +10 -1
- package/dist/property.js +12 -1
- package/dist/query.js +6 -1
- package/dist/runtime/generated-actions.js +7 -3
- package/dist/runtime/history.d.ts +3 -0
- package/dist/runtime/pipeline.d.ts +5 -1
- package/dist/runtime/pipeline.js +87 -3
- package/dist/runtime/records.js +9 -0
- package/dist/runtime/service-binding.d.ts +2 -1
- package/dist/runtime/service-binding.js +13 -2
- package/dist/runtime/writes.js +17 -3
- package/dist/service/api.js +15 -0
- package/dist/service/config.d.ts +3 -0
- package/dist/service/config.js +18 -0
- package/dist/service/handlers.js +40 -1
- package/dist/service/runtime-contract.d.ts +9 -0
- package/dist/service/schemas.d.ts +10 -2
- package/dist/service/schemas.js +18 -2
- package/dist/service/service.js +18 -3
- package/dist/storage/api.d.ts +4 -0
- package/dist/storage/api.js +39 -0
- package/dist/storage/authorization.d.ts +8 -0
- package/dist/storage/authorization.js +30 -0
- package/dist/storage/maintenance-auth.d.ts +20 -0
- package/dist/storage/maintenance-auth.js +24 -0
- package/dist/storage/maintenance.d.ts +7 -0
- package/dist/storage/maintenance.js +50 -0
- package/dist/storage/neon.d.ts +3 -0
- package/dist/storage/neon.js +212 -0
- package/dist/storage/operator.d.ts +10 -0
- package/dist/storage/operator.js +30 -0
- package/dist/storage/provider.d.ts +80 -0
- package/dist/storage/provider.js +20 -0
- package/dist/storage/sessions.d.ts +87 -0
- package/dist/storage/sessions.js +341 -0
- package/dist/storage/tables.d.ts +2 -0
- package/dist/storage/tables.js +55 -0
- package/dist/storage/writes.d.ts +7 -0
- package/dist/storage/writes.js +88 -0
- package/package.json +3 -1
package/dist/service/service.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { Cause, Context, Effect, Layer } from "effect";
|
|
1
|
+
import { Cause, Context, Effect, Layer, Redacted } from "effect";
|
|
2
2
|
import { HttpRouter, HttpServer, HttpServerError } from "effect/unstable/http";
|
|
3
3
|
import { HttpApiBuilder } from "effect/unstable/httpapi";
|
|
4
|
+
import { storageMaintenanceAuthenticationLayer } from "../storage/maintenance-auth.js";
|
|
4
5
|
import { nameToWire } from "../authoring-names.js";
|
|
5
6
|
import { sqlServiceRuntime } from "../runtime/service-binding.js";
|
|
6
7
|
import { ErrorResponsesLive, buildApi } from "./api.js";
|
|
@@ -58,18 +59,32 @@ function decorate(response, requestId, hash) {
|
|
|
58
59
|
* answer with a JSON error and never a stack or SQL.
|
|
59
60
|
*/
|
|
60
61
|
export function createOntologyService(ontology, config) {
|
|
62
|
+
if (config.blobStorage &&
|
|
63
|
+
(!config.storageMaintenanceKey || Redacted.value(config.storageMaintenanceKey).length < 32)) {
|
|
64
|
+
throw new OntologyServiceConfigError({
|
|
65
|
+
message: "File storage requires a deployment maintenance credential of at least 32 characters.",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
61
68
|
const runtime = config.runtime ?? sqlServiceRuntime;
|
|
62
69
|
const plan = planApi(ontology, runtime.generatedActions);
|
|
63
70
|
const built = buildApi(plan);
|
|
64
71
|
const hooks = afterCommitHooks(plan, config.afterCommit);
|
|
65
72
|
const verifyToken = tokenVerifier(config);
|
|
66
73
|
const exchangeSession = config.neonAuth === undefined ? undefined : neonAuthSessionExchange(config.neonAuth);
|
|
67
|
-
const backend = runtime.connect({
|
|
74
|
+
const backend = runtime.connect({
|
|
75
|
+
ontology,
|
|
76
|
+
databaseUrl: config.databaseUrl,
|
|
77
|
+
...(config.blobStorage ? { blobStorage: config.blobStorage } : {}),
|
|
78
|
+
});
|
|
68
79
|
const connections = config.connections ??
|
|
69
80
|
(config.localConnections === undefined
|
|
70
81
|
? deployedConnections(config)
|
|
71
82
|
: localConnections(config.localConnections));
|
|
72
|
-
const routes = HttpApiBuilder.layer(built.api).pipe(Layer.provide(handlersLayer(built, hooks, connections)), Layer.provide([
|
|
83
|
+
const routes = HttpApiBuilder.layer(built.api).pipe(Layer.provide(handlersLayer(built, hooks, connections)), Layer.provide([
|
|
84
|
+
authenticationLayer(verifyToken, exchangeSession),
|
|
85
|
+
ErrorResponsesLive,
|
|
86
|
+
storageMaintenanceAuthenticationLayer(config.storageMaintenanceKey),
|
|
87
|
+
]), Layer.provide(backend));
|
|
73
88
|
const application = Layer.mergeAll(routes, DocsRoute, UnmatchedRoutes).pipe(Layer.provide(HttpServer.layerServices));
|
|
74
89
|
const web = HttpRouter.toWebHandler(application, { disableLogger: true });
|
|
75
90
|
const hash = ontology.manifest.hash;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi";
|
|
3
|
+
export declare const STORAGE_GROUP = "BlobStorage";
|
|
4
|
+
export declare function storageGroup(authenticate: (endpoint: HttpApiEndpoint.Top) => HttpApiEndpoint.Top, errors: ReadonlyArray<Schema.Top>, maintenanceErrors: (endpoint: HttpApiEndpoint.Top) => HttpApiEndpoint.Top): HttpApiGroup.HttpApiGroup<"BlobStorage", HttpApiEndpoint.Top | HttpApiEndpoint.HttpApiEndpoint<string, import("effect/unstable/http/HttpMethod").HttpMethod, string, Schema.Top, Schema.Top, Schema.Top, Schema.Top, Schema.Top, Schema.Top, any, unknown>, false>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi";
|
|
3
|
+
import { StorageMaintenanceAuthentication } from "./maintenance-auth.js";
|
|
4
|
+
import { BeginUpload, UploadView, TransferGrant } from "./sessions.js";
|
|
5
|
+
export const STORAGE_GROUP = "BlobStorage";
|
|
6
|
+
export function storageGroup(authenticate, errors, maintenanceErrors) {
|
|
7
|
+
const base = "/api/ontology/uploads";
|
|
8
|
+
const params = { id: Schema.String.check(Schema.isUUID()) };
|
|
9
|
+
const endpoints = [
|
|
10
|
+
HttpApiEndpoint.post("begin", base, {
|
|
11
|
+
payload: BeginUpload,
|
|
12
|
+
success: UploadView,
|
|
13
|
+
error: errors,
|
|
14
|
+
}),
|
|
15
|
+
HttpApiEndpoint.get("status", `${base}/:id`, { params, success: UploadView, error: errors }),
|
|
16
|
+
HttpApiEndpoint.post("part", `${base}/:id/parts`, {
|
|
17
|
+
params,
|
|
18
|
+
success: TransferGrant,
|
|
19
|
+
error: errors,
|
|
20
|
+
}),
|
|
21
|
+
HttpApiEndpoint.post("complete", `${base}/:id/complete`, {
|
|
22
|
+
params,
|
|
23
|
+
success: UploadView,
|
|
24
|
+
error: errors,
|
|
25
|
+
}),
|
|
26
|
+
HttpApiEndpoint.post("cancel", `${base}/:id/cancel`, {
|
|
27
|
+
params,
|
|
28
|
+
success: UploadView,
|
|
29
|
+
error: errors,
|
|
30
|
+
}),
|
|
31
|
+
];
|
|
32
|
+
const [first, ...rest] = endpoints.map(authenticate);
|
|
33
|
+
return HttpApiGroup.make(STORAGE_GROUP)
|
|
34
|
+
.add(first, ...rest)
|
|
35
|
+
.add(maintenanceErrors(HttpApiEndpoint.post("maintain", "/api/ontology/internal/storage/maintenance", {
|
|
36
|
+
success: Schema.Struct({ removed: Schema.Int }),
|
|
37
|
+
error: errors,
|
|
38
|
+
})).middleware(StorageMaintenanceAuthentication));
|
|
39
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import type { SqlError } from "effect/unstable/sql/SqlError";
|
|
3
|
+
import type { Ontology } from "../ontology.js";
|
|
4
|
+
import { PermissionDenied } from "../errors.js";
|
|
5
|
+
/** Re-read user and roles in the same transaction that authorizes a storage operation. */
|
|
6
|
+
export declare function authorizeBlobOperation(ontology: Ontology, sql: {
|
|
7
|
+
readonly unsafe: (text: string, args: ReadonlyArray<unknown>) => Effect.Effect<ReadonlyArray<Readonly<Record<string, unknown>>>, SqlError>;
|
|
8
|
+
}, principalId: string, required: ReadonlyArray<string>): Effect.Effect<undefined, PermissionDenied | SqlError, never>;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { missingPermission, permissionsOf } from "../permissions.js";
|
|
3
|
+
import { PermissionDenied } from "../errors.js";
|
|
4
|
+
/** Re-read user and roles in the same transaction that authorizes a storage operation. */
|
|
5
|
+
export function authorizeBlobOperation(ontology, sql, principalId, required) {
|
|
6
|
+
return Effect.gen(function* () {
|
|
7
|
+
const [catalog] = yield* sql.unsafe(`SELECT hash FROM "_ontology" WHERE id = true FOR SHARE`, []);
|
|
8
|
+
if (catalog?.hash !== ontology.manifest.hash) {
|
|
9
|
+
return yield* new PermissionDenied({
|
|
10
|
+
message: "The deployed ontology no longer matches the database catalog.",
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
const users = yield* sql.unsafe(`SELECT kind::text, active FROM "user" WHERE id = $1::uuid FOR SHARE`, [principalId]);
|
|
14
|
+
const user = users[0];
|
|
15
|
+
const roles = yield* sql.unsafe(`SELECT role FROM "_user_roles" WHERE user_id = $1::uuid FOR SHARE`, [principalId]);
|
|
16
|
+
const held = roles.map((row) => String(row.role));
|
|
17
|
+
if (user?.kind === "member") {
|
|
18
|
+
held.push("member");
|
|
19
|
+
}
|
|
20
|
+
const missing = missingPermission(required, {
|
|
21
|
+
permissions: permissionsOf(ontology.roles, held),
|
|
22
|
+
});
|
|
23
|
+
if (!user?.active || missing) {
|
|
24
|
+
return yield* new PermissionDenied({
|
|
25
|
+
message: "The member cannot use this file operation.",
|
|
26
|
+
...(missing ? { permission: missing } : {}),
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Layer, Redacted } from "effect";
|
|
2
|
+
import { HttpApiMiddleware, HttpApiSecurity } from "effect/unstable/httpapi";
|
|
3
|
+
import { RequestId } from "../service/errors.js";
|
|
4
|
+
declare const StorageMaintenanceAuthentication_base: HttpApiMiddleware.ServiceClass<StorageMaintenanceAuthentication, "@segmentstream-cloud/ontology/StorageMaintenanceAuthentication", {
|
|
5
|
+
requires: RequestId;
|
|
6
|
+
provides: never;
|
|
7
|
+
error: readonly [import("effect/Schema").Top];
|
|
8
|
+
clientError: never;
|
|
9
|
+
requiredForClient: false;
|
|
10
|
+
security: {
|
|
11
|
+
readonly maintenanceBearer: HttpApiSecurity.Http;
|
|
12
|
+
};
|
|
13
|
+
}, HttpApiMiddleware.HttpApiMiddlewareSecurity<{
|
|
14
|
+
readonly maintenanceBearer: HttpApiSecurity.Http;
|
|
15
|
+
}, never, readonly [import("effect/Schema").Top], RequestId>>;
|
|
16
|
+
/** Deployment scheduler identity: this credential grants only orphan maintenance. */
|
|
17
|
+
export declare class StorageMaintenanceAuthentication extends StorageMaintenanceAuthentication_base {
|
|
18
|
+
}
|
|
19
|
+
export declare function storageMaintenanceAuthenticationLayer(key?: Redacted.Redacted<string>): Layer.Layer<StorageMaintenanceAuthentication, never, never>;
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { createHash, timingSafeEqual } from "node:crypto";
|
|
2
|
+
import { Effect, Layer, Redacted } from "effect";
|
|
3
|
+
import { HttpApiMiddleware, HttpApiSecurity } from "effect/unstable/httpapi";
|
|
4
|
+
import { RequestId, toServerResponse, unauthorized } from "../service/errors.js";
|
|
5
|
+
import { ErrorSchemas } from "../service/schemas.js";
|
|
6
|
+
/** Deployment scheduler identity: this credential grants only orphan maintenance. */
|
|
7
|
+
export class StorageMaintenanceAuthentication extends HttpApiMiddleware.Service()("@segmentstream-cloud/ontology/StorageMaintenanceAuthentication", {
|
|
8
|
+
security: { maintenanceBearer: HttpApiSecurity.bearer },
|
|
9
|
+
error: [ErrorSchemas.unauthorized],
|
|
10
|
+
}) {
|
|
11
|
+
}
|
|
12
|
+
export function storageMaintenanceAuthenticationLayer(key) {
|
|
13
|
+
return Layer.succeed(StorageMaintenanceAuthentication)({
|
|
14
|
+
maintenanceBearer: (httpEffect, { credential }) => Effect.gen(function* () {
|
|
15
|
+
const requestId = yield* RequestId;
|
|
16
|
+
const digest = (value) => createHash("sha256").update(value).digest();
|
|
17
|
+
if (!key ||
|
|
18
|
+
!timingSafeEqual(digest(Redacted.value(credential)), digest(Redacted.value(key)))) {
|
|
19
|
+
return toServerResponse(unauthorized(requestId, "A deployment maintenance credential is required."));
|
|
20
|
+
}
|
|
21
|
+
return yield* httpEffect;
|
|
22
|
+
}),
|
|
23
|
+
});
|
|
24
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import type { SqlClient } from "effect/unstable/sql";
|
|
3
|
+
import type { BlobProvider } from "./provider.js";
|
|
4
|
+
/** A scheduler-owned capability: lifecycle operations only, with persisted deletion fencing. */
|
|
5
|
+
export declare function maintainBlobStorage(sql: SqlClient.SqlClient, provider: BlobProvider["Service"], limit?: number): Effect.Effect<{
|
|
6
|
+
removed: number;
|
|
7
|
+
}, import("effect/unstable/sql/SqlError").SqlError | import("./provider.js").BlobStorageError, never>;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
/** A scheduler-owned capability: lifecycle operations only, with persisted deletion fencing. */
|
|
4
|
+
export function maintainBlobStorage(sql, provider, limit = 100) {
|
|
5
|
+
return Effect.gen(function* () {
|
|
6
|
+
let removed = 0;
|
|
7
|
+
for (let count = 0; count < limit; count += 1) {
|
|
8
|
+
const claim = randomUUID();
|
|
9
|
+
const candidate = yield* sql.withTransaction(Effect.gen(function* () {
|
|
10
|
+
const [row] = yield* sql.unsafe(`SELECT u.id::text, u.blob_id::text, b.key, u.provider_upload_id
|
|
11
|
+
FROM "_blob_uploads" u JOIN "_blobs" b ON b.id = u.blob_id
|
|
12
|
+
WHERE u.state NOT IN ('consumed', 'deleted') AND b.state IN ('ready', 'deleting')
|
|
13
|
+
AND (u.branch_id <> $1 OR u.expires_at <= now() OR u.state IN ('canceled', 'deleting'))
|
|
14
|
+
AND (u.claim IS NULL OR u.claimed_at < now() - interval '5 minutes')
|
|
15
|
+
AND NOT EXISTS (SELECT 1 FROM "_blob_references" r WHERE r.blob_id = b.id)
|
|
16
|
+
ORDER BY u.created_at LIMIT 1 FOR UPDATE OF u, b SKIP LOCKED`, [provider.binding.branchId]);
|
|
17
|
+
if (!row) {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
yield* sql.unsafe(`UPDATE "_blob_uploads" SET state = 'deleting', claim = $1::uuid,
|
|
21
|
+
claimed_at = now() WHERE id = $2::uuid`, [claim, row.id]);
|
|
22
|
+
yield* sql.unsafe(`UPDATE "_blobs" SET state = 'deleting' WHERE id = $1::uuid`, [
|
|
23
|
+
row.blob_id,
|
|
24
|
+
]);
|
|
25
|
+
return row;
|
|
26
|
+
}));
|
|
27
|
+
if (!candidate) {
|
|
28
|
+
break;
|
|
29
|
+
}
|
|
30
|
+
const handles = new Set([
|
|
31
|
+
...(candidate.provider_upload_id ? [candidate.provider_upload_id] : []),
|
|
32
|
+
...(yield* provider.pending(candidate.key)),
|
|
33
|
+
]);
|
|
34
|
+
for (const uploadId of handles) {
|
|
35
|
+
yield* provider.abort({ key: candidate.key, uploadId });
|
|
36
|
+
}
|
|
37
|
+
yield* provider.remove(candidate.key);
|
|
38
|
+
yield* sql.withTransaction(Effect.gen(function* () {
|
|
39
|
+
const done = yield* sql.unsafe(`UPDATE "_blob_uploads" SET state = 'deleted', claim = NULL,
|
|
40
|
+
claimed_at = NULL WHERE id = $1::uuid AND state = 'deleting' AND claim = $2::uuid
|
|
41
|
+
RETURNING blob_id`, [candidate.id, claim]);
|
|
42
|
+
if (done.length) {
|
|
43
|
+
yield* sql.unsafe(`UPDATE "_blobs" SET state = 'deleted' WHERE id = $1::uuid AND state = 'deleting'`, [candidate.blob_id]);
|
|
44
|
+
removed += 1;
|
|
45
|
+
}
|
|
46
|
+
}));
|
|
47
|
+
}
|
|
48
|
+
return { removed };
|
|
49
|
+
});
|
|
50
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListMultipartUploadsCommand, ListPartsCommand, S3Client, UploadPartCommand, } from "@aws-sdk/client-s3";
|
|
2
|
+
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
3
|
+
import { Effect } from "effect";
|
|
4
|
+
import { BlobStorageError, } from "./provider.js";
|
|
5
|
+
function missing(error) {
|
|
6
|
+
return error instanceof Error && ["NotFound", "NoSuchKey", "NoSuchUpload"].includes(error.name);
|
|
7
|
+
}
|
|
8
|
+
function redactStorageFailure(cause, binding) {
|
|
9
|
+
const seen = new WeakSet();
|
|
10
|
+
const redact = (value) => {
|
|
11
|
+
if (typeof value === "string") {
|
|
12
|
+
return [binding.accessKeyId, binding.secretAccessKey]
|
|
13
|
+
.reduce((text, secret) => secret
|
|
14
|
+
? text
|
|
15
|
+
.replaceAll(secret, "[redacted]")
|
|
16
|
+
.replaceAll(encodeURIComponent(secret), "[redacted]")
|
|
17
|
+
: text, value)
|
|
18
|
+
.replace(/https?:\/\/[^\s"'<>]*[?&]X-Amz-(?:Signature|Credential)=[^\s"'<>]*/gi, "[redacted storage grant]")
|
|
19
|
+
.replace(/(X-Amz-Signature=)[^&\s]+/gi, "$1[redacted]");
|
|
20
|
+
}
|
|
21
|
+
if (value === null || typeof value !== "object") {
|
|
22
|
+
return value;
|
|
23
|
+
}
|
|
24
|
+
if (seen.has(value)) {
|
|
25
|
+
return "[Circular]";
|
|
26
|
+
}
|
|
27
|
+
seen.add(value);
|
|
28
|
+
if (Array.isArray(value)) {
|
|
29
|
+
return value.map(redact);
|
|
30
|
+
}
|
|
31
|
+
const entries = value instanceof Error
|
|
32
|
+
? Object.fromEntries([
|
|
33
|
+
["name", value.name],
|
|
34
|
+
["message", value.message],
|
|
35
|
+
["stack", value.stack],
|
|
36
|
+
["cause", value.cause],
|
|
37
|
+
...(value instanceof AggregateError ? [["errors", value.errors]] : []),
|
|
38
|
+
...Object.entries(value),
|
|
39
|
+
])
|
|
40
|
+
: value;
|
|
41
|
+
return Object.fromEntries(Object.entries(entries).map(([key, child]) => [key, redact(child)]));
|
|
42
|
+
};
|
|
43
|
+
return redact(cause);
|
|
44
|
+
}
|
|
45
|
+
/** A branch-bound S3 adapter; never uses the ambient AWS credential chain. */
|
|
46
|
+
export function neonBlobProvider(binding) {
|
|
47
|
+
const endpoint = new URL(binding.endpoint);
|
|
48
|
+
if (endpoint.protocol !== "https:" ||
|
|
49
|
+
endpoint.username ||
|
|
50
|
+
endpoint.password ||
|
|
51
|
+
!endpoint.hostname.startsWith(`${binding.branchId}.storage.`) ||
|
|
52
|
+
!endpoint.hostname.endsWith(".aws.neon.tech") ||
|
|
53
|
+
endpoint.pathname !== "/" ||
|
|
54
|
+
endpoint.search ||
|
|
55
|
+
endpoint.hash ||
|
|
56
|
+
endpoint.port) {
|
|
57
|
+
throw new Error("The ontology storage endpoint does not match its verified branch.");
|
|
58
|
+
}
|
|
59
|
+
const client = new S3Client({
|
|
60
|
+
endpoint: binding.endpoint,
|
|
61
|
+
region: binding.region,
|
|
62
|
+
credentials: { accessKeyId: binding.accessKeyId, secretAccessKey: binding.secretAccessKey },
|
|
63
|
+
forcePathStyle: true,
|
|
64
|
+
requestChecksumCalculation: "WHEN_REQUIRED",
|
|
65
|
+
maxAttempts: 1,
|
|
66
|
+
requestHandler: { requestTimeout: 30000 },
|
|
67
|
+
});
|
|
68
|
+
let storageClockOffset;
|
|
69
|
+
// The SDK's normal skew correction tolerates four minutes, longer than a download grant.
|
|
70
|
+
// Calibrate every grant from an authenticated response instead of extending its lifetime.
|
|
71
|
+
client.middlewareStack.add((next) => async (args) => {
|
|
72
|
+
const result = await next(args);
|
|
73
|
+
const response = result.response;
|
|
74
|
+
const serverTime = Date.parse(response.headers?.date ?? "");
|
|
75
|
+
if (Number.isFinite(serverTime)) {
|
|
76
|
+
storageClockOffset = serverTime - Date.now();
|
|
77
|
+
}
|
|
78
|
+
return result;
|
|
79
|
+
}, { step: "deserialize", name: "ontologyStorageClock" });
|
|
80
|
+
const signingDate = () => {
|
|
81
|
+
if (storageClockOffset === undefined) {
|
|
82
|
+
throw new Error("Neon did not return a valid server time for the transfer grant.");
|
|
83
|
+
}
|
|
84
|
+
return new Date(Date.now() + storageClockOffset);
|
|
85
|
+
};
|
|
86
|
+
const object = (key) => ({ Bucket: binding.bucket, Key: key });
|
|
87
|
+
const disposition = (filename, inline = false) => `${inline ? "inline" : "attachment"}; filename="${filename.replace(/[^a-zA-Z0-9_. -]/g, "_")}"`;
|
|
88
|
+
const attempt = (operation, run) => Effect.tryPromise({
|
|
89
|
+
try: run,
|
|
90
|
+
catch: (cause) => new BlobStorageError({ operation, cause: redactStorageFailure(cause, binding) }),
|
|
91
|
+
});
|
|
92
|
+
const grant = (url, method, seconds, signedAt) => ({
|
|
93
|
+
url,
|
|
94
|
+
method,
|
|
95
|
+
headers: {},
|
|
96
|
+
expiresAt: new Date(signedAt.getTime() + seconds * 1000).toISOString(),
|
|
97
|
+
});
|
|
98
|
+
return {
|
|
99
|
+
binding,
|
|
100
|
+
close: () => client.destroy(),
|
|
101
|
+
initiate: ({ key, filename, contentType, sessionId }) => attempt("blob.initiate", async () => {
|
|
102
|
+
const response = await client.send(new CreateMultipartUploadCommand({
|
|
103
|
+
...object(key),
|
|
104
|
+
ContentType: contentType,
|
|
105
|
+
ContentDisposition: disposition(filename),
|
|
106
|
+
CacheControl: "private, no-store",
|
|
107
|
+
Metadata: { session: sessionId },
|
|
108
|
+
}));
|
|
109
|
+
if (!response.UploadId) {
|
|
110
|
+
throw new Error("Neon did not return a multipart upload ID.");
|
|
111
|
+
}
|
|
112
|
+
return response.UploadId;
|
|
113
|
+
}),
|
|
114
|
+
signPart: ({ key, uploadId, partNumber, size }) => attempt("blob.sign_part", async () => {
|
|
115
|
+
await client.send(new ListPartsCommand({ ...object(key), UploadId: uploadId }));
|
|
116
|
+
const signedAt = signingDate();
|
|
117
|
+
const url = await getSignedUrl(client, new UploadPartCommand({
|
|
118
|
+
...object(key),
|
|
119
|
+
UploadId: uploadId,
|
|
120
|
+
PartNumber: partNumber,
|
|
121
|
+
ContentLength: size,
|
|
122
|
+
}), {
|
|
123
|
+
expiresIn: 300,
|
|
124
|
+
signingDate: signedAt,
|
|
125
|
+
signableHeaders: new Set(["content-length"]),
|
|
126
|
+
});
|
|
127
|
+
return grant(url, "PUT", 300, signedAt);
|
|
128
|
+
}),
|
|
129
|
+
parts: ({ key, uploadId }) => attempt("blob.parts", async () => {
|
|
130
|
+
const response = await client.send(new ListPartsCommand({ ...object(key), UploadId: uploadId }));
|
|
131
|
+
if (response.IsTruncated) {
|
|
132
|
+
throw new Error("Unexpected truncated multipart session.");
|
|
133
|
+
}
|
|
134
|
+
return (response.Parts ?? []).map((part) => {
|
|
135
|
+
if (!part.PartNumber || part.Size === undefined || !part.ETag) {
|
|
136
|
+
throw new Error("Neon returned incomplete part metadata.");
|
|
137
|
+
}
|
|
138
|
+
return { number: part.PartNumber, size: part.Size, etag: part.ETag };
|
|
139
|
+
});
|
|
140
|
+
}),
|
|
141
|
+
complete: ({ key, uploadId, parts }) => attempt("blob.complete", async () => {
|
|
142
|
+
await client.send(new CompleteMultipartUploadCommand({
|
|
143
|
+
...object(key),
|
|
144
|
+
UploadId: uploadId,
|
|
145
|
+
MultipartUpload: {
|
|
146
|
+
Parts: parts.map((part) => ({ PartNumber: part.number, ETag: part.etag })),
|
|
147
|
+
},
|
|
148
|
+
}));
|
|
149
|
+
}),
|
|
150
|
+
inspect: (key) => attempt("blob.inspect", async () => {
|
|
151
|
+
try {
|
|
152
|
+
const response = await client.send(new HeadObjectCommand(object(key)));
|
|
153
|
+
if (response.ContentLength === undefined ||
|
|
154
|
+
!response.ContentType ||
|
|
155
|
+
!response.Metadata?.session) {
|
|
156
|
+
throw new Error("Neon returned incomplete object metadata.");
|
|
157
|
+
}
|
|
158
|
+
return {
|
|
159
|
+
size: response.ContentLength,
|
|
160
|
+
contentType: response.ContentType,
|
|
161
|
+
sessionId: response.Metadata.session,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
if (missing(error)) {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
throw error;
|
|
169
|
+
}
|
|
170
|
+
}),
|
|
171
|
+
readPrefix: (key) => attempt("blob.read_prefix", async () => {
|
|
172
|
+
const response = await client.send(new GetObjectCommand({ ...object(key), Range: "bytes=0-511" }));
|
|
173
|
+
if (!response.Body) {
|
|
174
|
+
throw new Error("Neon returned no object body.");
|
|
175
|
+
}
|
|
176
|
+
return response.Body.transformToByteArray();
|
|
177
|
+
}),
|
|
178
|
+
download: ({ key, filename, contentType, inline }) => attempt("blob.download", async () => {
|
|
179
|
+
await client.send(new HeadObjectCommand(object(key)));
|
|
180
|
+
const signedAt = signingDate();
|
|
181
|
+
const url = await getSignedUrl(client, new GetObjectCommand({
|
|
182
|
+
...object(key),
|
|
183
|
+
ResponseContentType: contentType,
|
|
184
|
+
ResponseContentDisposition: disposition(filename, inline),
|
|
185
|
+
ResponseCacheControl: "private, no-store",
|
|
186
|
+
}), { expiresIn: 60, signingDate: signedAt });
|
|
187
|
+
return grant(url, "GET", 60, signedAt);
|
|
188
|
+
}),
|
|
189
|
+
abort: ({ key, uploadId }) => attempt("blob.abort", async () => {
|
|
190
|
+
try {
|
|
191
|
+
await client.send(new AbortMultipartUploadCommand({ ...object(key), UploadId: uploadId }));
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
if (!missing(error)) {
|
|
195
|
+
throw error;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}),
|
|
199
|
+
remove: (key) => attempt("blob.remove", async () => {
|
|
200
|
+
await client.send(new DeleteObjectCommand(object(key)));
|
|
201
|
+
}),
|
|
202
|
+
pending: (key) => attempt("blob.pending", async () => {
|
|
203
|
+
const response = await client.send(new ListMultipartUploadsCommand({ Bucket: binding.bucket, Prefix: key }));
|
|
204
|
+
if (response.IsTruncated) {
|
|
205
|
+
throw new Error("Unexpected truncated initialization recovery.");
|
|
206
|
+
}
|
|
207
|
+
return (response.Uploads ?? [])
|
|
208
|
+
.filter((upload) => upload.Key === key && upload.UploadId)
|
|
209
|
+
.map((upload) => upload.UploadId);
|
|
210
|
+
}),
|
|
211
|
+
};
|
|
212
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Effect, Redacted } from "effect";
|
|
2
|
+
import { BlobStorageError, type BlobStorageBinding } from "./provider.js";
|
|
3
|
+
/** Scheduler capability that survives removal of blob declarations. Never reads file contents. */
|
|
4
|
+
export declare function runBlobStorageMaintenance(input: {
|
|
5
|
+
readonly databaseUrl: Redacted.Redacted<string>;
|
|
6
|
+
readonly binding: Redacted.Redacted<BlobStorageBinding>;
|
|
7
|
+
readonly limit?: number;
|
|
8
|
+
}): Effect.Effect<{
|
|
9
|
+
removed: number;
|
|
10
|
+
}, import("effect/unstable/sql/SqlError").SqlError | BlobStorageError, never>;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import * as PgClient from "@effect/sql-pg/PgClient";
|
|
2
|
+
import { Effect, Redacted } from "effect";
|
|
3
|
+
import { SqlClient } from "effect/unstable/sql";
|
|
4
|
+
import { postgresTls } from "../runtime/postgres-tls.js";
|
|
5
|
+
import { maintainBlobStorage } from "./maintenance.js";
|
|
6
|
+
import { neonBlobProvider } from "./neon.js";
|
|
7
|
+
import { BlobStorageError } from "./provider.js";
|
|
8
|
+
/** Scheduler capability that survives removal of blob declarations. Never reads file contents. */
|
|
9
|
+
export function runBlobStorageMaintenance(input) {
|
|
10
|
+
const limit = input.limit ?? 100;
|
|
11
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
|
|
12
|
+
return Effect.fail(new BlobStorageError({ operation: "maintenance.limit" }));
|
|
13
|
+
}
|
|
14
|
+
return Effect.gen(function* () {
|
|
15
|
+
const sql = yield* SqlClient.SqlClient;
|
|
16
|
+
const [tables] = yield* sql `
|
|
17
|
+
SELECT to_regclass('public._blob_uploads')::text AS uploads
|
|
18
|
+
`;
|
|
19
|
+
if (!tables?.uploads) {
|
|
20
|
+
return { removed: 0 };
|
|
21
|
+
}
|
|
22
|
+
const provider = yield* Effect.acquireRelease(Effect.sync(() => neonBlobProvider(Redacted.value(input.binding))), (provider) => Effect.sync(() => provider.close?.()));
|
|
23
|
+
return yield* maintainBlobStorage(sql, provider, limit);
|
|
24
|
+
}).pipe(Effect.provide(PgClient.layer({
|
|
25
|
+
url: input.databaseUrl,
|
|
26
|
+
ssl: postgresTls(input.databaseUrl),
|
|
27
|
+
maxConnections: 1,
|
|
28
|
+
applicationName: "segmentstream-ontology-maintenance",
|
|
29
|
+
})), Effect.scoped);
|
|
30
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { Context, Schema, type Effect } from "effect";
|
|
2
|
+
/** Validated deployment binding, independent of the action and browser contracts. */
|
|
3
|
+
export declare const BlobStorageBinding: Schema.Struct<{
|
|
4
|
+
readonly projectId: Schema.String;
|
|
5
|
+
readonly branchId: Schema.String;
|
|
6
|
+
readonly endpoint: Schema.String;
|
|
7
|
+
readonly region: Schema.String;
|
|
8
|
+
readonly bucket: Schema.Literal<"ontology-files">;
|
|
9
|
+
readonly accessKeyId: Schema.String;
|
|
10
|
+
readonly secretAccessKey: Schema.String;
|
|
11
|
+
readonly quotaBytes: Schema.Int;
|
|
12
|
+
}>;
|
|
13
|
+
export type BlobStorageBinding = typeof BlobStorageBinding.Type;
|
|
14
|
+
declare const BlobStorageError_base: Schema.Class<BlobStorageError, Schema.TaggedStruct<"BlobStorageError", {
|
|
15
|
+
readonly operation: Schema.String;
|
|
16
|
+
readonly cause: Schema.optionalKey<Schema.Unknown>;
|
|
17
|
+
}>, import("effect/Cause").YieldableError>;
|
|
18
|
+
export declare class BlobStorageError extends BlobStorageError_base {
|
|
19
|
+
}
|
|
20
|
+
export interface TransferGrant {
|
|
21
|
+
readonly url: string;
|
|
22
|
+
readonly method: "GET" | "PUT";
|
|
23
|
+
readonly headers: Readonly<Record<string, string>>;
|
|
24
|
+
readonly expiresAt: string;
|
|
25
|
+
}
|
|
26
|
+
export interface StoredObject {
|
|
27
|
+
readonly size: number;
|
|
28
|
+
readonly contentType: string;
|
|
29
|
+
readonly sessionId: string;
|
|
30
|
+
}
|
|
31
|
+
declare const BlobProvider_base: Context.ServiceClass<BlobProvider, "@segmentstream-cloud/ontology/BlobProvider", {
|
|
32
|
+
readonly binding: BlobStorageBinding;
|
|
33
|
+
readonly close?: () => void;
|
|
34
|
+
readonly initiate: (input: {
|
|
35
|
+
key: string;
|
|
36
|
+
filename: string;
|
|
37
|
+
contentType: string;
|
|
38
|
+
sessionId: string;
|
|
39
|
+
}) => Effect.Effect<string, BlobStorageError>;
|
|
40
|
+
readonly signPart: (input: {
|
|
41
|
+
key: string;
|
|
42
|
+
uploadId: string;
|
|
43
|
+
partNumber: number;
|
|
44
|
+
size: number;
|
|
45
|
+
}) => Effect.Effect<TransferGrant, BlobStorageError>;
|
|
46
|
+
readonly parts: (input: {
|
|
47
|
+
key: string;
|
|
48
|
+
uploadId: string;
|
|
49
|
+
}) => Effect.Effect<ReadonlyArray<{
|
|
50
|
+
number: number;
|
|
51
|
+
size: number;
|
|
52
|
+
etag: string;
|
|
53
|
+
}>, BlobStorageError>;
|
|
54
|
+
readonly complete: (input: {
|
|
55
|
+
key: string;
|
|
56
|
+
uploadId: string;
|
|
57
|
+
parts: ReadonlyArray<{
|
|
58
|
+
number: number;
|
|
59
|
+
etag: string;
|
|
60
|
+
}>;
|
|
61
|
+
}) => Effect.Effect<void, BlobStorageError>;
|
|
62
|
+
readonly inspect: (key: string) => Effect.Effect<StoredObject | undefined, BlobStorageError>;
|
|
63
|
+
readonly readPrefix: (key: string) => Effect.Effect<Uint8Array, BlobStorageError>;
|
|
64
|
+
readonly download: (input: {
|
|
65
|
+
key: string;
|
|
66
|
+
filename: string;
|
|
67
|
+
contentType: string;
|
|
68
|
+
inline: boolean;
|
|
69
|
+
}) => Effect.Effect<TransferGrant, BlobStorageError>;
|
|
70
|
+
readonly abort: (input: {
|
|
71
|
+
key: string;
|
|
72
|
+
uploadId: string;
|
|
73
|
+
}) => Effect.Effect<void, BlobStorageError>;
|
|
74
|
+
readonly remove: (key: string) => Effect.Effect<void, BlobStorageError>;
|
|
75
|
+
readonly pending: (key: string) => Effect.Effect<ReadonlyArray<string>, BlobStorageError>;
|
|
76
|
+
}>;
|
|
77
|
+
/** Private provider capability. Actions never receive storage credentials or this service. */
|
|
78
|
+
export declare class BlobProvider extends BlobProvider_base {
|
|
79
|
+
}
|
|
80
|
+
export {};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Context, Schema } from "effect";
|
|
2
|
+
/** Validated deployment binding, independent of the action and browser contracts. */
|
|
3
|
+
export const BlobStorageBinding = Schema.Struct({
|
|
4
|
+
projectId: Schema.String,
|
|
5
|
+
branchId: Schema.String,
|
|
6
|
+
endpoint: Schema.String,
|
|
7
|
+
region: Schema.String,
|
|
8
|
+
bucket: Schema.Literal("ontology-files"),
|
|
9
|
+
accessKeyId: Schema.String,
|
|
10
|
+
secretAccessKey: Schema.String,
|
|
11
|
+
quotaBytes: Schema.Int.check(Schema.isGreaterThan(0)),
|
|
12
|
+
});
|
|
13
|
+
export class BlobStorageError extends Schema.TaggedError()("BlobStorageError", {
|
|
14
|
+
operation: Schema.String,
|
|
15
|
+
cause: Schema.optionalKey(Schema.Unknown),
|
|
16
|
+
}) {
|
|
17
|
+
}
|
|
18
|
+
/** Private provider capability. Actions never receive storage credentials or this service. */
|
|
19
|
+
export class BlobProvider extends Context.Service()("@segmentstream-cloud/ontology/BlobProvider") {
|
|
20
|
+
}
|