@open-pencil/cloud 0.17.0-beta.2 → 0.17.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/{ai-relay.mjs → services.mjs} +8 -8
- package/dist/bin/services.mjs.map +1 -0
- package/dist/client.d.ts +236 -46
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +649 -400
- package/dist/client.js.map +1 -1
- package/dist/contract.d.ts +2 -2
- package/dist/contract.js +2 -2
- package/dist/contract2.js +680 -443
- package/dist/contract2.js.map +1 -1
- package/dist/email.d.ts +9 -1
- package/dist/email.d.ts.map +1 -1
- package/dist/email.js +99 -58
- package/dist/email.js.map +1 -1
- package/dist/index.d.ts +469 -65
- package/dist/index.d.ts.map +1 -1
- package/dist/listen.mjs +505 -787
- package/dist/listen.mjs.map +1 -1
- package/dist/runtime-node.d.mts +3920 -2524
- package/dist/runtime-node.d.mts.map +1 -1
- package/dist/runtime-node.mjs +6 -5
- package/dist/runtime-node.mjs.map +1 -1
- package/dist/server.d.ts +3586 -1897
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +5216 -4272
- package/dist/server.js.map +1 -1
- package/dist/server.mjs +5179 -3623
- package/dist/server.mjs.map +1 -1
- package/dist/server2.mjs +101 -79
- package/dist/server2.mjs.map +1 -1
- package/package.json +7 -6
- package/src/admin/api/routes.ts +59 -0
- package/src/admin/audit/service.ts +25 -6
- package/src/admin/documents/service.ts +361 -0
- package/src/admin/errors.ts +9 -1
- package/src/admin/index.ts +6 -0
- package/src/admin/operations/service.ts +7 -0
- package/src/admin/reports/service.ts +147 -0
- package/src/admin/schema.ts +6 -0
- package/src/client/api.ts +36 -21
- package/src/client/auth/navigation.ts +6 -5
- package/src/client/index.ts +7 -7
- package/src/client/portal/account.ts +25 -0
- package/src/client/portal/administration.ts +56 -0
- package/src/client/{ai.ts → services.ts} +44 -27
- package/src/contract/account.ts +58 -2
- package/src/contract/admin/index.ts +67 -3
- package/src/contract/collaboration.ts +6 -1
- package/src/contract/discovery.ts +22 -4
- package/src/contract/documents.ts +27 -0
- package/src/contract/entitlements.ts +2 -2
- package/src/contract/index.ts +63 -37
- package/src/contract/relay.ts +6 -0
- package/src/contract/reports.ts +42 -0
- package/src/contract/services.ts +233 -0
- package/src/contract/sharing.ts +3 -9
- package/src/email/render.ts +26 -0
- package/src/email/schema.ts +11 -1
- package/src/email/templates/brand.ts +1 -1
- package/src/runtime/cloudflare/environment.ts +22 -11
- package/src/runtime/cloudflare/index.ts +1 -1
- package/src/runtime/cloudflare/logging.ts +15 -0
- package/src/runtime/cloudflare/maintenance.ts +31 -0
- package/src/runtime/cloudflare/relay/index.ts +62 -14
- package/src/runtime/cloudflare/worker.ts +39 -13
- package/src/runtime/node/application.ts +5 -3
- package/src/runtime/node/bootstrap.ts +15 -4
- package/src/runtime/node/cleanup.ts +16 -3
- package/src/runtime/node/entitlements/main.ts +14 -11
- package/src/runtime/node/index.ts +1 -1
- package/src/runtime/node/migrate.ts +5 -1
- package/src/runtime/node/relay.ts +8 -2
- package/src/runtime/node/request-id.ts +13 -0
- package/src/runtime/node/server.ts +39 -15
- package/src/runtime/node/{ai-relay → services}/main.ts +6 -6
- package/src/runtime/node/{ai-relay → services}/server.ts +18 -18
- package/src/runtime/postgres/database.ts +20 -1
- package/src/runtime/posthog/errors.ts +15 -31
- package/src/runtime/s3/objects.ts +24 -1
- package/src/runtime/site/server.ts +32 -5
- package/src/server/account/actions.ts +5 -0
- package/src/server/account/deletion.ts +302 -0
- package/src/server/account/routes.ts +47 -3
- package/src/server/account/terms.ts +42 -0
- package/src/server/api/router.ts +9 -5
- package/src/server/api/types.ts +2 -0
- package/src/server/app.ts +90 -16
- package/src/server/auth/access-control.ts +63 -0
- package/src/server/auth/adapter.ts +11 -0
- package/src/server/auth/endpoints.ts +5 -1
- package/src/server/auth/factory.ts +30 -3
- package/src/server/auth/index.ts +10 -1
- package/src/server/cleanup/configured.ts +12 -5
- package/src/server/cleanup/documents.ts +165 -71
- package/src/server/cleanup/revisions.ts +16 -5
- package/src/server/cleanup/uploads.ts +8 -2
- package/src/server/cleanup/worker.ts +5 -2
- package/src/server/collaboration/relay/connection.ts +4 -3
- package/src/server/collaboration/relay/documents.ts +13 -10
- package/src/server/collaboration/relay/hub.ts +97 -25
- package/src/server/collaboration/relay/log.ts +5 -2
- package/src/server/collaboration/service.ts +27 -31
- package/src/server/config/deployment.ts +24 -4
- package/src/server/config/public.ts +5 -4
- package/src/server/config/schema.ts +82 -16
- package/src/server/db/expressions.ts +15 -1
- package/src/server/db/migrate.ts +34 -116
- package/src/server/db/migrations/001_accounts.ts +79 -0
- package/src/server/db/migrations/002_workspaces.ts +48 -0
- package/src/server/db/migrations/003_storage.ts +44 -0
- package/src/server/db/migrations/004_documents.ts +193 -0
- package/src/server/db/migrations/005_sharing.ts +95 -0
- package/src/server/db/migrations/006_operations.ts +110 -0
- package/src/server/db/migrations/007_trash.ts +33 -0
- package/src/server/db/migrations/008_terms.ts +18 -0
- package/src/server/db/migrations/schema.ts +71 -0
- package/src/server/db/tables.ts +52 -3
- package/src/server/documents/access.ts +66 -55
- package/src/server/documents/errors.ts +7 -0
- package/src/server/documents/images.ts +2 -1
- package/src/server/documents/limits.ts +9 -9
- package/src/server/documents/purge.ts +75 -0
- package/src/server/documents/repository.ts +2 -0
- package/src/server/documents/routes.ts +26 -0
- package/src/server/documents/service.ts +18 -4
- package/src/server/documents/summary.ts +1 -0
- package/src/server/documents/trash.ts +203 -0
- package/src/server/documents/versions/checkpoint.ts +2 -1
- package/src/server/documents/versions/history.ts +11 -7
- package/src/server/documents/versions/service.ts +13 -4
- package/src/server/errors/index.ts +59 -16
- package/src/server/errors/posthog.ts +80 -0
- package/src/server/index.ts +46 -17
- package/src/server/internal/index.ts +3 -3
- package/src/server/internal/routes.ts +7 -10
- package/src/server/internal/schema.ts +1 -1
- package/src/server/internal/service.ts +17 -23
- package/src/server/invitations/budget.ts +66 -0
- package/src/server/invitations/index.ts +1 -0
- package/src/server/logging.ts +76 -0
- package/src/server/pictures/service.ts +2 -1
- package/src/server/policy/database.ts +39 -25
- package/src/server/policy/index.ts +2 -1
- package/src/server/policy/keys.ts +5 -2
- package/src/server/policy/policy.ts +1 -0
- package/src/server/policy/schema.ts +2 -2
- package/src/server/policy/service.ts +12 -9
- package/src/server/policy/static.ts +5 -5
- package/src/server/policy/workspace.ts +41 -0
- package/src/server/quota/reconcile.ts +2 -0
- package/src/server/quota/storage.ts +69 -31
- package/src/server/rate-limit/cleanup.ts +17 -5
- package/src/server/rate-limit/middleware.ts +2 -2
- package/src/server/reports/index.ts +8 -0
- package/src/server/reports/routes.ts +37 -0
- package/src/server/reports/service.ts +123 -0
- package/src/server/services/errors.ts +7 -0
- package/src/server/services/index.ts +8 -0
- package/src/server/{ai → services}/keys.ts +11 -11
- package/src/server/services/routes.ts +47 -0
- package/src/server/{ai/service.ts → services/tokens.ts} +23 -20
- package/src/server/sharing/capabilities/service.ts +68 -22
- package/src/server/sharing/grants/service.ts +11 -18
- package/src/server/sharing/http.ts +4 -0
- package/src/server/sharing/invitations/service.ts +21 -11
- package/src/server/sharing/policy.ts +9 -14
- package/src/server/sharing/routes.ts +0 -16
- package/src/server/sharing/service.ts +1 -17
- package/src/server/validation.ts +1 -0
- package/src/server/workspaces/index.ts +1 -0
- package/src/server/workspaces/members.ts +89 -38
- package/src/server/workspaces/repository.ts +2 -1
- package/src/server/workspaces/routes.ts +4 -0
- package/src/{ai-relay → services}/app.ts +48 -44
- package/src/{ai-relay → services}/config.ts +25 -19
- package/src/services/db/index.ts +10 -0
- package/src/services/db/migrate.ts +37 -0
- package/src/{ai-relay/db/migrations/001_usage.ts → services/db/migrations/001_baseline.ts} +18 -7
- package/src/services/db/schema.ts +22 -0
- package/src/services/index.ts +18 -0
- package/src/{ai-relay → services}/metering.ts +2 -2
- package/src/{ai-relay → services}/usage.ts +43 -19
- package/dist/bin/ai-relay.mjs.map +0 -1
- package/src/ai-relay/db/index.ts +0 -10
- package/src/ai-relay/db/migrate.ts +0 -32
- package/src/ai-relay/db/schema.ts +0 -20
- package/src/ai-relay/index.ts +0 -18
- package/src/contract/ai.ts +0 -150
- package/src/server/ai/errors.ts +0 -7
- package/src/server/ai/index.ts +0 -4
- package/src/server/ai/routes.ts +0 -43
- package/src/server/auth/workspace-access.ts +0 -23
- package/src/server/db/migrations/001_foundation.ts +0 -164
- package/src/server/db/migrations/002_upload_cleanup_claims.ts +0 -34
- package/src/server/db/migrations/003_document_cleanup_claims.ts +0 -23
- package/src/server/db/migrations/004_document_sharing.ts +0 -82
- package/src/server/db/migrations/005_document_collaboration_epoch.ts +0 -13
- package/src/server/db/migrations/006_invitation_continuation.ts +0 -27
- package/src/server/db/migrations/007_collaboration_state.ts +0 -25
- package/src/server/db/migrations/008_storage_reservations.ts +0 -37
- package/src/server/db/migrations/009_workspace_entitlements.ts +0 -22
- package/src/server/db/migrations/010_upload_finalization.ts +0 -22
- package/src/server/db/migrations/011_upload_finalization_lease.ts +0 -12
- package/src/server/db/migrations/012_transactional_email.ts +0 -48
- package/src/server/db/migrations/013_admin_enrollment.ts +0 -54
- package/src/server/db/migrations/014_enrollment_email_kinds.ts +0 -30
- package/src/server/db/migrations/015_cloud_rate_limit.ts +0 -22
- package/src/server/db/migrations/016_auth_schema.ts +0 -16
- package/src/server/db/migrations/017_authentication_email_kinds.ts +0 -32
- package/src/server/db/migrations/018_authentication_mfa.ts +0 -78
- package/src/server/db/migrations/019_better_auth_account_issuer.ts +0 -42
- package/src/server/db/migrations/020_workspace_invitations.ts +0 -57
- package/src/server/db/migrations/021_document_versions.ts +0 -66
- package/src/server/db/migrations/022_entitlement_external_version.ts +0 -17
- package/src/server/db/migrations/023_document_images.ts +0 -61
- package/src/server/db/migrations/024_account_activity.ts +0 -66
- package/src/server/db/migrations/025_table_names.ts +0 -58
- package/src/server/db/migrations/026_user_picture.ts +0 -30
- package/src/server/db/migrations/027_document_kind.ts +0 -19
- package/src/server/db/migrations/028_collaboration_updates.ts +0 -84
- package/src/server/db/migrations/helpers.ts +0 -78
- /package/dist/bin/{ai-relay.d.mts → services.d.mts} +0 -0
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
import type { CloudDatabase } from '#cloud/server/db'
|
|
2
|
+
import { reportCloudError, type CloudErrorReporter } from '#cloud/server/errors'
|
|
2
3
|
import type { ObjectStore } from '#cloud/server/objects'
|
|
3
|
-
import
|
|
4
|
+
import { omit } from 'es-toolkit'
|
|
5
|
+
import type { ExpressionBuilder, Kysely, Transaction } from 'kysely'
|
|
4
6
|
|
|
5
7
|
import { unreferencedObject } from './objects'
|
|
6
8
|
|
|
7
9
|
export type DocumentCleanupOptions = {
|
|
8
10
|
batchSize: number
|
|
9
11
|
leaseDurationMs: number
|
|
12
|
+
/** How long a document stays in Trash before cleanup destroys it. */
|
|
10
13
|
retentionMs: number
|
|
14
|
+
/** How long a document deleted forever keeps its files before cleanup destroys it; 0 is at once. */
|
|
15
|
+
purgeGraceMs: number
|
|
11
16
|
now?: Date
|
|
12
17
|
}
|
|
13
18
|
|
|
@@ -21,12 +26,27 @@ export type DocumentCleanupResult = {
|
|
|
21
26
|
|
|
22
27
|
export type DocumentCleanupService = {
|
|
23
28
|
cleanupDeletedDocuments(options: DocumentCleanupOptions): Promise<DocumentCleanupResult>
|
|
29
|
+
/**
|
|
30
|
+
* Destroys one document deleted forever now, as cleanup would once its grace passed. False when
|
|
31
|
+
* it isn't deleted forever, is being cleaned already, or has a save in flight; it then waits for
|
|
32
|
+
* the next pass. Throws, with the failure as its cause, when destroying it failed; the failure
|
|
33
|
+
* is reported, and the next pass tries again.
|
|
34
|
+
*/
|
|
35
|
+
destroyPurgedDocument(
|
|
36
|
+
documentId: string,
|
|
37
|
+
options: { leaseDurationMs: number; now?: Date }
|
|
38
|
+
): Promise<boolean>
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type DocumentCleanupServiceOptions = {
|
|
42
|
+
/** Where each document cleanup couldn't destroy is reported; a pass carries on with the rest. */
|
|
43
|
+
reportError?: CloudErrorReporter
|
|
24
44
|
}
|
|
25
45
|
|
|
26
46
|
type ClaimedDocument = {
|
|
27
47
|
id: string
|
|
28
48
|
workspaceId: string
|
|
29
|
-
|
|
49
|
+
purgedBytes: number
|
|
30
50
|
}
|
|
31
51
|
|
|
32
52
|
type StoredRevisionObject = {
|
|
@@ -35,70 +55,124 @@ type StoredRevisionObject = {
|
|
|
35
55
|
byteSize: number
|
|
36
56
|
}
|
|
37
57
|
|
|
58
|
+
/**
|
|
59
|
+
* Which deleted documents a pass claims: those in Trash deleted by `deletedBefore`, and those
|
|
60
|
+
* deleted forever by `purgedBefore`, or only one of them.
|
|
61
|
+
*/
|
|
62
|
+
type ClaimFilter = {
|
|
63
|
+
batchSize: number
|
|
64
|
+
deletedBefore: Date
|
|
65
|
+
purgedBefore: Date
|
|
66
|
+
staleBefore: Date
|
|
67
|
+
now: Date
|
|
68
|
+
claimId: string
|
|
69
|
+
documentId?: string
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** A deleted document whose time is up: past retention in Trash, or past grace deleted forever. */
|
|
73
|
+
function due(deletedBefore: Date, purgedBefore: Date) {
|
|
74
|
+
return (expression: ExpressionBuilder<CloudDatabase, 'document'>) =>
|
|
75
|
+
expression.or([
|
|
76
|
+
expression.and([
|
|
77
|
+
expression('purgedAt', 'is', null),
|
|
78
|
+
expression('deletedAt', 'is not', null),
|
|
79
|
+
expression('deletedAt', '<=', deletedBefore)
|
|
80
|
+
]),
|
|
81
|
+
expression('purgedAt', '<=', purgedBefore)
|
|
82
|
+
])
|
|
83
|
+
}
|
|
84
|
+
|
|
38
85
|
export function createDocumentCleanupService(
|
|
39
86
|
database: Kysely<CloudDatabase>,
|
|
40
|
-
objects: ObjectStore
|
|
87
|
+
objects: ObjectStore,
|
|
88
|
+
options: DocumentCleanupServiceOptions = {}
|
|
41
89
|
): DocumentCleanupService {
|
|
90
|
+
async function clean(
|
|
91
|
+
filter: ClaimFilter
|
|
92
|
+
): Promise<DocumentCleanupResult & { failures: unknown[] }> {
|
|
93
|
+
const documents = await claimDocuments(database, filter)
|
|
94
|
+
const result = {
|
|
95
|
+
claimed: documents.length,
|
|
96
|
+
cleaned: 0,
|
|
97
|
+
failed: 0,
|
|
98
|
+
objectsDeleted: 0,
|
|
99
|
+
bytesReclaimed: 0,
|
|
100
|
+
failures: [] as unknown[]
|
|
101
|
+
}
|
|
102
|
+
for (const document of documents) {
|
|
103
|
+
try {
|
|
104
|
+
const revisionObjects = await documentObjects(database, document.id)
|
|
105
|
+
for (const object of revisionObjects) await objects.delete(object.objectKey)
|
|
106
|
+
const reclaimed = await removeDocumentMetadata(database, document, revisionObjects, filter)
|
|
107
|
+
if (!reclaimed) continue
|
|
108
|
+
result.cleaned++
|
|
109
|
+
result.objectsDeleted += reclaimed.objectsDeleted
|
|
110
|
+
result.bytesReclaimed += reclaimed.bytesReclaimed
|
|
111
|
+
} catch (error) {
|
|
112
|
+
result.failed++
|
|
113
|
+
result.failures.push(error)
|
|
114
|
+
// Only the document's ID: its name and the people on it stay out of error tracking.
|
|
115
|
+
await reportCloudError(
|
|
116
|
+
options.reportError,
|
|
117
|
+
`Cleaning up document ${document.id} failed`,
|
|
118
|
+
error,
|
|
119
|
+
{ source: 'cleanup' }
|
|
120
|
+
)
|
|
121
|
+
// Out of Trash already, a document whose files are partly gone is never restorable; the
|
|
122
|
+
// next pass tries again.
|
|
123
|
+
await releaseClaim(database, document.id, filter.claimId)
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return result
|
|
127
|
+
}
|
|
128
|
+
|
|
42
129
|
return {
|
|
43
|
-
async cleanupDeletedDocuments(
|
|
44
|
-
const now =
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
options.batchSize,
|
|
51
|
-
retentionBefore,
|
|
52
|
-
staleBefore,
|
|
130
|
+
async cleanupDeletedDocuments(input) {
|
|
131
|
+
const now = input.now ?? new Date()
|
|
132
|
+
const result = await clean({
|
|
133
|
+
batchSize: input.batchSize,
|
|
134
|
+
deletedBefore: new Date(now.getTime() - input.retentionMs),
|
|
135
|
+
purgedBefore: new Date(now.getTime() - input.purgeGraceMs),
|
|
136
|
+
staleBefore: new Date(now.getTime() - input.leaseDurationMs),
|
|
53
137
|
now,
|
|
54
|
-
claimId
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
result.cleaned++
|
|
76
|
-
result.objectsDeleted += reclaimed.objectsDeleted
|
|
77
|
-
result.bytesReclaimed += reclaimed.bytesReclaimed
|
|
78
|
-
} catch {
|
|
79
|
-
result.failed++
|
|
80
|
-
await releaseClaim(database, document.id, claimId)
|
|
81
|
-
}
|
|
138
|
+
claimId: crypto.randomUUID()
|
|
139
|
+
})
|
|
140
|
+
return omit(result, ['failures'])
|
|
141
|
+
},
|
|
142
|
+
|
|
143
|
+
async destroyPurgedDocument(documentId, input) {
|
|
144
|
+
const now = input.now ?? new Date()
|
|
145
|
+
const result = await clean({
|
|
146
|
+
batchSize: 1,
|
|
147
|
+
// Only a document deleted forever, never one still in Trash.
|
|
148
|
+
deletedBefore: new Date(0),
|
|
149
|
+
purgedBefore: now,
|
|
150
|
+
staleBefore: new Date(now.getTime() - input.leaseDurationMs),
|
|
151
|
+
now,
|
|
152
|
+
claimId: crypto.randomUUID(),
|
|
153
|
+
documentId
|
|
154
|
+
})
|
|
155
|
+
if (result.failures.length > 0) {
|
|
156
|
+
throw new Error(`Document ${documentId} could not be destroyed`, {
|
|
157
|
+
cause: result.failures[0]
|
|
158
|
+
})
|
|
82
159
|
}
|
|
83
|
-
return result
|
|
160
|
+
return result.cleaned > 0
|
|
84
161
|
}
|
|
85
162
|
}
|
|
86
163
|
}
|
|
87
164
|
|
|
88
165
|
async function claimDocuments(
|
|
89
166
|
database: Kysely<CloudDatabase>,
|
|
90
|
-
batchSize:
|
|
91
|
-
retentionBefore: Date,
|
|
92
|
-
staleBefore: Date,
|
|
93
|
-
now: Date,
|
|
94
|
-
claimId: string
|
|
167
|
+
{ batchSize, deletedBefore, purgedBefore, staleBefore, now, claimId, documentId }: ClaimFilter
|
|
95
168
|
): Promise<ClaimedDocument[]> {
|
|
96
169
|
return database.transaction().execute(async (transaction) => {
|
|
97
|
-
|
|
170
|
+
let candidates = transaction
|
|
98
171
|
.selectFrom('document')
|
|
99
|
-
.select(['id', 'workspaceId', '
|
|
100
|
-
.where(
|
|
101
|
-
|
|
172
|
+
.select(['id', 'workspaceId', 'purgedBytes'])
|
|
173
|
+
.where(due(deletedBefore, purgedBefore))
|
|
174
|
+
if (documentId !== undefined) candidates = candidates.where('id', '=', documentId)
|
|
175
|
+
const rows = await candidates
|
|
102
176
|
.where((expression) =>
|
|
103
177
|
expression.or([
|
|
104
178
|
expression('cleanupClaimId', 'is', null),
|
|
@@ -121,25 +195,23 @@ async function claimDocuments(
|
|
|
121
195
|
.forUpdate()
|
|
122
196
|
.skipLocked()
|
|
123
197
|
.execute()
|
|
124
|
-
|
|
125
|
-
(document): document is ClaimedDocument => document.deletedAt !== null
|
|
126
|
-
)
|
|
127
|
-
if (claimed.length === 0) return []
|
|
198
|
+
if (rows.length === 0) return []
|
|
128
199
|
await transaction
|
|
129
200
|
.updateTable('document')
|
|
130
201
|
.set({ cleanupClaimId: claimId, cleanupClaimedAt: now })
|
|
131
202
|
.where(
|
|
132
203
|
'id',
|
|
133
204
|
'in',
|
|
134
|
-
|
|
205
|
+
rows.map((document) => document.id)
|
|
135
206
|
)
|
|
136
207
|
.execute()
|
|
137
|
-
return
|
|
208
|
+
return rows.map((row) => ({ ...row, purgedBytes: Number(row.purgedBytes) }))
|
|
138
209
|
})
|
|
139
210
|
}
|
|
140
211
|
|
|
141
|
-
|
|
142
|
-
|
|
212
|
+
/** The files only this document's revisions hold, which destroying it frees. */
|
|
213
|
+
export async function documentObjects(
|
|
214
|
+
database: Kysely<CloudDatabase> | Transaction<CloudDatabase>,
|
|
143
215
|
documentId: string
|
|
144
216
|
): Promise<StoredRevisionObject[]> {
|
|
145
217
|
const rows = await database
|
|
@@ -166,23 +238,18 @@ async function removeDocumentMetadata(
|
|
|
166
238
|
database: Kysely<CloudDatabase>,
|
|
167
239
|
document: ClaimedDocument,
|
|
168
240
|
objects: StoredRevisionObject[],
|
|
169
|
-
|
|
170
|
-
claimId: string
|
|
241
|
+
{ deletedBefore, purgedBefore, claimId }: ClaimFilter
|
|
171
242
|
): Promise<{ objectsDeleted: number; bytesReclaimed: number } | null> {
|
|
172
243
|
return database.transaction().execute(async (transaction) => {
|
|
173
244
|
const locked = await transaction
|
|
174
245
|
.selectFrom('document')
|
|
175
|
-
.select(
|
|
246
|
+
.select('id')
|
|
176
247
|
.where('id', '=', document.id)
|
|
248
|
+
.where(due(deletedBefore, purgedBefore))
|
|
249
|
+
.where('cleanupClaimId', '=', claimId)
|
|
177
250
|
.forUpdate()
|
|
178
251
|
.executeTakeFirst()
|
|
179
|
-
if (
|
|
180
|
-
!locked?.deletedAt ||
|
|
181
|
-
locked.deletedAt > retentionBefore ||
|
|
182
|
-
locked.cleanupClaimId !== claimId
|
|
183
|
-
) {
|
|
184
|
-
return null
|
|
185
|
-
}
|
|
252
|
+
if (!locked) return null
|
|
186
253
|
await transaction
|
|
187
254
|
.updateTable('document')
|
|
188
255
|
.set({ currentRevisionId: null })
|
|
@@ -202,17 +269,44 @@ async function removeDocumentMetadata(
|
|
|
202
269
|
bytesReclaimed += object.byteSize
|
|
203
270
|
}
|
|
204
271
|
}
|
|
205
|
-
|
|
272
|
+
// Deleting it forever already took its files off the workspace's usage.
|
|
273
|
+
const released = Math.max(0, bytesReclaimed - document.purgedBytes)
|
|
274
|
+
if (released > 0) {
|
|
206
275
|
await transaction
|
|
207
276
|
.updateTable('workspaceStorageUsage')
|
|
208
277
|
.set({
|
|
209
|
-
committedBytes: (expression) => expression('committedBytes', '-',
|
|
278
|
+
committedBytes: (expression) => expression('committedBytes', '-', released),
|
|
210
279
|
updatedAt: new Date()
|
|
211
280
|
})
|
|
212
281
|
.where('workspaceId', '=', document.workspaceId)
|
|
213
282
|
.execute()
|
|
214
283
|
}
|
|
215
284
|
await transaction.deleteFrom('document').where('id', '=', document.id).execute()
|
|
285
|
+
// A workspace whose account was deleted waits, with nobody in it, for its last document.
|
|
286
|
+
await transaction
|
|
287
|
+
.deleteFrom('workspace')
|
|
288
|
+
.where('id', '=', document.workspaceId)
|
|
289
|
+
.where((expression) =>
|
|
290
|
+
expression.and([
|
|
291
|
+
expression.not(
|
|
292
|
+
expression.exists(
|
|
293
|
+
expression
|
|
294
|
+
.selectFrom('workspaceMember')
|
|
295
|
+
.select('workspaceMember.userId')
|
|
296
|
+
.whereRef('workspaceMember.workspaceId', '=', 'workspace.id')
|
|
297
|
+
)
|
|
298
|
+
),
|
|
299
|
+
expression.not(
|
|
300
|
+
expression.exists(
|
|
301
|
+
expression
|
|
302
|
+
.selectFrom('document')
|
|
303
|
+
.select('document.id')
|
|
304
|
+
.whereRef('document.workspaceId', '=', 'workspace.id')
|
|
305
|
+
)
|
|
306
|
+
)
|
|
307
|
+
])
|
|
308
|
+
)
|
|
309
|
+
.execute()
|
|
216
310
|
return { objectsDeleted, bytesReclaimed }
|
|
217
311
|
})
|
|
218
312
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { CloudDatabase } from '#cloud/server/db'
|
|
2
2
|
import { IMAGE_RELEASE_GRACE_MS, releaseUnusedImages } from '#cloud/server/documents/images'
|
|
3
3
|
import { historyStart, LIMITED_HISTORY_DAYS } from '#cloud/server/documents/versions/history'
|
|
4
|
+
import { reportCloudError, type CloudErrorReporter } from '#cloud/server/errors'
|
|
4
5
|
import type { ObjectStore } from '#cloud/server/objects'
|
|
5
6
|
import type { CloudPolicyContext } from '#cloud/server/policy/keys'
|
|
6
7
|
import type { CloudPolicy } from '#cloud/server/policy/policy'
|
|
@@ -47,9 +48,16 @@ export type RevisionCompactionService = {
|
|
|
47
48
|
export function createRevisionCompactionService(
|
|
48
49
|
database: Kysely<CloudDatabase>,
|
|
49
50
|
objects: ObjectStore,
|
|
50
|
-
options: {
|
|
51
|
+
options: {
|
|
52
|
+
policy?: CloudPolicy
|
|
53
|
+
deploymentMode?: CloudPolicyContext['deploymentMode']
|
|
54
|
+
/** Where each step compaction couldn't finish is reported; a pass carries on with the rest. */
|
|
55
|
+
reportError?: CloudErrorReporter
|
|
56
|
+
} = {}
|
|
51
57
|
): RevisionCompactionService {
|
|
52
58
|
const deploymentMode = options.deploymentMode ?? 'self-hosted'
|
|
59
|
+
const report = (message: string, error: unknown) =>
|
|
60
|
+
reportCloudError(options.reportError, message, error, { source: 'cleanup' })
|
|
53
61
|
async function pruneVersions(now: Date, batchSize: number): Promise<number> {
|
|
54
62
|
if (!options.policy) return 0
|
|
55
63
|
const limitedBefore = new Date(now.getTime() - LIMITED_HISTORY_DAYS * 24 * 60 * 60_000)
|
|
@@ -62,7 +70,7 @@ export function createRevisionCompactionService(
|
|
|
62
70
|
.execute()
|
|
63
71
|
let pruned = 0
|
|
64
72
|
for (const { workspaceId } of workspaces) {
|
|
65
|
-
const start = await historyStart(options.policy, deploymentMode, workspaceId, now)
|
|
73
|
+
const start = await historyStart(database, options.policy, deploymentMode, workspaceId, now)
|
|
66
74
|
if (!start) continue
|
|
67
75
|
const deleted = await database
|
|
68
76
|
.deleteFrom('documentVersion')
|
|
@@ -181,8 +189,9 @@ export function createRevisionCompactionService(
|
|
|
181
189
|
.where(unreferencedObject)
|
|
182
190
|
.execute()
|
|
183
191
|
result.objectsDeleted++
|
|
184
|
-
} catch {
|
|
192
|
+
} catch (error) {
|
|
185
193
|
result.failed++
|
|
194
|
+
await report(`Deleting file ${object.id} failed`, error)
|
|
186
195
|
}
|
|
187
196
|
}
|
|
188
197
|
}
|
|
@@ -214,8 +223,9 @@ export function createRevisionCompactionService(
|
|
|
214
223
|
capacity -= compacted.revisions
|
|
215
224
|
result.revisionsDeleted += compacted.revisions
|
|
216
225
|
result.bytesReclaimed += compacted.bytes
|
|
217
|
-
} catch {
|
|
226
|
+
} catch (error) {
|
|
218
227
|
result.failed++
|
|
228
|
+
await report(`Compacting document ${documentId} failed`, error)
|
|
219
229
|
}
|
|
220
230
|
}
|
|
221
231
|
// Compaction just freed the images of the saves it deleted; the release lets their grace pass.
|
|
@@ -228,8 +238,9 @@ export function createRevisionCompactionService(
|
|
|
228
238
|
)
|
|
229
239
|
result.imagesReleased = images.released
|
|
230
240
|
result.bytesReclaimed += images.bytes
|
|
231
|
-
} catch {
|
|
241
|
+
} catch (error) {
|
|
232
242
|
result.failed++
|
|
243
|
+
await report('Releasing unused images failed', error)
|
|
233
244
|
}
|
|
234
245
|
await sweepObjects(now, input.batchSize, result)
|
|
235
246
|
return result
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { CloudDatabase } from '#cloud/server/db'
|
|
2
|
+
import { reportCloudError, type CloudErrorReporter } from '#cloud/server/errors'
|
|
2
3
|
import type { ObjectStore } from '#cloud/server/objects'
|
|
3
4
|
import { createStorageQuotaService } from '#cloud/server/quota'
|
|
4
5
|
import type { Kysely } from 'kysely'
|
|
@@ -27,7 +28,9 @@ type ClaimedUpload = {
|
|
|
27
28
|
|
|
28
29
|
export function createUploadCleanupService(
|
|
29
30
|
database: Kysely<CloudDatabase>,
|
|
30
|
-
objects: ObjectStore
|
|
31
|
+
objects: ObjectStore,
|
|
32
|
+
/** Where each upload cleanup couldn't remove is reported; a pass carries on with the rest. */
|
|
33
|
+
reportError?: CloudErrorReporter
|
|
31
34
|
): UploadCleanupService {
|
|
32
35
|
const quota = createStorageQuotaService(database)
|
|
33
36
|
return {
|
|
@@ -96,8 +99,11 @@ export function createUploadCleanupService(
|
|
|
96
99
|
await quota.release(upload.id)
|
|
97
100
|
cleaned++
|
|
98
101
|
}
|
|
99
|
-
} catch {
|
|
102
|
+
} catch (error) {
|
|
100
103
|
failed++
|
|
104
|
+
await reportCloudError(reportError, `Cleaning up upload ${upload.id} failed`, error, {
|
|
105
|
+
source: 'cleanup'
|
|
106
|
+
})
|
|
101
107
|
await database
|
|
102
108
|
.updateTable('upload')
|
|
103
109
|
.set({
|
|
@@ -10,6 +10,8 @@ import type { UploadCleanupResult, UploadCleanupService } from './uploads'
|
|
|
10
10
|
export type CleanupWorkerOptions = {
|
|
11
11
|
batchSize: number
|
|
12
12
|
documentRetentionMs: number
|
|
13
|
+
/** How long a document deleted forever keeps its files; 0 destroys it at once. */
|
|
14
|
+
documentPurgeGraceMs: number
|
|
13
15
|
intervalMs: number
|
|
14
16
|
leaseDurationMs: number
|
|
15
17
|
signal?: AbortSignal
|
|
@@ -37,7 +39,7 @@ export type CleanupWorker = {
|
|
|
37
39
|
|
|
38
40
|
export type CleanupPassOptions = Pick<
|
|
39
41
|
CleanupWorkerOptions,
|
|
40
|
-
'batchSize' | 'documentRetentionMs' | 'leaseDurationMs'
|
|
42
|
+
'batchSize' | 'documentRetentionMs' | 'documentPurgeGraceMs' | 'leaseDurationMs'
|
|
41
43
|
>
|
|
42
44
|
|
|
43
45
|
/** One cleanup pass, for runtimes that schedule passes themselves. */
|
|
@@ -52,7 +54,8 @@ export async function runCleanup(
|
|
|
52
54
|
const documents = await cleanup.documents.cleanupDeletedDocuments({
|
|
53
55
|
batchSize: options.batchSize,
|
|
54
56
|
leaseDurationMs: options.leaseDurationMs,
|
|
55
|
-
retentionMs: options.documentRetentionMs
|
|
57
|
+
retentionMs: options.documentRetentionMs,
|
|
58
|
+
purgeGraceMs: options.documentPurgeGraceMs
|
|
56
59
|
})
|
|
57
60
|
const revisions = cleanup.revisions
|
|
58
61
|
? await cleanup.revisions.compactRevisions({
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createRelayClientAssembler,
|
|
3
3
|
decodeRelayClientFrame,
|
|
4
|
-
MAX_COLLAB_MESSAGE_BYTES,
|
|
5
4
|
RELAY_CLOSE,
|
|
6
5
|
type RelayClientFrame
|
|
7
6
|
} from '#cloud/contract'
|
|
@@ -43,6 +42,8 @@ export type ConnectionRelay = {
|
|
|
43
42
|
/** The one document the relay serves, when it serves one. */
|
|
44
43
|
documentId?: string
|
|
45
44
|
maximumMessageBytes: number
|
|
45
|
+
/** The most an editor's update may carry once its fragments are joined. */
|
|
46
|
+
maximumUpdateBytes: number
|
|
46
47
|
authTimeoutMs: number
|
|
47
48
|
/** Admits a newcomer whose ticket verified, or turns it away and answers null. */
|
|
48
49
|
admit(socket: RelaySocket, authorization: RelayAuthorization): Promise<RelayPeer | null>
|
|
@@ -73,7 +74,7 @@ export function openRelayConnection(
|
|
|
73
74
|
// Only an editor's writes can be large: a viewer's frames, and a peer's before it is
|
|
74
75
|
// admitted, fit in one message, so nobody else can make the relay hold more.
|
|
75
76
|
const assembler = createRelayClientAssembler(() =>
|
|
76
|
-
peer?.authorization.permission === 'edit' ?
|
|
77
|
+
peer?.authorization.permission === 'edit' ? relay.maximumUpdateBytes : relay.maximumMessageBytes
|
|
77
78
|
)
|
|
78
79
|
const authTimer = resumed
|
|
79
80
|
? undefined
|
|
@@ -137,7 +138,7 @@ export function openRelayConnection(
|
|
|
137
138
|
state.busy = true
|
|
138
139
|
void authenticate(frame.token)
|
|
139
140
|
.catch((error: unknown) => {
|
|
140
|
-
relay.report('Admitting a collaboration peer failed
|
|
141
|
+
relay.report('Admitting a collaboration peer failed', error)
|
|
141
142
|
socket.close(RELAY_CLOSE.protocol, 'Could not open the collaboration room')
|
|
142
143
|
})
|
|
143
144
|
.finally(() => {
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import type { CloudDatabase } from '#cloud/server/db'
|
|
2
2
|
import { CLOUD_FEATURE_KEYS } from '#cloud/server/policy/keys'
|
|
3
3
|
import type { CloudPolicy } from '#cloud/server/policy/policy'
|
|
4
|
+
import { workspacePolicyContext } from '#cloud/server/policy/workspace'
|
|
4
5
|
import type { Kysely } from 'kysely'
|
|
5
6
|
|
|
6
7
|
import type { CollaborationRelayOptions } from './hub'
|
|
7
8
|
|
|
8
|
-
/**
|
|
9
|
+
/**
|
|
10
|
+
* The relay's lookups of a document: its participant limit under `policy`, none while it is taken
|
|
11
|
+
* down, and its epoch.
|
|
12
|
+
*/
|
|
9
13
|
export function relayDocumentLookups(
|
|
10
14
|
database: Kysely<CloudDatabase>,
|
|
11
15
|
policy: CloudPolicy,
|
|
@@ -13,18 +17,17 @@ export function relayDocumentLookups(
|
|
|
13
17
|
): Pick<CollaborationRelayOptions, 'maximumParticipants' | 'currentRoomEpoch'> {
|
|
14
18
|
return {
|
|
15
19
|
async maximumParticipants(documentId) {
|
|
16
|
-
|
|
20
|
+
// A taken-down document's room takes nobody, even with a ticket issued before.
|
|
21
|
+
const open = await database
|
|
17
22
|
.selectFrom('document')
|
|
18
|
-
.select('
|
|
23
|
+
.select('id')
|
|
19
24
|
.where('id', '=', documentId)
|
|
25
|
+
.where('disabledAt', 'is', null)
|
|
20
26
|
.executeTakeFirst()
|
|
21
|
-
if (!
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
documentId,
|
|
26
|
-
deploymentMode
|
|
27
|
-
})
|
|
27
|
+
if (!open) return 0
|
|
28
|
+
const context = await workspacePolicyContext(database, { documentId }, { deploymentMode })
|
|
29
|
+
if (!context) return 0
|
|
30
|
+
const limit = await policy.number(CLOUD_FEATURE_KEYS.maximumParticipants, -1, context)
|
|
28
31
|
return limit < 0 ? null : limit
|
|
29
32
|
},
|
|
30
33
|
async currentRoomEpoch(documentId) {
|