@open-pencil/cloud 0.0.0-stage → 0.17.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/LICENSE-MIT +21 -0
- package/NOTICE +7 -0
- package/README.md +37 -2
- package/dist/bin/ai-relay.d.mts +1 -0
- package/dist/bin/ai-relay.mjs +31 -0
- package/dist/bin/ai-relay.mjs.map +1 -0
- package/dist/bin/server.d.mts +1 -0
- package/dist/bin/server.mjs +29 -0
- package/dist/bin/server.mjs.map +1 -0
- package/dist/client.d.ts +1520 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +1917 -0
- package/dist/client.js.map +1 -0
- package/dist/contract.d.ts +2 -0
- package/dist/contract.js +2 -0
- package/dist/contract2.js +1453 -0
- package/dist/contract2.js.map +1 -0
- package/dist/email.d.ts +169 -0
- package/dist/email.d.ts.map +1 -0
- package/dist/email.js +670 -0
- package/dist/email.js.map +1 -0
- package/dist/index.d.ts +1755 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/listen.mjs +1881 -0
- package/dist/listen.mjs.map +1 -0
- package/dist/runtime-node.d.mts +10040 -0
- package/dist/runtime-node.d.mts.map +1 -0
- package/dist/runtime-node.mjs +32 -0
- package/dist/runtime-node.mjs.map +1 -0
- package/dist/server.d.ts +12792 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +8475 -0
- package/dist/server.js.map +1 -0
- package/dist/server.mjs +8324 -0
- package/dist/server.mjs.map +1 -0
- package/dist/server2.mjs +505 -0
- package/dist/server2.mjs.map +1 -0
- package/package.json +124 -4
- package/src/admin/api/routes.ts +223 -0
- package/src/admin/api/security.ts +12 -0
- package/src/admin/audit/service.ts +138 -0
- package/src/admin/email/service.ts +140 -0
- package/src/admin/enrollment/email.ts +77 -0
- package/src/admin/enrollment/service.ts +308 -0
- package/src/admin/errors.ts +20 -0
- package/src/admin/index.ts +19 -0
- package/src/admin/list.ts +61 -0
- package/src/admin/operations/service.ts +70 -0
- package/src/admin/schema.ts +28 -0
- package/src/admin/users/service.ts +369 -0
- package/src/ai-relay/app.ts +251 -0
- package/src/ai-relay/config.ts +120 -0
- package/src/ai-relay/db/index.ts +10 -0
- package/src/ai-relay/db/migrate.ts +32 -0
- package/src/ai-relay/db/migrations/001_usage.ts +26 -0
- package/src/ai-relay/db/schema.ts +20 -0
- package/src/ai-relay/index.ts +18 -0
- package/src/ai-relay/metering.ts +68 -0
- package/src/ai-relay/usage.ts +115 -0
- package/src/client/ai.ts +96 -0
- package/src/client/api.ts +528 -0
- package/src/client/auth/client.ts +188 -0
- package/src/client/auth/failure.ts +54 -0
- package/src/client/auth/navigation.ts +45 -0
- package/src/client/device-authorization.ts +134 -0
- package/src/client/discovery.ts +105 -0
- package/src/client/images.ts +81 -0
- package/src/client/index.ts +53 -0
- package/src/client/portal/account.ts +183 -0
- package/src/client/portal/administration.ts +130 -0
- package/src/client/portal/client.ts +18 -0
- package/src/client/portal/request.ts +115 -0
- package/src/client/response.ts +51 -0
- package/src/client/workspaces.ts +142 -0
- package/src/contract/account.ts +204 -0
- package/src/contract/admin/index.ts +187 -0
- package/src/contract/ai.ts +150 -0
- package/src/contract/collaboration.ts +45 -0
- package/src/contract/discovery.ts +81 -0
- package/src/contract/documents.ts +265 -0
- package/src/contract/entitlements.ts +28 -0
- package/src/contract/index.ts +204 -0
- package/src/contract/links.ts +87 -0
- package/src/contract/portal.ts +11 -0
- package/src/contract/relay.ts +368 -0
- package/src/contract/sharing.ts +191 -0
- package/src/contract/versions.ts +87 -0
- package/src/contract/workspaces.ts +100 -0
- package/src/email/index.ts +14 -0
- package/src/email/render.ts +276 -0
- package/src/email/schema.ts +144 -0
- package/src/email/templates/brand.ts +105 -0
- package/src/email/templates/document-invitation.ts +35 -0
- package/src/email/templates/enrollment.ts +27 -0
- package/src/email/templates/layout.ts +18 -0
- package/src/runtime/cloudflare/email.ts +70 -0
- package/src/runtime/cloudflare/environment.ts +57 -0
- package/src/runtime/cloudflare/index.ts +3 -0
- package/src/runtime/cloudflare/relay.ts +136 -0
- package/src/runtime/cloudflare/site.ts +13 -0
- package/src/runtime/cloudflare/tsconfig.json +21 -0
- package/src/runtime/cloudflare/worker.ts +160 -0
- package/src/runtime/node/admin.ts +59 -0
- package/src/runtime/node/ai-relay/main.ts +35 -0
- package/src/runtime/node/ai-relay/server.ts +63 -0
- package/src/runtime/node/application.ts +50 -0
- package/src/runtime/node/bootstrap.ts +22 -0
- package/src/runtime/node/cleanup.ts +24 -0
- package/src/runtime/node/command.ts +17 -0
- package/src/runtime/node/config.ts +16 -0
- package/src/runtime/node/email/runtime.ts +43 -0
- package/src/runtime/node/email/transport.ts +112 -0
- package/src/runtime/node/entitlements/main.ts +41 -0
- package/src/runtime/node/entitlements/refusal.ts +15 -0
- package/src/runtime/node/index.ts +25 -0
- package/src/runtime/node/listen.ts +35 -0
- package/src/runtime/node/main.ts +29 -0
- package/src/runtime/node/migrate.ts +13 -0
- package/src/runtime/node/quota-reconcile.ts +9 -0
- package/src/runtime/node/relay.ts +54 -0
- package/src/runtime/node/server.ts +121 -0
- package/src/runtime/node/site.ts +64 -0
- package/src/runtime/postgres/database.ts +19 -0
- package/src/runtime/s3/objects.ts +244 -0
- package/src/runtime/site/server.ts +82 -0
- package/src/server/account/actions.ts +66 -0
- package/src/server/account/routes.ts +300 -0
- package/src/server/ai/errors.ts +7 -0
- package/src/server/ai/index.ts +4 -0
- package/src/server/ai/keys.ts +43 -0
- package/src/server/ai/routes.ts +43 -0
- package/src/server/ai/service.ts +70 -0
- package/src/server/api/index.ts +8 -0
- package/src/server/api/router.ts +117 -0
- package/src/server/api/types.ts +7 -0
- package/src/server/app.ts +375 -0
- package/src/server/auth/account.ts +73 -0
- package/src/server/auth/activity.ts +213 -0
- package/src/server/auth/adapter.ts +79 -0
- package/src/server/auth/email.ts +125 -0
- package/src/server/auth/endpoints.ts +57 -0
- package/src/server/auth/factory.ts +583 -0
- package/src/server/auth/index.ts +17 -0
- package/src/server/auth/mfa-assurance.ts +55 -0
- package/src/server/auth/runtime.ts +26 -0
- package/src/server/auth/session.ts +19 -0
- package/src/server/auth/social-providers.ts +55 -0
- package/src/server/auth/user-agent.ts +16 -0
- package/src/server/auth/workspace-access.ts +23 -0
- package/src/server/cleanup/configured.ts +38 -0
- package/src/server/cleanup/documents.ts +231 -0
- package/src/server/cleanup/index.ts +29 -0
- package/src/server/cleanup/objects.ts +27 -0
- package/src/server/cleanup/revisions.ts +232 -0
- package/src/server/cleanup/uploads.ts +129 -0
- package/src/server/cleanup/worker.ts +79 -0
- package/src/server/collaboration/index.ts +28 -0
- package/src/server/collaboration/relay/authorize.ts +53 -0
- package/src/server/collaboration/relay/awareness.ts +104 -0
- package/src/server/collaboration/relay/documents.ts +39 -0
- package/src/server/collaboration/relay/hub.ts +524 -0
- package/src/server/collaboration/relay/index.ts +15 -0
- package/src/server/collaboration/relay/persistence.ts +48 -0
- package/src/server/collaboration/routes.ts +77 -0
- package/src/server/collaboration/service.ts +188 -0
- package/src/server/config/deployment.ts +178 -0
- package/src/server/config/index.ts +20 -0
- package/src/server/config/limits.ts +8 -0
- package/src/server/config/public.ts +66 -0
- package/src/server/config/schema.ts +512 -0
- package/src/server/db/database.ts +15 -0
- package/src/server/db/expressions.ts +13 -0
- package/src/server/db/index.ts +21 -0
- package/src/server/db/migrate.ts +177 -0
- package/src/server/db/migrations/001_foundation.ts +164 -0
- package/src/server/db/migrations/002_upload_cleanup_claims.ts +34 -0
- package/src/server/db/migrations/003_document_cleanup_claims.ts +23 -0
- package/src/server/db/migrations/004_document_sharing.ts +82 -0
- package/src/server/db/migrations/005_document_collaboration_epoch.ts +13 -0
- package/src/server/db/migrations/006_invitation_continuation.ts +27 -0
- package/src/server/db/migrations/007_collaboration_state.ts +25 -0
- package/src/server/db/migrations/008_storage_reservations.ts +37 -0
- package/src/server/db/migrations/009_workspace_entitlements.ts +22 -0
- package/src/server/db/migrations/010_upload_finalization.ts +22 -0
- package/src/server/db/migrations/011_upload_finalization_lease.ts +12 -0
- package/src/server/db/migrations/012_transactional_email.ts +48 -0
- package/src/server/db/migrations/013_admin_enrollment.ts +54 -0
- package/src/server/db/migrations/014_enrollment_email_kinds.ts +30 -0
- package/src/server/db/migrations/015_cloud_rate_limit.ts +22 -0
- package/src/server/db/migrations/016_auth_schema.ts +16 -0
- package/src/server/db/migrations/017_authentication_email_kinds.ts +32 -0
- package/src/server/db/migrations/018_authentication_mfa.ts +78 -0
- package/src/server/db/migrations/019_better_auth_account_issuer.ts +42 -0
- package/src/server/db/migrations/020_workspace_invitations.ts +57 -0
- package/src/server/db/migrations/021_document_versions.ts +66 -0
- package/src/server/db/migrations/022_entitlement_external_version.ts +17 -0
- package/src/server/db/migrations/023_document_images.ts +61 -0
- package/src/server/db/migrations/024_account_activity.ts +66 -0
- package/src/server/db/migrations/025_table_names.ts +58 -0
- package/src/server/db/migrations/026_user_picture.ts +30 -0
- package/src/server/db/migrations/027_document_kind.ts +19 -0
- package/src/server/db/migrations/helpers.ts +78 -0
- package/src/server/db/tables.ts +394 -0
- package/src/server/documents/access.ts +103 -0
- package/src/server/documents/errors.ts +16 -0
- package/src/server/documents/images.ts +407 -0
- package/src/server/documents/index.ts +14 -0
- package/src/server/documents/limits.ts +57 -0
- package/src/server/documents/repository.ts +141 -0
- package/src/server/documents/routes.ts +239 -0
- package/src/server/documents/service.ts +597 -0
- package/src/server/documents/summary.ts +37 -0
- package/src/server/documents/types.ts +14 -0
- package/src/server/documents/versions/checkpoint.ts +98 -0
- package/src/server/documents/versions/history.ts +25 -0
- package/src/server/documents/versions/index.ts +4 -0
- package/src/server/documents/versions/routes.ts +112 -0
- package/src/server/documents/versions/service.ts +397 -0
- package/src/server/email/crypto.ts +57 -0
- package/src/server/email/index.ts +18 -0
- package/src/server/email/invitations.ts +30 -0
- package/src/server/email/service.ts +245 -0
- package/src/server/email/types.ts +56 -0
- package/src/server/email/worker.ts +33 -0
- package/src/server/index.ts +279 -0
- package/src/server/internal/index.ts +8 -0
- package/src/server/internal/routes.ts +41 -0
- package/src/server/internal/schema.ts +10 -0
- package/src/server/internal/service.ts +44 -0
- package/src/server/invitations/index.ts +6 -0
- package/src/server/invitations/types.ts +30 -0
- package/src/server/objects/index.ts +16 -0
- package/src/server/objects/types.ts +81 -0
- package/src/server/pictures/index.ts +9 -0
- package/src/server/pictures/provider.ts +61 -0
- package/src/server/pictures/routes.ts +21 -0
- package/src/server/pictures/service.ts +235 -0
- package/src/server/policy/configured.ts +23 -0
- package/src/server/policy/database.ts +108 -0
- package/src/server/policy/entitlements.ts +29 -0
- package/src/server/policy/index.ts +17 -0
- package/src/server/policy/keys.ts +23 -0
- package/src/server/policy/policy.ts +117 -0
- package/src/server/policy/provider.ts +67 -0
- package/src/server/policy/schema.ts +69 -0
- package/src/server/policy/service.ts +85 -0
- package/src/server/policy/static.ts +62 -0
- package/src/server/quota/index.ts +7 -0
- package/src/server/quota/reconcile.ts +61 -0
- package/src/server/quota/storage.ts +165 -0
- package/src/server/rate-limit/cleanup.ts +16 -0
- package/src/server/rate-limit/index.ts +12 -0
- package/src/server/rate-limit/keys.ts +38 -0
- package/src/server/rate-limit/middleware.ts +137 -0
- package/src/server/rate-limit/postgres.ts +79 -0
- package/src/server/sharing/access.ts +29 -0
- package/src/server/sharing/actor.ts +12 -0
- package/src/server/sharing/capabilities/service.ts +262 -0
- package/src/server/sharing/continuation.ts +85 -0
- package/src/server/sharing/contracts.ts +80 -0
- package/src/server/sharing/crypto.ts +48 -0
- package/src/server/sharing/document-routes.ts +41 -0
- package/src/server/sharing/errors.ts +12 -0
- package/src/server/sharing/grants/service.ts +79 -0
- package/src/server/sharing/http.ts +41 -0
- package/src/server/sharing/index.ts +39 -0
- package/src/server/sharing/invitations/service.ts +315 -0
- package/src/server/sharing/policy.ts +44 -0
- package/src/server/sharing/routes.ts +251 -0
- package/src/server/sharing/service.ts +117 -0
- package/src/server/validation.ts +24 -0
- package/src/server/worker/interval.ts +23 -0
- package/src/server/worker/periodic.ts +42 -0
- package/src/server/workspaces/index.ts +16 -0
- package/src/server/workspaces/members.ts +502 -0
- package/src/server/workspaces/repository.ts +91 -0
- package/src/server/workspaces/routes.ts +198 -0
- package/src/server/workspaces/service.ts +84 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { CloudAuthAdapter, CloudIdentity } from '#cloud/server/auth/adapter'
|
|
2
|
+
|
|
3
|
+
export type CloudActor = {
|
|
4
|
+
userId: string
|
|
5
|
+
email: string
|
|
6
|
+
name: string
|
|
7
|
+
deploymentRole?: 'user' | 'admin'
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export type CloudIdentityResolver = (request: Request) => Promise<CloudIdentity | null>
|
|
11
|
+
export type CloudSessionResolver = (request: Request) => Promise<CloudActor | null>
|
|
12
|
+
|
|
13
|
+
export function createCloudIdentityResolver(auth: CloudAuthAdapter): CloudIdentityResolver {
|
|
14
|
+
return (request) => auth.resolveIdentity(request.headers)
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function createCloudSessionResolver(auth: CloudAuthAdapter): CloudSessionResolver {
|
|
18
|
+
return (request) => auth.resolveSession(request.headers)
|
|
19
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { CloudServerConfig } from '#cloud/server/config'
|
|
2
|
+
import type { BetterAuthOptions } from 'better-auth'
|
|
3
|
+
import { importPKCS8, SignJWT } from 'jose'
|
|
4
|
+
|
|
5
|
+
/** The audience Apple's sign-in tokens carry, which the server also trusts as an origin. */
|
|
6
|
+
export const APPLE_AUDIENCE = 'https://appleid.apple.com'
|
|
7
|
+
const APPLE_SECRET_LIFETIME_SECONDS = 180 * 24 * 60 * 60
|
|
8
|
+
async function generateAppleClientSecret(config: CloudServerConfig): Promise<string> {
|
|
9
|
+
if (
|
|
10
|
+
!config.appleClientId ||
|
|
11
|
+
!config.appleTeamId ||
|
|
12
|
+
!config.appleKeyId ||
|
|
13
|
+
!config.applePrivateKey
|
|
14
|
+
) {
|
|
15
|
+
throw new Error('Apple authentication is not configured')
|
|
16
|
+
}
|
|
17
|
+
const privateKey = await importPKCS8(config.applePrivateKey.replaceAll('\\n', '\n'), 'ES256')
|
|
18
|
+
const issuedAt = Math.floor(Date.now() / 1000)
|
|
19
|
+
return new SignJWT({})
|
|
20
|
+
.setProtectedHeader({ alg: 'ES256', kid: config.appleKeyId })
|
|
21
|
+
.setIssuer(config.appleTeamId)
|
|
22
|
+
.setSubject(config.appleClientId)
|
|
23
|
+
.setAudience(APPLE_AUDIENCE)
|
|
24
|
+
.setIssuedAt(issuedAt)
|
|
25
|
+
.setExpirationTime(issuedAt + APPLE_SECRET_LIFETIME_SECONDS)
|
|
26
|
+
.sign(privateKey)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The sign-in providers the deployment configures; Apple's secret is signed when it is needed. */
|
|
30
|
+
export function socialProviders(config: CloudServerConfig): BetterAuthOptions['socialProviders'] {
|
|
31
|
+
const providers: BetterAuthOptions['socialProviders'] = {}
|
|
32
|
+
if (config.googleClientId && config.googleClientSecret) {
|
|
33
|
+
providers.google = {
|
|
34
|
+
clientId: config.googleClientId,
|
|
35
|
+
clientSecret: config.googleClientSecret
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (config.appleClientId) {
|
|
39
|
+
providers.apple = async () => {
|
|
40
|
+
const provider: {
|
|
41
|
+
clientId: string
|
|
42
|
+
clientSecret: string
|
|
43
|
+
appBundleIdentifier?: string
|
|
44
|
+
} = {
|
|
45
|
+
clientId: config.appleClientId ?? '',
|
|
46
|
+
clientSecret: await generateAppleClientSecret(config)
|
|
47
|
+
}
|
|
48
|
+
if (config.appleAppBundleIdentifier) {
|
|
49
|
+
provider.appBundleIdentifier = config.appleAppBundleIdentifier
|
|
50
|
+
}
|
|
51
|
+
return provider
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return providers
|
|
55
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import Bowser from 'bowser'
|
|
2
|
+
import { compact } from 'es-toolkit'
|
|
3
|
+
|
|
4
|
+
/** The browser and system a user agent names, as words for a person: `Chrome 141`, `macOS`. */
|
|
5
|
+
export function describeUserAgent(userAgent: string | null | undefined): {
|
|
6
|
+
browser: string | null
|
|
7
|
+
system: string | null
|
|
8
|
+
} {
|
|
9
|
+
if (!userAgent) return { browser: null, system: null }
|
|
10
|
+
const { browser, os } = Bowser.parse(userAgent)
|
|
11
|
+
const version = browser.version?.split('.', 1)[0]
|
|
12
|
+
return {
|
|
13
|
+
browser: browser.name ? compact([browser.name, version]).join(' ') : null,
|
|
14
|
+
system: os.name || null
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { createAccessControl } from 'better-auth/plugins/access'
|
|
2
|
+
|
|
3
|
+
export const workspaceStatements = {
|
|
4
|
+
workspace: ['read', 'update', 'delete', 'invite', 'manage-members'],
|
|
5
|
+
document: ['create', 'read', 'edit', 'share', 'delete']
|
|
6
|
+
} as const
|
|
7
|
+
|
|
8
|
+
export const workspaceAccessControl = createAccessControl(workspaceStatements)
|
|
9
|
+
|
|
10
|
+
export const workspaceRoles = {
|
|
11
|
+
admin: workspaceAccessControl.newRole({
|
|
12
|
+
workspace: ['read', 'update', 'delete', 'invite', 'manage-members'],
|
|
13
|
+
document: ['create', 'read', 'edit', 'share', 'delete']
|
|
14
|
+
}),
|
|
15
|
+
editor: workspaceAccessControl.newRole({
|
|
16
|
+
workspace: ['read'],
|
|
17
|
+
document: ['create', 'read', 'edit', 'share']
|
|
18
|
+
}),
|
|
19
|
+
viewer: workspaceAccessControl.newRole({
|
|
20
|
+
workspace: ['read'],
|
|
21
|
+
document: ['read']
|
|
22
|
+
})
|
|
23
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { CloudServerConfig } from '#cloud/server/config'
|
|
2
|
+
import type { CloudDatabase } from '#cloud/server/db'
|
|
3
|
+
import type { ObjectStore } from '#cloud/server/objects'
|
|
4
|
+
import type { CloudPolicy } from '#cloud/server/policy/policy'
|
|
5
|
+
import { createRateLimitCleanupService } from '#cloud/server/rate-limit/cleanup'
|
|
6
|
+
import type { Kysely } from 'kysely'
|
|
7
|
+
|
|
8
|
+
import { createDocumentCleanupService } from './documents'
|
|
9
|
+
import { createRevisionCompactionService } from './revisions'
|
|
10
|
+
import { createUploadCleanupService } from './uploads'
|
|
11
|
+
import type { CleanupPassOptions, CleanupServices } from './worker'
|
|
12
|
+
|
|
13
|
+
/** Every cleanup service, over a deployment's database and bucket. */
|
|
14
|
+
export function createCleanupServices(
|
|
15
|
+
config: Pick<CloudServerConfig, 'deployment'>,
|
|
16
|
+
database: Kysely<CloudDatabase>,
|
|
17
|
+
objects: ObjectStore,
|
|
18
|
+
policy: CloudPolicy
|
|
19
|
+
): CleanupServices {
|
|
20
|
+
return {
|
|
21
|
+
uploads: createUploadCleanupService(database, objects),
|
|
22
|
+
documents: createDocumentCleanupService(database, objects),
|
|
23
|
+
revisions: createRevisionCompactionService(database, objects, {
|
|
24
|
+
policy,
|
|
25
|
+
deploymentMode: config.deployment
|
|
26
|
+
}),
|
|
27
|
+
rateLimits: createRateLimitCleanupService(database)
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** The batch, lease, and retention a deployment sets for each cleanup pass. */
|
|
32
|
+
export function cleanupPassOptions(config: CloudServerConfig): CleanupPassOptions {
|
|
33
|
+
return {
|
|
34
|
+
batchSize: config.cleanupBatchSize,
|
|
35
|
+
documentRetentionMs: config.documentRetentionMs,
|
|
36
|
+
leaseDurationMs: config.cleanupLeaseDurationMs
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import type { CloudDatabase } from '#cloud/server/db'
|
|
2
|
+
import type { ObjectStore } from '#cloud/server/objects'
|
|
3
|
+
import type { Kysely } from 'kysely'
|
|
4
|
+
|
|
5
|
+
import { unreferencedObject } from './objects'
|
|
6
|
+
|
|
7
|
+
export type DocumentCleanupOptions = {
|
|
8
|
+
batchSize: number
|
|
9
|
+
leaseDurationMs: number
|
|
10
|
+
retentionMs: number
|
|
11
|
+
now?: Date
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export type DocumentCleanupResult = {
|
|
15
|
+
claimed: number
|
|
16
|
+
cleaned: number
|
|
17
|
+
failed: number
|
|
18
|
+
objectsDeleted: number
|
|
19
|
+
bytesReclaimed: number
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export type DocumentCleanupService = {
|
|
23
|
+
cleanupDeletedDocuments(options: DocumentCleanupOptions): Promise<DocumentCleanupResult>
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
type ClaimedDocument = {
|
|
27
|
+
id: string
|
|
28
|
+
workspaceId: string
|
|
29
|
+
deletedAt: Date
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
type StoredRevisionObject = {
|
|
33
|
+
id: string
|
|
34
|
+
objectKey: string
|
|
35
|
+
byteSize: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function createDocumentCleanupService(
|
|
39
|
+
database: Kysely<CloudDatabase>,
|
|
40
|
+
objects: ObjectStore
|
|
41
|
+
): DocumentCleanupService {
|
|
42
|
+
return {
|
|
43
|
+
async cleanupDeletedDocuments(options) {
|
|
44
|
+
const now = options.now ?? new Date()
|
|
45
|
+
const retentionBefore = new Date(now.getTime() - options.retentionMs)
|
|
46
|
+
const staleBefore = new Date(now.getTime() - options.leaseDurationMs)
|
|
47
|
+
const claimId = crypto.randomUUID()
|
|
48
|
+
const documents = await claimDocuments(
|
|
49
|
+
database,
|
|
50
|
+
options.batchSize,
|
|
51
|
+
retentionBefore,
|
|
52
|
+
staleBefore,
|
|
53
|
+
now,
|
|
54
|
+
claimId
|
|
55
|
+
)
|
|
56
|
+
const result: DocumentCleanupResult = {
|
|
57
|
+
claimed: documents.length,
|
|
58
|
+
cleaned: 0,
|
|
59
|
+
failed: 0,
|
|
60
|
+
objectsDeleted: 0,
|
|
61
|
+
bytesReclaimed: 0
|
|
62
|
+
}
|
|
63
|
+
for (const document of documents) {
|
|
64
|
+
try {
|
|
65
|
+
const revisionObjects = await documentObjects(database, document.id)
|
|
66
|
+
for (const object of revisionObjects) await objects.delete(object.objectKey)
|
|
67
|
+
const reclaimed = await removeDocumentMetadata(
|
|
68
|
+
database,
|
|
69
|
+
document,
|
|
70
|
+
revisionObjects,
|
|
71
|
+
retentionBefore,
|
|
72
|
+
claimId
|
|
73
|
+
)
|
|
74
|
+
if (!reclaimed) continue
|
|
75
|
+
result.cleaned++
|
|
76
|
+
result.objectsDeleted += reclaimed.objectsDeleted
|
|
77
|
+
result.bytesReclaimed += reclaimed.bytesReclaimed
|
|
78
|
+
} catch {
|
|
79
|
+
result.failed++
|
|
80
|
+
await releaseClaim(database, document.id, claimId)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return result
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async function claimDocuments(
|
|
89
|
+
database: Kysely<CloudDatabase>,
|
|
90
|
+
batchSize: number,
|
|
91
|
+
retentionBefore: Date,
|
|
92
|
+
staleBefore: Date,
|
|
93
|
+
now: Date,
|
|
94
|
+
claimId: string
|
|
95
|
+
): Promise<ClaimedDocument[]> {
|
|
96
|
+
return database.transaction().execute(async (transaction) => {
|
|
97
|
+
const candidates = await transaction
|
|
98
|
+
.selectFrom('document')
|
|
99
|
+
.select(['id', 'workspaceId', 'deletedAt'])
|
|
100
|
+
.where('deletedAt', 'is not', null)
|
|
101
|
+
.where('deletedAt', '<=', retentionBefore)
|
|
102
|
+
.where((expression) =>
|
|
103
|
+
expression.or([
|
|
104
|
+
expression('cleanupClaimId', 'is', null),
|
|
105
|
+
expression('cleanupClaimedAt', '<=', staleBefore)
|
|
106
|
+
])
|
|
107
|
+
)
|
|
108
|
+
.where((expression) =>
|
|
109
|
+
expression.not(
|
|
110
|
+
expression.exists(
|
|
111
|
+
expression
|
|
112
|
+
.selectFrom('upload')
|
|
113
|
+
.select('upload.id')
|
|
114
|
+
.whereRef('upload.documentId', '=', 'document.id')
|
|
115
|
+
.where('upload.status', 'in', ['pending', 'cleaning'])
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
.orderBy('deletedAt')
|
|
120
|
+
.limit(batchSize)
|
|
121
|
+
.forUpdate()
|
|
122
|
+
.skipLocked()
|
|
123
|
+
.execute()
|
|
124
|
+
const claimed = candidates.filter(
|
|
125
|
+
(document): document is ClaimedDocument => document.deletedAt !== null
|
|
126
|
+
)
|
|
127
|
+
if (claimed.length === 0) return []
|
|
128
|
+
await transaction
|
|
129
|
+
.updateTable('document')
|
|
130
|
+
.set({ cleanupClaimId: claimId, cleanupClaimedAt: now })
|
|
131
|
+
.where(
|
|
132
|
+
'id',
|
|
133
|
+
'in',
|
|
134
|
+
claimed.map((document) => document.id)
|
|
135
|
+
)
|
|
136
|
+
.execute()
|
|
137
|
+
return claimed
|
|
138
|
+
})
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async function documentObjects(
|
|
142
|
+
database: Kysely<CloudDatabase>,
|
|
143
|
+
documentId: string
|
|
144
|
+
): Promise<StoredRevisionObject[]> {
|
|
145
|
+
const rows = await database
|
|
146
|
+
.selectFrom('documentRevision')
|
|
147
|
+
.innerJoin('storageObject', 'storageObject.id', 'documentRevision.storageObjectId')
|
|
148
|
+
.select(['storageObject.id', 'storageObject.objectKey', 'storageObject.byteSize'])
|
|
149
|
+
.where('documentRevision.documentId', '=', documentId)
|
|
150
|
+
.where((expression) =>
|
|
151
|
+
expression.not(
|
|
152
|
+
expression.exists(
|
|
153
|
+
expression
|
|
154
|
+
.selectFrom('documentRevision as otherRevision')
|
|
155
|
+
.select('otherRevision.id')
|
|
156
|
+
.whereRef('otherRevision.storageObjectId', '=', 'storageObject.id')
|
|
157
|
+
.where('otherRevision.documentId', '!=', documentId)
|
|
158
|
+
)
|
|
159
|
+
)
|
|
160
|
+
)
|
|
161
|
+
.execute()
|
|
162
|
+
return rows.map((row) => ({ ...row, byteSize: Number(row.byteSize) }))
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function removeDocumentMetadata(
|
|
166
|
+
database: Kysely<CloudDatabase>,
|
|
167
|
+
document: ClaimedDocument,
|
|
168
|
+
objects: StoredRevisionObject[],
|
|
169
|
+
retentionBefore: Date,
|
|
170
|
+
claimId: string
|
|
171
|
+
): Promise<{ objectsDeleted: number; bytesReclaimed: number } | null> {
|
|
172
|
+
return database.transaction().execute(async (transaction) => {
|
|
173
|
+
const locked = await transaction
|
|
174
|
+
.selectFrom('document')
|
|
175
|
+
.select(['deletedAt', 'cleanupClaimId'])
|
|
176
|
+
.where('id', '=', document.id)
|
|
177
|
+
.forUpdate()
|
|
178
|
+
.executeTakeFirst()
|
|
179
|
+
if (
|
|
180
|
+
!locked?.deletedAt ||
|
|
181
|
+
locked.deletedAt > retentionBefore ||
|
|
182
|
+
locked.cleanupClaimId !== claimId
|
|
183
|
+
) {
|
|
184
|
+
return null
|
|
185
|
+
}
|
|
186
|
+
await transaction
|
|
187
|
+
.updateTable('document')
|
|
188
|
+
.set({ currentRevisionId: null })
|
|
189
|
+
.where('id', '=', document.id)
|
|
190
|
+
.execute()
|
|
191
|
+
await transaction.deleteFrom('documentRevision').where('documentId', '=', document.id).execute()
|
|
192
|
+
let objectsDeleted = 0
|
|
193
|
+
let bytesReclaimed = 0
|
|
194
|
+
for (const object of objects) {
|
|
195
|
+
const deleted = await transaction
|
|
196
|
+
.deleteFrom('storageObject')
|
|
197
|
+
.where('id', '=', object.id)
|
|
198
|
+
.where(unreferencedObject)
|
|
199
|
+
.executeTakeFirst()
|
|
200
|
+
if (Number(deleted.numDeletedRows) > 0) {
|
|
201
|
+
objectsDeleted++
|
|
202
|
+
bytesReclaimed += object.byteSize
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
if (bytesReclaimed > 0) {
|
|
206
|
+
await transaction
|
|
207
|
+
.updateTable('workspaceStorageUsage')
|
|
208
|
+
.set({
|
|
209
|
+
committedBytes: (expression) => expression('committedBytes', '-', bytesReclaimed),
|
|
210
|
+
updatedAt: new Date()
|
|
211
|
+
})
|
|
212
|
+
.where('workspaceId', '=', document.workspaceId)
|
|
213
|
+
.execute()
|
|
214
|
+
}
|
|
215
|
+
await transaction.deleteFrom('document').where('id', '=', document.id).execute()
|
|
216
|
+
return { objectsDeleted, bytesReclaimed }
|
|
217
|
+
})
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async function releaseClaim(
|
|
221
|
+
database: Kysely<CloudDatabase>,
|
|
222
|
+
documentId: string,
|
|
223
|
+
claimId: string
|
|
224
|
+
): Promise<void> {
|
|
225
|
+
await database
|
|
226
|
+
.updateTable('document')
|
|
227
|
+
.set({ cleanupClaimId: null, cleanupClaimedAt: null })
|
|
228
|
+
.where('id', '=', documentId)
|
|
229
|
+
.where('cleanupClaimId', '=', claimId)
|
|
230
|
+
.execute()
|
|
231
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export { cleanupPassOptions, createCleanupServices } from './configured'
|
|
2
|
+
export {
|
|
3
|
+
createDocumentCleanupService,
|
|
4
|
+
type DocumentCleanupOptions,
|
|
5
|
+
type DocumentCleanupResult,
|
|
6
|
+
type DocumentCleanupService
|
|
7
|
+
} from './documents'
|
|
8
|
+
export {
|
|
9
|
+
createRevisionCompactionService,
|
|
10
|
+
REVISION_COMPACTION_GRACE_MS,
|
|
11
|
+
type RevisionCompactionOptions,
|
|
12
|
+
type RevisionCompactionResult,
|
|
13
|
+
type RevisionCompactionService
|
|
14
|
+
} from './revisions'
|
|
15
|
+
export {
|
|
16
|
+
createUploadCleanupService,
|
|
17
|
+
type UploadCleanupOptions,
|
|
18
|
+
type UploadCleanupResult,
|
|
19
|
+
type UploadCleanupService
|
|
20
|
+
} from './uploads'
|
|
21
|
+
export {
|
|
22
|
+
runCleanup,
|
|
23
|
+
startCleanupWorker,
|
|
24
|
+
type CleanupPassOptions,
|
|
25
|
+
type CleanupResult,
|
|
26
|
+
type CleanupServices,
|
|
27
|
+
type CleanupWorker,
|
|
28
|
+
type CleanupWorkerOptions
|
|
29
|
+
} from './worker'
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { CloudDatabase } from '#cloud/server/db'
|
|
2
|
+
import type { ExpressionBuilder } from 'kysely'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* No revision refers to the stored file any more, and no workspace keeps it as an image, so it
|
|
6
|
+
* can leave the object store.
|
|
7
|
+
*/
|
|
8
|
+
export function unreferencedObject(expression: ExpressionBuilder<CloudDatabase, 'storageObject'>) {
|
|
9
|
+
return expression.and([
|
|
10
|
+
expression.not(
|
|
11
|
+
expression.exists(
|
|
12
|
+
expression
|
|
13
|
+
.selectFrom('documentRevision')
|
|
14
|
+
.select('documentRevision.id')
|
|
15
|
+
.whereRef('documentRevision.storageObjectId', '=', 'storageObject.id')
|
|
16
|
+
)
|
|
17
|
+
),
|
|
18
|
+
expression.not(
|
|
19
|
+
expression.exists(
|
|
20
|
+
expression
|
|
21
|
+
.selectFrom('workspaceImage')
|
|
22
|
+
.select('workspaceImage.checksum')
|
|
23
|
+
.whereRef('workspaceImage.storageObjectId', '=', 'storageObject.id')
|
|
24
|
+
)
|
|
25
|
+
)
|
|
26
|
+
])
|
|
27
|
+
}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import type { CloudDatabase } from '#cloud/server/db'
|
|
2
|
+
import { IMAGE_RELEASE_GRACE_MS, releaseUnusedImages } from '#cloud/server/documents/images'
|
|
3
|
+
import { historyStart, LIMITED_HISTORY_DAYS } from '#cloud/server/documents/versions/history'
|
|
4
|
+
import type { ObjectStore } from '#cloud/server/objects'
|
|
5
|
+
import type { CloudPolicyContext } from '#cloud/server/policy/keys'
|
|
6
|
+
import type { CloudPolicy } from '#cloud/server/policy/policy'
|
|
7
|
+
import { groupBy, sumBy, uniq } from 'es-toolkit'
|
|
8
|
+
import type { Kysely, Transaction } from 'kysely'
|
|
9
|
+
|
|
10
|
+
import { unreferencedObject } from './objects'
|
|
11
|
+
|
|
12
|
+
/** How long a save between versions is kept before compaction may delete it. */
|
|
13
|
+
export const REVISION_COMPACTION_GRACE_MS = 24 * 60 * 60_000
|
|
14
|
+
|
|
15
|
+
export type RevisionCompactionOptions = {
|
|
16
|
+
batchSize: number
|
|
17
|
+
/** How long a save stays before it can be compacted, so recent work is never lost to it. */
|
|
18
|
+
graceMs: number
|
|
19
|
+
now?: Date
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export type RevisionCompactionResult = {
|
|
23
|
+
versionsPruned: number
|
|
24
|
+
/** Images no revision used any more, released after their grace period. */
|
|
25
|
+
imagesReleased: number
|
|
26
|
+
revisionsDeleted: number
|
|
27
|
+
objectsDeleted: number
|
|
28
|
+
bytesReclaimed: number
|
|
29
|
+
failed: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export type RevisionCompactionService = {
|
|
33
|
+
compactRevisions(options: RevisionCompactionOptions): Promise<RevisionCompactionResult>
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Keeps a document's stored history to its versions. A save that is neither the document's
|
|
38
|
+
* current state nor kept by a version is deleted once it is older than the grace period, and its
|
|
39
|
+
* file with it when nothing else uses it; a workspace without full history first loses the
|
|
40
|
+
* versions older than its 30 days.
|
|
41
|
+
*/
|
|
42
|
+
export function createRevisionCompactionService(
|
|
43
|
+
database: Kysely<CloudDatabase>,
|
|
44
|
+
objects: ObjectStore,
|
|
45
|
+
options: { policy?: CloudPolicy; deploymentMode?: CloudPolicyContext['deploymentMode'] } = {}
|
|
46
|
+
): RevisionCompactionService {
|
|
47
|
+
const deploymentMode = options.deploymentMode ?? 'self-hosted'
|
|
48
|
+
async function pruneVersions(now: Date, batchSize: number): Promise<number> {
|
|
49
|
+
if (!options.policy) return 0
|
|
50
|
+
const limitedBefore = new Date(now.getTime() - LIMITED_HISTORY_DAYS * 24 * 60 * 60_000)
|
|
51
|
+
const workspaces = await database
|
|
52
|
+
.selectFrom('documentVersion')
|
|
53
|
+
.innerJoin('document', 'document.id', 'documentVersion.documentId')
|
|
54
|
+
.select('document.workspaceId')
|
|
55
|
+
.distinct()
|
|
56
|
+
.where('documentVersion.createdAt', '<', limitedBefore)
|
|
57
|
+
.execute()
|
|
58
|
+
let pruned = 0
|
|
59
|
+
for (const { workspaceId } of workspaces) {
|
|
60
|
+
const start = await historyStart(options.policy, deploymentMode, workspaceId, now)
|
|
61
|
+
if (!start) continue
|
|
62
|
+
const deleted = await database
|
|
63
|
+
.deleteFrom('documentVersion')
|
|
64
|
+
.where('id', 'in', (query) =>
|
|
65
|
+
query
|
|
66
|
+
.selectFrom('documentVersion as version')
|
|
67
|
+
.innerJoin('document', 'document.id', 'version.documentId')
|
|
68
|
+
.select('version.id')
|
|
69
|
+
.where('document.workspaceId', '=', workspaceId)
|
|
70
|
+
.where('version.createdAt', '<', start)
|
|
71
|
+
// The document's current state stays listed however old it is.
|
|
72
|
+
.whereRef('version.revisionId', '!=', 'document.currentRevisionId')
|
|
73
|
+
.limit(batchSize)
|
|
74
|
+
)
|
|
75
|
+
.executeTakeFirst()
|
|
76
|
+
pruned += Number(deleted.numDeletedRows)
|
|
77
|
+
}
|
|
78
|
+
return pruned
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Saves that may go: old, unversioned, not current, in a document that is not deleted. */
|
|
82
|
+
function compactable(source: Kysely<CloudDatabase> | Transaction<CloudDatabase>, before: Date) {
|
|
83
|
+
return source
|
|
84
|
+
.selectFrom('documentRevision')
|
|
85
|
+
.innerJoin('document', 'document.id', 'documentRevision.documentId')
|
|
86
|
+
.where('documentRevision.createdAt', '<', before)
|
|
87
|
+
.where('document.deletedAt', 'is', null)
|
|
88
|
+
.where((expression) =>
|
|
89
|
+
expression.or([
|
|
90
|
+
expression('document.currentRevisionId', 'is', null),
|
|
91
|
+
expression('document.currentRevisionId', '!=', expression.ref('documentRevision.id'))
|
|
92
|
+
])
|
|
93
|
+
)
|
|
94
|
+
.where((expression) =>
|
|
95
|
+
expression.not(
|
|
96
|
+
expression.exists(
|
|
97
|
+
expression
|
|
98
|
+
.selectFrom('documentVersion')
|
|
99
|
+
.select('documentVersion.id')
|
|
100
|
+
.whereRef('documentVersion.revisionId', '=', 'documentRevision.id')
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** A file no revision refers to: freed by compaction, waiting for the object store. */
|
|
107
|
+
function unreferencedObjects(source: Kysely<CloudDatabase> | Transaction<CloudDatabase>) {
|
|
108
|
+
return source.selectFrom('storageObject').where(unreferencedObject)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Deletes up to `limit` compactable saves of one document and returns the bytes of files nothing
|
|
113
|
+
* refers to any more to the workspace's usage. Their rows stay until the files are gone, so a
|
|
114
|
+
* delete that fails is tried again on the next pass.
|
|
115
|
+
*/
|
|
116
|
+
async function compactDocument(
|
|
117
|
+
documentId: string,
|
|
118
|
+
workspaceId: string,
|
|
119
|
+
before: Date,
|
|
120
|
+
limit: number
|
|
121
|
+
) {
|
|
122
|
+
return database.transaction().execute(async (transaction) => {
|
|
123
|
+
// Saves and restores lock the document too, so none can keep a revision deleted here.
|
|
124
|
+
await transaction
|
|
125
|
+
.selectFrom('document')
|
|
126
|
+
.select('id')
|
|
127
|
+
.where('id', '=', documentId)
|
|
128
|
+
.forUpdate()
|
|
129
|
+
.executeTakeFirst()
|
|
130
|
+
const revisions = await compactable(transaction, before)
|
|
131
|
+
.select(['documentRevision.id', 'documentRevision.storageObjectId'])
|
|
132
|
+
.where('documentRevision.documentId', '=', documentId)
|
|
133
|
+
.limit(limit)
|
|
134
|
+
.execute()
|
|
135
|
+
if (revisions.length === 0) return { revisions: 0, bytes: 0 }
|
|
136
|
+
await transaction
|
|
137
|
+
.deleteFrom('documentRevision')
|
|
138
|
+
.where(
|
|
139
|
+
'id',
|
|
140
|
+
'in',
|
|
141
|
+
revisions.map((revision) => revision.id)
|
|
142
|
+
)
|
|
143
|
+
.execute()
|
|
144
|
+
const freed = await unreferencedObjects(transaction)
|
|
145
|
+
.select('byteSize')
|
|
146
|
+
.where('id', 'in', uniq(revisions.map((revision) => revision.storageObjectId)))
|
|
147
|
+
.execute()
|
|
148
|
+
const bytes = sumBy(freed, (object) => Number(object.byteSize))
|
|
149
|
+
if (bytes > 0) {
|
|
150
|
+
await transaction
|
|
151
|
+
.updateTable('workspaceStorageUsage')
|
|
152
|
+
.set({
|
|
153
|
+
committedBytes: (expression) => expression('committedBytes', '-', bytes),
|
|
154
|
+
updatedAt: new Date()
|
|
155
|
+
})
|
|
156
|
+
.where('workspaceId', '=', workspaceId)
|
|
157
|
+
.execute()
|
|
158
|
+
}
|
|
159
|
+
return { revisions: revisions.length, bytes }
|
|
160
|
+
})
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Deletes freed files from the object store, then their rows; a failure stays for next time. */
|
|
164
|
+
async function sweepObjects(batchSize: number, result: RevisionCompactionResult) {
|
|
165
|
+
const freed = await unreferencedObjects(database)
|
|
166
|
+
.select(['id', 'objectKey'])
|
|
167
|
+
.limit(batchSize)
|
|
168
|
+
.execute()
|
|
169
|
+
for (const object of freed) {
|
|
170
|
+
try {
|
|
171
|
+
await objects.delete(object.objectKey)
|
|
172
|
+
await database
|
|
173
|
+
.deleteFrom('storageObject')
|
|
174
|
+
.where('id', '=', object.id)
|
|
175
|
+
.where(unreferencedObject)
|
|
176
|
+
.execute()
|
|
177
|
+
result.objectsDeleted++
|
|
178
|
+
} catch {
|
|
179
|
+
result.failed++
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
return {
|
|
185
|
+
async compactRevisions(input) {
|
|
186
|
+
const now = input.now ?? new Date()
|
|
187
|
+
const before = new Date(now.getTime() - input.graceMs)
|
|
188
|
+
const result: RevisionCompactionResult = {
|
|
189
|
+
versionsPruned: await pruneVersions(now, input.batchSize),
|
|
190
|
+
imagesReleased: 0,
|
|
191
|
+
revisionsDeleted: 0,
|
|
192
|
+
objectsDeleted: 0,
|
|
193
|
+
bytesReclaimed: 0,
|
|
194
|
+
failed: 0
|
|
195
|
+
}
|
|
196
|
+
const candidates = await compactable(database, before)
|
|
197
|
+
.select(['documentRevision.documentId', 'document.workspaceId'])
|
|
198
|
+
.limit(input.batchSize)
|
|
199
|
+
.execute()
|
|
200
|
+
let capacity = input.batchSize
|
|
201
|
+
for (const [documentId, rows] of Object.entries(
|
|
202
|
+
groupBy(candidates, (row) => row.documentId)
|
|
203
|
+
)) {
|
|
204
|
+
const workspaceId = rows[0]?.workspaceId
|
|
205
|
+
if (!workspaceId || capacity <= 0) continue
|
|
206
|
+
try {
|
|
207
|
+
const compacted = await compactDocument(documentId, workspaceId, before, capacity)
|
|
208
|
+
capacity -= compacted.revisions
|
|
209
|
+
result.revisionsDeleted += compacted.revisions
|
|
210
|
+
result.bytesReclaimed += compacted.bytes
|
|
211
|
+
} catch {
|
|
212
|
+
result.failed++
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// Compaction just freed the images of the saves it deleted; the release lets their grace pass.
|
|
216
|
+
// A release that raced a save referring to an image fails whole and is tried next time.
|
|
217
|
+
try {
|
|
218
|
+
const images = await releaseUnusedImages(
|
|
219
|
+
database,
|
|
220
|
+
new Date(now.getTime() - IMAGE_RELEASE_GRACE_MS),
|
|
221
|
+
input.batchSize
|
|
222
|
+
)
|
|
223
|
+
result.imagesReleased = images.released
|
|
224
|
+
result.bytesReclaimed += images.bytes
|
|
225
|
+
} catch {
|
|
226
|
+
result.failed++
|
|
227
|
+
}
|
|
228
|
+
await sweepObjects(input.batchSize, result)
|
|
229
|
+
return result
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|