@open-pencil/cloud 0.0.0-stage → 0.17.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/LICENSE +202 -0
  2. package/LICENSE-MIT +21 -0
  3. package/NOTICE +7 -0
  4. package/README.md +37 -2
  5. package/dist/bin/ai-relay.d.mts +1 -0
  6. package/dist/bin/ai-relay.mjs +31 -0
  7. package/dist/bin/ai-relay.mjs.map +1 -0
  8. package/dist/bin/server.d.mts +1 -0
  9. package/dist/bin/server.mjs +29 -0
  10. package/dist/bin/server.mjs.map +1 -0
  11. package/dist/client.d.ts +1520 -0
  12. package/dist/client.d.ts.map +1 -0
  13. package/dist/client.js +1917 -0
  14. package/dist/client.js.map +1 -0
  15. package/dist/contract.d.ts +2 -0
  16. package/dist/contract.js +2 -0
  17. package/dist/contract2.js +1453 -0
  18. package/dist/contract2.js.map +1 -0
  19. package/dist/email.d.ts +169 -0
  20. package/dist/email.d.ts.map +1 -0
  21. package/dist/email.js +670 -0
  22. package/dist/email.js.map +1 -0
  23. package/dist/index.d.ts +1755 -0
  24. package/dist/index.d.ts.map +1 -0
  25. package/dist/listen.mjs +1881 -0
  26. package/dist/listen.mjs.map +1 -0
  27. package/dist/runtime-node.d.mts +10040 -0
  28. package/dist/runtime-node.d.mts.map +1 -0
  29. package/dist/runtime-node.mjs +32 -0
  30. package/dist/runtime-node.mjs.map +1 -0
  31. package/dist/server.d.ts +12792 -0
  32. package/dist/server.d.ts.map +1 -0
  33. package/dist/server.js +8475 -0
  34. package/dist/server.js.map +1 -0
  35. package/dist/server.mjs +8324 -0
  36. package/dist/server.mjs.map +1 -0
  37. package/dist/server2.mjs +505 -0
  38. package/dist/server2.mjs.map +1 -0
  39. package/package.json +124 -4
  40. package/src/admin/api/routes.ts +223 -0
  41. package/src/admin/api/security.ts +12 -0
  42. package/src/admin/audit/service.ts +138 -0
  43. package/src/admin/email/service.ts +140 -0
  44. package/src/admin/enrollment/email.ts +77 -0
  45. package/src/admin/enrollment/service.ts +308 -0
  46. package/src/admin/errors.ts +20 -0
  47. package/src/admin/index.ts +19 -0
  48. package/src/admin/list.ts +61 -0
  49. package/src/admin/operations/service.ts +70 -0
  50. package/src/admin/schema.ts +28 -0
  51. package/src/admin/users/service.ts +369 -0
  52. package/src/ai-relay/app.ts +251 -0
  53. package/src/ai-relay/config.ts +120 -0
  54. package/src/ai-relay/db/index.ts +10 -0
  55. package/src/ai-relay/db/migrate.ts +32 -0
  56. package/src/ai-relay/db/migrations/001_usage.ts +26 -0
  57. package/src/ai-relay/db/schema.ts +20 -0
  58. package/src/ai-relay/index.ts +18 -0
  59. package/src/ai-relay/metering.ts +68 -0
  60. package/src/ai-relay/usage.ts +115 -0
  61. package/src/client/ai.ts +96 -0
  62. package/src/client/api.ts +528 -0
  63. package/src/client/auth/client.ts +188 -0
  64. package/src/client/auth/failure.ts +54 -0
  65. package/src/client/auth/navigation.ts +45 -0
  66. package/src/client/device-authorization.ts +134 -0
  67. package/src/client/discovery.ts +105 -0
  68. package/src/client/images.ts +81 -0
  69. package/src/client/index.ts +53 -0
  70. package/src/client/portal/account.ts +183 -0
  71. package/src/client/portal/administration.ts +130 -0
  72. package/src/client/portal/client.ts +18 -0
  73. package/src/client/portal/request.ts +115 -0
  74. package/src/client/response.ts +51 -0
  75. package/src/client/workspaces.ts +142 -0
  76. package/src/contract/account.ts +204 -0
  77. package/src/contract/admin/index.ts +187 -0
  78. package/src/contract/ai.ts +150 -0
  79. package/src/contract/collaboration.ts +45 -0
  80. package/src/contract/discovery.ts +81 -0
  81. package/src/contract/documents.ts +265 -0
  82. package/src/contract/entitlements.ts +28 -0
  83. package/src/contract/index.ts +204 -0
  84. package/src/contract/links.ts +87 -0
  85. package/src/contract/portal.ts +11 -0
  86. package/src/contract/relay.ts +368 -0
  87. package/src/contract/sharing.ts +191 -0
  88. package/src/contract/versions.ts +87 -0
  89. package/src/contract/workspaces.ts +100 -0
  90. package/src/email/index.ts +14 -0
  91. package/src/email/render.ts +276 -0
  92. package/src/email/schema.ts +144 -0
  93. package/src/email/templates/brand.ts +105 -0
  94. package/src/email/templates/document-invitation.ts +35 -0
  95. package/src/email/templates/enrollment.ts +27 -0
  96. package/src/email/templates/layout.ts +18 -0
  97. package/src/runtime/cloudflare/email.ts +70 -0
  98. package/src/runtime/cloudflare/environment.ts +57 -0
  99. package/src/runtime/cloudflare/index.ts +3 -0
  100. package/src/runtime/cloudflare/relay.ts +136 -0
  101. package/src/runtime/cloudflare/site.ts +13 -0
  102. package/src/runtime/cloudflare/tsconfig.json +21 -0
  103. package/src/runtime/cloudflare/worker.ts +160 -0
  104. package/src/runtime/node/admin.ts +59 -0
  105. package/src/runtime/node/ai-relay/main.ts +35 -0
  106. package/src/runtime/node/ai-relay/server.ts +63 -0
  107. package/src/runtime/node/application.ts +50 -0
  108. package/src/runtime/node/bootstrap.ts +22 -0
  109. package/src/runtime/node/cleanup.ts +24 -0
  110. package/src/runtime/node/command.ts +17 -0
  111. package/src/runtime/node/config.ts +16 -0
  112. package/src/runtime/node/email/runtime.ts +43 -0
  113. package/src/runtime/node/email/transport.ts +112 -0
  114. package/src/runtime/node/entitlements/main.ts +41 -0
  115. package/src/runtime/node/entitlements/refusal.ts +15 -0
  116. package/src/runtime/node/index.ts +25 -0
  117. package/src/runtime/node/listen.ts +35 -0
  118. package/src/runtime/node/main.ts +29 -0
  119. package/src/runtime/node/migrate.ts +13 -0
  120. package/src/runtime/node/quota-reconcile.ts +9 -0
  121. package/src/runtime/node/relay.ts +54 -0
  122. package/src/runtime/node/server.ts +121 -0
  123. package/src/runtime/node/site.ts +64 -0
  124. package/src/runtime/postgres/database.ts +19 -0
  125. package/src/runtime/s3/objects.ts +244 -0
  126. package/src/runtime/site/server.ts +82 -0
  127. package/src/server/account/actions.ts +66 -0
  128. package/src/server/account/routes.ts +300 -0
  129. package/src/server/ai/errors.ts +7 -0
  130. package/src/server/ai/index.ts +4 -0
  131. package/src/server/ai/keys.ts +43 -0
  132. package/src/server/ai/routes.ts +43 -0
  133. package/src/server/ai/service.ts +70 -0
  134. package/src/server/api/index.ts +8 -0
  135. package/src/server/api/router.ts +117 -0
  136. package/src/server/api/types.ts +7 -0
  137. package/src/server/app.ts +375 -0
  138. package/src/server/auth/account.ts +73 -0
  139. package/src/server/auth/activity.ts +213 -0
  140. package/src/server/auth/adapter.ts +79 -0
  141. package/src/server/auth/email.ts +125 -0
  142. package/src/server/auth/endpoints.ts +57 -0
  143. package/src/server/auth/factory.ts +583 -0
  144. package/src/server/auth/index.ts +17 -0
  145. package/src/server/auth/mfa-assurance.ts +55 -0
  146. package/src/server/auth/runtime.ts +26 -0
  147. package/src/server/auth/session.ts +19 -0
  148. package/src/server/auth/social-providers.ts +55 -0
  149. package/src/server/auth/user-agent.ts +16 -0
  150. package/src/server/auth/workspace-access.ts +23 -0
  151. package/src/server/cleanup/configured.ts +38 -0
  152. package/src/server/cleanup/documents.ts +231 -0
  153. package/src/server/cleanup/index.ts +29 -0
  154. package/src/server/cleanup/objects.ts +27 -0
  155. package/src/server/cleanup/revisions.ts +232 -0
  156. package/src/server/cleanup/uploads.ts +129 -0
  157. package/src/server/cleanup/worker.ts +79 -0
  158. package/src/server/collaboration/index.ts +28 -0
  159. package/src/server/collaboration/relay/authorize.ts +53 -0
  160. package/src/server/collaboration/relay/awareness.ts +104 -0
  161. package/src/server/collaboration/relay/documents.ts +39 -0
  162. package/src/server/collaboration/relay/hub.ts +524 -0
  163. package/src/server/collaboration/relay/index.ts +15 -0
  164. package/src/server/collaboration/relay/persistence.ts +48 -0
  165. package/src/server/collaboration/routes.ts +77 -0
  166. package/src/server/collaboration/service.ts +188 -0
  167. package/src/server/config/deployment.ts +178 -0
  168. package/src/server/config/index.ts +20 -0
  169. package/src/server/config/limits.ts +8 -0
  170. package/src/server/config/public.ts +66 -0
  171. package/src/server/config/schema.ts +512 -0
  172. package/src/server/db/database.ts +15 -0
  173. package/src/server/db/expressions.ts +13 -0
  174. package/src/server/db/index.ts +21 -0
  175. package/src/server/db/migrate.ts +177 -0
  176. package/src/server/db/migrations/001_foundation.ts +164 -0
  177. package/src/server/db/migrations/002_upload_cleanup_claims.ts +34 -0
  178. package/src/server/db/migrations/003_document_cleanup_claims.ts +23 -0
  179. package/src/server/db/migrations/004_document_sharing.ts +82 -0
  180. package/src/server/db/migrations/005_document_collaboration_epoch.ts +13 -0
  181. package/src/server/db/migrations/006_invitation_continuation.ts +27 -0
  182. package/src/server/db/migrations/007_collaboration_state.ts +25 -0
  183. package/src/server/db/migrations/008_storage_reservations.ts +37 -0
  184. package/src/server/db/migrations/009_workspace_entitlements.ts +22 -0
  185. package/src/server/db/migrations/010_upload_finalization.ts +22 -0
  186. package/src/server/db/migrations/011_upload_finalization_lease.ts +12 -0
  187. package/src/server/db/migrations/012_transactional_email.ts +48 -0
  188. package/src/server/db/migrations/013_admin_enrollment.ts +54 -0
  189. package/src/server/db/migrations/014_enrollment_email_kinds.ts +30 -0
  190. package/src/server/db/migrations/015_cloud_rate_limit.ts +22 -0
  191. package/src/server/db/migrations/016_auth_schema.ts +16 -0
  192. package/src/server/db/migrations/017_authentication_email_kinds.ts +32 -0
  193. package/src/server/db/migrations/018_authentication_mfa.ts +78 -0
  194. package/src/server/db/migrations/019_better_auth_account_issuer.ts +42 -0
  195. package/src/server/db/migrations/020_workspace_invitations.ts +57 -0
  196. package/src/server/db/migrations/021_document_versions.ts +66 -0
  197. package/src/server/db/migrations/022_entitlement_external_version.ts +17 -0
  198. package/src/server/db/migrations/023_document_images.ts +61 -0
  199. package/src/server/db/migrations/024_account_activity.ts +66 -0
  200. package/src/server/db/migrations/025_table_names.ts +58 -0
  201. package/src/server/db/migrations/026_user_picture.ts +30 -0
  202. package/src/server/db/migrations/027_document_kind.ts +19 -0
  203. package/src/server/db/migrations/helpers.ts +78 -0
  204. package/src/server/db/tables.ts +394 -0
  205. package/src/server/documents/access.ts +103 -0
  206. package/src/server/documents/errors.ts +16 -0
  207. package/src/server/documents/images.ts +407 -0
  208. package/src/server/documents/index.ts +14 -0
  209. package/src/server/documents/limits.ts +57 -0
  210. package/src/server/documents/repository.ts +141 -0
  211. package/src/server/documents/routes.ts +239 -0
  212. package/src/server/documents/service.ts +597 -0
  213. package/src/server/documents/summary.ts +37 -0
  214. package/src/server/documents/types.ts +14 -0
  215. package/src/server/documents/versions/checkpoint.ts +98 -0
  216. package/src/server/documents/versions/history.ts +25 -0
  217. package/src/server/documents/versions/index.ts +4 -0
  218. package/src/server/documents/versions/routes.ts +112 -0
  219. package/src/server/documents/versions/service.ts +397 -0
  220. package/src/server/email/crypto.ts +57 -0
  221. package/src/server/email/index.ts +18 -0
  222. package/src/server/email/invitations.ts +30 -0
  223. package/src/server/email/service.ts +245 -0
  224. package/src/server/email/types.ts +56 -0
  225. package/src/server/email/worker.ts +33 -0
  226. package/src/server/index.ts +279 -0
  227. package/src/server/internal/index.ts +8 -0
  228. package/src/server/internal/routes.ts +41 -0
  229. package/src/server/internal/schema.ts +10 -0
  230. package/src/server/internal/service.ts +44 -0
  231. package/src/server/invitations/index.ts +6 -0
  232. package/src/server/invitations/types.ts +30 -0
  233. package/src/server/objects/index.ts +16 -0
  234. package/src/server/objects/types.ts +81 -0
  235. package/src/server/pictures/index.ts +9 -0
  236. package/src/server/pictures/provider.ts +61 -0
  237. package/src/server/pictures/routes.ts +21 -0
  238. package/src/server/pictures/service.ts +235 -0
  239. package/src/server/policy/configured.ts +23 -0
  240. package/src/server/policy/database.ts +108 -0
  241. package/src/server/policy/entitlements.ts +29 -0
  242. package/src/server/policy/index.ts +17 -0
  243. package/src/server/policy/keys.ts +23 -0
  244. package/src/server/policy/policy.ts +117 -0
  245. package/src/server/policy/provider.ts +67 -0
  246. package/src/server/policy/schema.ts +69 -0
  247. package/src/server/policy/service.ts +85 -0
  248. package/src/server/policy/static.ts +62 -0
  249. package/src/server/quota/index.ts +7 -0
  250. package/src/server/quota/reconcile.ts +61 -0
  251. package/src/server/quota/storage.ts +165 -0
  252. package/src/server/rate-limit/cleanup.ts +16 -0
  253. package/src/server/rate-limit/index.ts +12 -0
  254. package/src/server/rate-limit/keys.ts +38 -0
  255. package/src/server/rate-limit/middleware.ts +137 -0
  256. package/src/server/rate-limit/postgres.ts +79 -0
  257. package/src/server/sharing/access.ts +29 -0
  258. package/src/server/sharing/actor.ts +12 -0
  259. package/src/server/sharing/capabilities/service.ts +262 -0
  260. package/src/server/sharing/continuation.ts +85 -0
  261. package/src/server/sharing/contracts.ts +80 -0
  262. package/src/server/sharing/crypto.ts +48 -0
  263. package/src/server/sharing/document-routes.ts +41 -0
  264. package/src/server/sharing/errors.ts +12 -0
  265. package/src/server/sharing/grants/service.ts +79 -0
  266. package/src/server/sharing/http.ts +41 -0
  267. package/src/server/sharing/index.ts +39 -0
  268. package/src/server/sharing/invitations/service.ts +315 -0
  269. package/src/server/sharing/policy.ts +44 -0
  270. package/src/server/sharing/routes.ts +251 -0
  271. package/src/server/sharing/service.ts +117 -0
  272. package/src/server/validation.ts +24 -0
  273. package/src/server/worker/interval.ts +23 -0
  274. package/src/server/worker/periodic.ts +42 -0
  275. package/src/server/workspaces/index.ts +16 -0
  276. package/src/server/workspaces/members.ts +502 -0
  277. package/src/server/workspaces/repository.ts +91 -0
  278. package/src/server/workspaces/routes.ts +198 -0
  279. package/src/server/workspaces/service.ts +84 -0
@@ -0,0 +1,19 @@
1
+ import type { CloudAuthAdapter, CloudIdentity } from '#cloud/server/auth/adapter'
2
+
3
+ export type CloudActor = {
4
+ userId: string
5
+ email: string
6
+ name: string
7
+ deploymentRole?: 'user' | 'admin'
8
+ }
9
+
10
+ export type CloudIdentityResolver = (request: Request) => Promise<CloudIdentity | null>
11
+ export type CloudSessionResolver = (request: Request) => Promise<CloudActor | null>
12
+
13
+ export function createCloudIdentityResolver(auth: CloudAuthAdapter): CloudIdentityResolver {
14
+ return (request) => auth.resolveIdentity(request.headers)
15
+ }
16
+
17
+ export function createCloudSessionResolver(auth: CloudAuthAdapter): CloudSessionResolver {
18
+ return (request) => auth.resolveSession(request.headers)
19
+ }
@@ -0,0 +1,55 @@
1
+ import type { CloudServerConfig } from '#cloud/server/config'
2
+ import type { BetterAuthOptions } from 'better-auth'
3
+ import { importPKCS8, SignJWT } from 'jose'
4
+
5
+ /** The audience Apple's sign-in tokens carry, which the server also trusts as an origin. */
6
+ export const APPLE_AUDIENCE = 'https://appleid.apple.com'
7
+ const APPLE_SECRET_LIFETIME_SECONDS = 180 * 24 * 60 * 60
8
+ async function generateAppleClientSecret(config: CloudServerConfig): Promise<string> {
9
+ if (
10
+ !config.appleClientId ||
11
+ !config.appleTeamId ||
12
+ !config.appleKeyId ||
13
+ !config.applePrivateKey
14
+ ) {
15
+ throw new Error('Apple authentication is not configured')
16
+ }
17
+ const privateKey = await importPKCS8(config.applePrivateKey.replaceAll('\\n', '\n'), 'ES256')
18
+ const issuedAt = Math.floor(Date.now() / 1000)
19
+ return new SignJWT({})
20
+ .setProtectedHeader({ alg: 'ES256', kid: config.appleKeyId })
21
+ .setIssuer(config.appleTeamId)
22
+ .setSubject(config.appleClientId)
23
+ .setAudience(APPLE_AUDIENCE)
24
+ .setIssuedAt(issuedAt)
25
+ .setExpirationTime(issuedAt + APPLE_SECRET_LIFETIME_SECONDS)
26
+ .sign(privateKey)
27
+ }
28
+
29
+ /** The sign-in providers the deployment configures; Apple's secret is signed when it is needed. */
30
+ export function socialProviders(config: CloudServerConfig): BetterAuthOptions['socialProviders'] {
31
+ const providers: BetterAuthOptions['socialProviders'] = {}
32
+ if (config.googleClientId && config.googleClientSecret) {
33
+ providers.google = {
34
+ clientId: config.googleClientId,
35
+ clientSecret: config.googleClientSecret
36
+ }
37
+ }
38
+ if (config.appleClientId) {
39
+ providers.apple = async () => {
40
+ const provider: {
41
+ clientId: string
42
+ clientSecret: string
43
+ appBundleIdentifier?: string
44
+ } = {
45
+ clientId: config.appleClientId ?? '',
46
+ clientSecret: await generateAppleClientSecret(config)
47
+ }
48
+ if (config.appleAppBundleIdentifier) {
49
+ provider.appBundleIdentifier = config.appleAppBundleIdentifier
50
+ }
51
+ return provider
52
+ }
53
+ }
54
+ return providers
55
+ }
@@ -0,0 +1,16 @@
1
+ import Bowser from 'bowser'
2
+ import { compact } from 'es-toolkit'
3
+
4
+ /** The browser and system a user agent names, as words for a person: `Chrome 141`, `macOS`. */
5
+ export function describeUserAgent(userAgent: string | null | undefined): {
6
+ browser: string | null
7
+ system: string | null
8
+ } {
9
+ if (!userAgent) return { browser: null, system: null }
10
+ const { browser, os } = Bowser.parse(userAgent)
11
+ const version = browser.version?.split('.', 1)[0]
12
+ return {
13
+ browser: browser.name ? compact([browser.name, version]).join(' ') : null,
14
+ system: os.name || null
15
+ }
16
+ }
@@ -0,0 +1,23 @@
1
+ import { createAccessControl } from 'better-auth/plugins/access'
2
+
3
+ export const workspaceStatements = {
4
+ workspace: ['read', 'update', 'delete', 'invite', 'manage-members'],
5
+ document: ['create', 'read', 'edit', 'share', 'delete']
6
+ } as const
7
+
8
+ export const workspaceAccessControl = createAccessControl(workspaceStatements)
9
+
10
+ export const workspaceRoles = {
11
+ admin: workspaceAccessControl.newRole({
12
+ workspace: ['read', 'update', 'delete', 'invite', 'manage-members'],
13
+ document: ['create', 'read', 'edit', 'share', 'delete']
14
+ }),
15
+ editor: workspaceAccessControl.newRole({
16
+ workspace: ['read'],
17
+ document: ['create', 'read', 'edit', 'share']
18
+ }),
19
+ viewer: workspaceAccessControl.newRole({
20
+ workspace: ['read'],
21
+ document: ['read']
22
+ })
23
+ }
@@ -0,0 +1,38 @@
1
+ import type { CloudServerConfig } from '#cloud/server/config'
2
+ import type { CloudDatabase } from '#cloud/server/db'
3
+ import type { ObjectStore } from '#cloud/server/objects'
4
+ import type { CloudPolicy } from '#cloud/server/policy/policy'
5
+ import { createRateLimitCleanupService } from '#cloud/server/rate-limit/cleanup'
6
+ import type { Kysely } from 'kysely'
7
+
8
+ import { createDocumentCleanupService } from './documents'
9
+ import { createRevisionCompactionService } from './revisions'
10
+ import { createUploadCleanupService } from './uploads'
11
+ import type { CleanupPassOptions, CleanupServices } from './worker'
12
+
13
+ /** Every cleanup service, over a deployment's database and bucket. */
14
+ export function createCleanupServices(
15
+ config: Pick<CloudServerConfig, 'deployment'>,
16
+ database: Kysely<CloudDatabase>,
17
+ objects: ObjectStore,
18
+ policy: CloudPolicy
19
+ ): CleanupServices {
20
+ return {
21
+ uploads: createUploadCleanupService(database, objects),
22
+ documents: createDocumentCleanupService(database, objects),
23
+ revisions: createRevisionCompactionService(database, objects, {
24
+ policy,
25
+ deploymentMode: config.deployment
26
+ }),
27
+ rateLimits: createRateLimitCleanupService(database)
28
+ }
29
+ }
30
+
31
+ /** The batch, lease, and retention a deployment sets for each cleanup pass. */
32
+ export function cleanupPassOptions(config: CloudServerConfig): CleanupPassOptions {
33
+ return {
34
+ batchSize: config.cleanupBatchSize,
35
+ documentRetentionMs: config.documentRetentionMs,
36
+ leaseDurationMs: config.cleanupLeaseDurationMs
37
+ }
38
+ }
@@ -0,0 +1,231 @@
1
+ import type { CloudDatabase } from '#cloud/server/db'
2
+ import type { ObjectStore } from '#cloud/server/objects'
3
+ import type { Kysely } from 'kysely'
4
+
5
+ import { unreferencedObject } from './objects'
6
+
7
+ export type DocumentCleanupOptions = {
8
+ batchSize: number
9
+ leaseDurationMs: number
10
+ retentionMs: number
11
+ now?: Date
12
+ }
13
+
14
+ export type DocumentCleanupResult = {
15
+ claimed: number
16
+ cleaned: number
17
+ failed: number
18
+ objectsDeleted: number
19
+ bytesReclaimed: number
20
+ }
21
+
22
+ export type DocumentCleanupService = {
23
+ cleanupDeletedDocuments(options: DocumentCleanupOptions): Promise<DocumentCleanupResult>
24
+ }
25
+
26
+ type ClaimedDocument = {
27
+ id: string
28
+ workspaceId: string
29
+ deletedAt: Date
30
+ }
31
+
32
+ type StoredRevisionObject = {
33
+ id: string
34
+ objectKey: string
35
+ byteSize: number
36
+ }
37
+
38
+ export function createDocumentCleanupService(
39
+ database: Kysely<CloudDatabase>,
40
+ objects: ObjectStore
41
+ ): DocumentCleanupService {
42
+ return {
43
+ async cleanupDeletedDocuments(options) {
44
+ const now = options.now ?? new Date()
45
+ const retentionBefore = new Date(now.getTime() - options.retentionMs)
46
+ const staleBefore = new Date(now.getTime() - options.leaseDurationMs)
47
+ const claimId = crypto.randomUUID()
48
+ const documents = await claimDocuments(
49
+ database,
50
+ options.batchSize,
51
+ retentionBefore,
52
+ staleBefore,
53
+ now,
54
+ claimId
55
+ )
56
+ const result: DocumentCleanupResult = {
57
+ claimed: documents.length,
58
+ cleaned: 0,
59
+ failed: 0,
60
+ objectsDeleted: 0,
61
+ bytesReclaimed: 0
62
+ }
63
+ for (const document of documents) {
64
+ try {
65
+ const revisionObjects = await documentObjects(database, document.id)
66
+ for (const object of revisionObjects) await objects.delete(object.objectKey)
67
+ const reclaimed = await removeDocumentMetadata(
68
+ database,
69
+ document,
70
+ revisionObjects,
71
+ retentionBefore,
72
+ claimId
73
+ )
74
+ if (!reclaimed) continue
75
+ result.cleaned++
76
+ result.objectsDeleted += reclaimed.objectsDeleted
77
+ result.bytesReclaimed += reclaimed.bytesReclaimed
78
+ } catch {
79
+ result.failed++
80
+ await releaseClaim(database, document.id, claimId)
81
+ }
82
+ }
83
+ return result
84
+ }
85
+ }
86
+ }
87
+
88
+ async function claimDocuments(
89
+ database: Kysely<CloudDatabase>,
90
+ batchSize: number,
91
+ retentionBefore: Date,
92
+ staleBefore: Date,
93
+ now: Date,
94
+ claimId: string
95
+ ): Promise<ClaimedDocument[]> {
96
+ return database.transaction().execute(async (transaction) => {
97
+ const candidates = await transaction
98
+ .selectFrom('document')
99
+ .select(['id', 'workspaceId', 'deletedAt'])
100
+ .where('deletedAt', 'is not', null)
101
+ .where('deletedAt', '<=', retentionBefore)
102
+ .where((expression) =>
103
+ expression.or([
104
+ expression('cleanupClaimId', 'is', null),
105
+ expression('cleanupClaimedAt', '<=', staleBefore)
106
+ ])
107
+ )
108
+ .where((expression) =>
109
+ expression.not(
110
+ expression.exists(
111
+ expression
112
+ .selectFrom('upload')
113
+ .select('upload.id')
114
+ .whereRef('upload.documentId', '=', 'document.id')
115
+ .where('upload.status', 'in', ['pending', 'cleaning'])
116
+ )
117
+ )
118
+ )
119
+ .orderBy('deletedAt')
120
+ .limit(batchSize)
121
+ .forUpdate()
122
+ .skipLocked()
123
+ .execute()
124
+ const claimed = candidates.filter(
125
+ (document): document is ClaimedDocument => document.deletedAt !== null
126
+ )
127
+ if (claimed.length === 0) return []
128
+ await transaction
129
+ .updateTable('document')
130
+ .set({ cleanupClaimId: claimId, cleanupClaimedAt: now })
131
+ .where(
132
+ 'id',
133
+ 'in',
134
+ claimed.map((document) => document.id)
135
+ )
136
+ .execute()
137
+ return claimed
138
+ })
139
+ }
140
+
141
+ async function documentObjects(
142
+ database: Kysely<CloudDatabase>,
143
+ documentId: string
144
+ ): Promise<StoredRevisionObject[]> {
145
+ const rows = await database
146
+ .selectFrom('documentRevision')
147
+ .innerJoin('storageObject', 'storageObject.id', 'documentRevision.storageObjectId')
148
+ .select(['storageObject.id', 'storageObject.objectKey', 'storageObject.byteSize'])
149
+ .where('documentRevision.documentId', '=', documentId)
150
+ .where((expression) =>
151
+ expression.not(
152
+ expression.exists(
153
+ expression
154
+ .selectFrom('documentRevision as otherRevision')
155
+ .select('otherRevision.id')
156
+ .whereRef('otherRevision.storageObjectId', '=', 'storageObject.id')
157
+ .where('otherRevision.documentId', '!=', documentId)
158
+ )
159
+ )
160
+ )
161
+ .execute()
162
+ return rows.map((row) => ({ ...row, byteSize: Number(row.byteSize) }))
163
+ }
164
+
165
+ async function removeDocumentMetadata(
166
+ database: Kysely<CloudDatabase>,
167
+ document: ClaimedDocument,
168
+ objects: StoredRevisionObject[],
169
+ retentionBefore: Date,
170
+ claimId: string
171
+ ): Promise<{ objectsDeleted: number; bytesReclaimed: number } | null> {
172
+ return database.transaction().execute(async (transaction) => {
173
+ const locked = await transaction
174
+ .selectFrom('document')
175
+ .select(['deletedAt', 'cleanupClaimId'])
176
+ .where('id', '=', document.id)
177
+ .forUpdate()
178
+ .executeTakeFirst()
179
+ if (
180
+ !locked?.deletedAt ||
181
+ locked.deletedAt > retentionBefore ||
182
+ locked.cleanupClaimId !== claimId
183
+ ) {
184
+ return null
185
+ }
186
+ await transaction
187
+ .updateTable('document')
188
+ .set({ currentRevisionId: null })
189
+ .where('id', '=', document.id)
190
+ .execute()
191
+ await transaction.deleteFrom('documentRevision').where('documentId', '=', document.id).execute()
192
+ let objectsDeleted = 0
193
+ let bytesReclaimed = 0
194
+ for (const object of objects) {
195
+ const deleted = await transaction
196
+ .deleteFrom('storageObject')
197
+ .where('id', '=', object.id)
198
+ .where(unreferencedObject)
199
+ .executeTakeFirst()
200
+ if (Number(deleted.numDeletedRows) > 0) {
201
+ objectsDeleted++
202
+ bytesReclaimed += object.byteSize
203
+ }
204
+ }
205
+ if (bytesReclaimed > 0) {
206
+ await transaction
207
+ .updateTable('workspaceStorageUsage')
208
+ .set({
209
+ committedBytes: (expression) => expression('committedBytes', '-', bytesReclaimed),
210
+ updatedAt: new Date()
211
+ })
212
+ .where('workspaceId', '=', document.workspaceId)
213
+ .execute()
214
+ }
215
+ await transaction.deleteFrom('document').where('id', '=', document.id).execute()
216
+ return { objectsDeleted, bytesReclaimed }
217
+ })
218
+ }
219
+
220
+ async function releaseClaim(
221
+ database: Kysely<CloudDatabase>,
222
+ documentId: string,
223
+ claimId: string
224
+ ): Promise<void> {
225
+ await database
226
+ .updateTable('document')
227
+ .set({ cleanupClaimId: null, cleanupClaimedAt: null })
228
+ .where('id', '=', documentId)
229
+ .where('cleanupClaimId', '=', claimId)
230
+ .execute()
231
+ }
@@ -0,0 +1,29 @@
1
+ export { cleanupPassOptions, createCleanupServices } from './configured'
2
+ export {
3
+ createDocumentCleanupService,
4
+ type DocumentCleanupOptions,
5
+ type DocumentCleanupResult,
6
+ type DocumentCleanupService
7
+ } from './documents'
8
+ export {
9
+ createRevisionCompactionService,
10
+ REVISION_COMPACTION_GRACE_MS,
11
+ type RevisionCompactionOptions,
12
+ type RevisionCompactionResult,
13
+ type RevisionCompactionService
14
+ } from './revisions'
15
+ export {
16
+ createUploadCleanupService,
17
+ type UploadCleanupOptions,
18
+ type UploadCleanupResult,
19
+ type UploadCleanupService
20
+ } from './uploads'
21
+ export {
22
+ runCleanup,
23
+ startCleanupWorker,
24
+ type CleanupPassOptions,
25
+ type CleanupResult,
26
+ type CleanupServices,
27
+ type CleanupWorker,
28
+ type CleanupWorkerOptions
29
+ } from './worker'
@@ -0,0 +1,27 @@
1
+ import type { CloudDatabase } from '#cloud/server/db'
2
+ import type { ExpressionBuilder } from 'kysely'
3
+
4
+ /**
5
+ * No revision refers to the stored file any more, and no workspace keeps it as an image, so it
6
+ * can leave the object store.
7
+ */
8
+ export function unreferencedObject(expression: ExpressionBuilder<CloudDatabase, 'storageObject'>) {
9
+ return expression.and([
10
+ expression.not(
11
+ expression.exists(
12
+ expression
13
+ .selectFrom('documentRevision')
14
+ .select('documentRevision.id')
15
+ .whereRef('documentRevision.storageObjectId', '=', 'storageObject.id')
16
+ )
17
+ ),
18
+ expression.not(
19
+ expression.exists(
20
+ expression
21
+ .selectFrom('workspaceImage')
22
+ .select('workspaceImage.checksum')
23
+ .whereRef('workspaceImage.storageObjectId', '=', 'storageObject.id')
24
+ )
25
+ )
26
+ ])
27
+ }
@@ -0,0 +1,232 @@
1
+ import type { CloudDatabase } from '#cloud/server/db'
2
+ import { IMAGE_RELEASE_GRACE_MS, releaseUnusedImages } from '#cloud/server/documents/images'
3
+ import { historyStart, LIMITED_HISTORY_DAYS } from '#cloud/server/documents/versions/history'
4
+ import type { ObjectStore } from '#cloud/server/objects'
5
+ import type { CloudPolicyContext } from '#cloud/server/policy/keys'
6
+ import type { CloudPolicy } from '#cloud/server/policy/policy'
7
+ import { groupBy, sumBy, uniq } from 'es-toolkit'
8
+ import type { Kysely, Transaction } from 'kysely'
9
+
10
+ import { unreferencedObject } from './objects'
11
+
12
+ /** How long a save between versions is kept before compaction may delete it. */
13
+ export const REVISION_COMPACTION_GRACE_MS = 24 * 60 * 60_000
14
+
15
+ export type RevisionCompactionOptions = {
16
+ batchSize: number
17
+ /** How long a save stays before it can be compacted, so recent work is never lost to it. */
18
+ graceMs: number
19
+ now?: Date
20
+ }
21
+
22
+ export type RevisionCompactionResult = {
23
+ versionsPruned: number
24
+ /** Images no revision used any more, released after their grace period. */
25
+ imagesReleased: number
26
+ revisionsDeleted: number
27
+ objectsDeleted: number
28
+ bytesReclaimed: number
29
+ failed: number
30
+ }
31
+
32
+ export type RevisionCompactionService = {
33
+ compactRevisions(options: RevisionCompactionOptions): Promise<RevisionCompactionResult>
34
+ }
35
+
36
+ /**
37
+ * Keeps a document's stored history to its versions. A save that is neither the document's
38
+ * current state nor kept by a version is deleted once it is older than the grace period, and its
39
+ * file with it when nothing else uses it; a workspace without full history first loses the
40
+ * versions older than its 30 days.
41
+ */
42
+ export function createRevisionCompactionService(
43
+ database: Kysely<CloudDatabase>,
44
+ objects: ObjectStore,
45
+ options: { policy?: CloudPolicy; deploymentMode?: CloudPolicyContext['deploymentMode'] } = {}
46
+ ): RevisionCompactionService {
47
+ const deploymentMode = options.deploymentMode ?? 'self-hosted'
48
+ async function pruneVersions(now: Date, batchSize: number): Promise<number> {
49
+ if (!options.policy) return 0
50
+ const limitedBefore = new Date(now.getTime() - LIMITED_HISTORY_DAYS * 24 * 60 * 60_000)
51
+ const workspaces = await database
52
+ .selectFrom('documentVersion')
53
+ .innerJoin('document', 'document.id', 'documentVersion.documentId')
54
+ .select('document.workspaceId')
55
+ .distinct()
56
+ .where('documentVersion.createdAt', '<', limitedBefore)
57
+ .execute()
58
+ let pruned = 0
59
+ for (const { workspaceId } of workspaces) {
60
+ const start = await historyStart(options.policy, deploymentMode, workspaceId, now)
61
+ if (!start) continue
62
+ const deleted = await database
63
+ .deleteFrom('documentVersion')
64
+ .where('id', 'in', (query) =>
65
+ query
66
+ .selectFrom('documentVersion as version')
67
+ .innerJoin('document', 'document.id', 'version.documentId')
68
+ .select('version.id')
69
+ .where('document.workspaceId', '=', workspaceId)
70
+ .where('version.createdAt', '<', start)
71
+ // The document's current state stays listed however old it is.
72
+ .whereRef('version.revisionId', '!=', 'document.currentRevisionId')
73
+ .limit(batchSize)
74
+ )
75
+ .executeTakeFirst()
76
+ pruned += Number(deleted.numDeletedRows)
77
+ }
78
+ return pruned
79
+ }
80
+
81
+ /** Saves that may go: old, unversioned, not current, in a document that is not deleted. */
82
+ function compactable(source: Kysely<CloudDatabase> | Transaction<CloudDatabase>, before: Date) {
83
+ return source
84
+ .selectFrom('documentRevision')
85
+ .innerJoin('document', 'document.id', 'documentRevision.documentId')
86
+ .where('documentRevision.createdAt', '<', before)
87
+ .where('document.deletedAt', 'is', null)
88
+ .where((expression) =>
89
+ expression.or([
90
+ expression('document.currentRevisionId', 'is', null),
91
+ expression('document.currentRevisionId', '!=', expression.ref('documentRevision.id'))
92
+ ])
93
+ )
94
+ .where((expression) =>
95
+ expression.not(
96
+ expression.exists(
97
+ expression
98
+ .selectFrom('documentVersion')
99
+ .select('documentVersion.id')
100
+ .whereRef('documentVersion.revisionId', '=', 'documentRevision.id')
101
+ )
102
+ )
103
+ )
104
+ }
105
+
106
+ /** A file no revision refers to: freed by compaction, waiting for the object store. */
107
+ function unreferencedObjects(source: Kysely<CloudDatabase> | Transaction<CloudDatabase>) {
108
+ return source.selectFrom('storageObject').where(unreferencedObject)
109
+ }
110
+
111
+ /**
112
+ * Deletes up to `limit` compactable saves of one document and returns the bytes of files nothing
113
+ * refers to any more to the workspace's usage. Their rows stay until the files are gone, so a
114
+ * delete that fails is tried again on the next pass.
115
+ */
116
+ async function compactDocument(
117
+ documentId: string,
118
+ workspaceId: string,
119
+ before: Date,
120
+ limit: number
121
+ ) {
122
+ return database.transaction().execute(async (transaction) => {
123
+ // Saves and restores lock the document too, so none can keep a revision deleted here.
124
+ await transaction
125
+ .selectFrom('document')
126
+ .select('id')
127
+ .where('id', '=', documentId)
128
+ .forUpdate()
129
+ .executeTakeFirst()
130
+ const revisions = await compactable(transaction, before)
131
+ .select(['documentRevision.id', 'documentRevision.storageObjectId'])
132
+ .where('documentRevision.documentId', '=', documentId)
133
+ .limit(limit)
134
+ .execute()
135
+ if (revisions.length === 0) return { revisions: 0, bytes: 0 }
136
+ await transaction
137
+ .deleteFrom('documentRevision')
138
+ .where(
139
+ 'id',
140
+ 'in',
141
+ revisions.map((revision) => revision.id)
142
+ )
143
+ .execute()
144
+ const freed = await unreferencedObjects(transaction)
145
+ .select('byteSize')
146
+ .where('id', 'in', uniq(revisions.map((revision) => revision.storageObjectId)))
147
+ .execute()
148
+ const bytes = sumBy(freed, (object) => Number(object.byteSize))
149
+ if (bytes > 0) {
150
+ await transaction
151
+ .updateTable('workspaceStorageUsage')
152
+ .set({
153
+ committedBytes: (expression) => expression('committedBytes', '-', bytes),
154
+ updatedAt: new Date()
155
+ })
156
+ .where('workspaceId', '=', workspaceId)
157
+ .execute()
158
+ }
159
+ return { revisions: revisions.length, bytes }
160
+ })
161
+ }
162
+
163
+ /** Deletes freed files from the object store, then their rows; a failure stays for next time. */
164
+ async function sweepObjects(batchSize: number, result: RevisionCompactionResult) {
165
+ const freed = await unreferencedObjects(database)
166
+ .select(['id', 'objectKey'])
167
+ .limit(batchSize)
168
+ .execute()
169
+ for (const object of freed) {
170
+ try {
171
+ await objects.delete(object.objectKey)
172
+ await database
173
+ .deleteFrom('storageObject')
174
+ .where('id', '=', object.id)
175
+ .where(unreferencedObject)
176
+ .execute()
177
+ result.objectsDeleted++
178
+ } catch {
179
+ result.failed++
180
+ }
181
+ }
182
+ }
183
+
184
+ return {
185
+ async compactRevisions(input) {
186
+ const now = input.now ?? new Date()
187
+ const before = new Date(now.getTime() - input.graceMs)
188
+ const result: RevisionCompactionResult = {
189
+ versionsPruned: await pruneVersions(now, input.batchSize),
190
+ imagesReleased: 0,
191
+ revisionsDeleted: 0,
192
+ objectsDeleted: 0,
193
+ bytesReclaimed: 0,
194
+ failed: 0
195
+ }
196
+ const candidates = await compactable(database, before)
197
+ .select(['documentRevision.documentId', 'document.workspaceId'])
198
+ .limit(input.batchSize)
199
+ .execute()
200
+ let capacity = input.batchSize
201
+ for (const [documentId, rows] of Object.entries(
202
+ groupBy(candidates, (row) => row.documentId)
203
+ )) {
204
+ const workspaceId = rows[0]?.workspaceId
205
+ if (!workspaceId || capacity <= 0) continue
206
+ try {
207
+ const compacted = await compactDocument(documentId, workspaceId, before, capacity)
208
+ capacity -= compacted.revisions
209
+ result.revisionsDeleted += compacted.revisions
210
+ result.bytesReclaimed += compacted.bytes
211
+ } catch {
212
+ result.failed++
213
+ }
214
+ }
215
+ // Compaction just freed the images of the saves it deleted; the release lets their grace pass.
216
+ // A release that raced a save referring to an image fails whole and is tried next time.
217
+ try {
218
+ const images = await releaseUnusedImages(
219
+ database,
220
+ new Date(now.getTime() - IMAGE_RELEASE_GRACE_MS),
221
+ input.batchSize
222
+ )
223
+ result.imagesReleased = images.released
224
+ result.bytesReclaimed += images.bytes
225
+ } catch {
226
+ result.failed++
227
+ }
228
+ await sweepObjects(input.batchSize, result)
229
+ return result
230
+ }
231
+ }
232
+ }