@open-pencil/cloud 0.17.0-beta.1 → 0.17.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (235) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/{ai-relay.mjs → services.mjs} +8 -8
  3. package/dist/bin/services.mjs.map +1 -0
  4. package/dist/client.d.ts +236 -46
  5. package/dist/client.d.ts.map +1 -1
  6. package/dist/client.js +649 -400
  7. package/dist/client.js.map +1 -1
  8. package/dist/contract.d.ts +2 -2
  9. package/dist/contract.js +2 -2
  10. package/dist/contract2.js +680 -443
  11. package/dist/contract2.js.map +1 -1
  12. package/dist/email.d.ts +9 -1
  13. package/dist/email.d.ts.map +1 -1
  14. package/dist/email.js +99 -58
  15. package/dist/email.js.map +1 -1
  16. package/dist/index.d.ts +468 -64
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/listen.mjs +575 -738
  19. package/dist/listen.mjs.map +1 -1
  20. package/dist/runtime-node.d.mts +6564 -2934
  21. package/dist/runtime-node.d.mts.map +1 -1
  22. package/dist/runtime-node.mjs +6 -5
  23. package/dist/runtime-node.mjs.map +1 -1
  24. package/dist/server.d.ts +8931 -4198
  25. package/dist/server.d.ts.map +1 -1
  26. package/dist/server.js +5260 -3859
  27. package/dist/server.js.map +1 -1
  28. package/dist/server.mjs +5686 -3736
  29. package/dist/server.mjs.map +1 -1
  30. package/dist/server2.mjs +101 -79
  31. package/dist/server2.mjs.map +1 -1
  32. package/package.json +9 -6
  33. package/src/admin/api/routes.ts +103 -33
  34. package/src/admin/audit/service.ts +25 -6
  35. package/src/admin/documents/service.ts +361 -0
  36. package/src/admin/errors.ts +9 -1
  37. package/src/admin/index.ts +6 -0
  38. package/src/admin/operations/service.ts +7 -0
  39. package/src/admin/reports/service.ts +147 -0
  40. package/src/admin/schema.ts +6 -0
  41. package/src/client/api.ts +36 -21
  42. package/src/client/auth/navigation.ts +6 -5
  43. package/src/client/index.ts +7 -7
  44. package/src/client/portal/account.ts +25 -0
  45. package/src/client/portal/administration.ts +56 -0
  46. package/src/client/{ai.ts → services.ts} +44 -27
  47. package/src/contract/account.ts +58 -2
  48. package/src/contract/admin/index.ts +67 -3
  49. package/src/contract/collaboration.ts +6 -1
  50. package/src/contract/discovery.ts +22 -4
  51. package/src/contract/documents.ts +27 -0
  52. package/src/contract/entitlements.ts +2 -2
  53. package/src/contract/index.ts +63 -37
  54. package/src/contract/relay.ts +6 -0
  55. package/src/contract/reports.ts +42 -0
  56. package/src/contract/services.ts +233 -0
  57. package/src/contract/sharing.ts +3 -9
  58. package/src/email/render.ts +26 -0
  59. package/src/email/schema.ts +11 -1
  60. package/src/email/templates/brand.ts +1 -1
  61. package/src/runtime/cloudflare/environment.ts +31 -6
  62. package/src/runtime/cloudflare/index.ts +1 -1
  63. package/src/runtime/cloudflare/logging.ts +15 -0
  64. package/src/runtime/cloudflare/maintenance.ts +31 -0
  65. package/src/runtime/cloudflare/relay/index.ts +176 -0
  66. package/src/runtime/cloudflare/relay/sockets.ts +112 -0
  67. package/src/runtime/cloudflare/worker.ts +89 -26
  68. package/src/runtime/node/application.ts +5 -3
  69. package/src/runtime/node/bootstrap.ts +20 -4
  70. package/src/runtime/node/cleanup.ts +16 -3
  71. package/src/runtime/node/entitlements/main.ts +14 -11
  72. package/src/runtime/node/index.ts +1 -1
  73. package/src/runtime/node/migrate.ts +5 -1
  74. package/src/runtime/node/relay.ts +8 -2
  75. package/src/runtime/node/request-id.ts +13 -0
  76. package/src/runtime/node/server.ts +71 -15
  77. package/src/runtime/node/{ai-relay → services}/main.ts +6 -6
  78. package/src/runtime/node/{ai-relay → services}/server.ts +18 -18
  79. package/src/runtime/postgres/database.ts +20 -1
  80. package/src/runtime/posthog/errors.ts +32 -0
  81. package/src/runtime/s3/objects.ts +50 -1
  82. package/src/runtime/site/server.ts +33 -4
  83. package/src/server/account/actions.ts +5 -0
  84. package/src/server/account/deletion.ts +302 -0
  85. package/src/server/account/routes.ts +47 -3
  86. package/src/server/account/terms.ts +42 -0
  87. package/src/server/api/router.ts +12 -7
  88. package/src/server/api/types.ts +9 -0
  89. package/src/server/app.ts +93 -15
  90. package/src/server/auth/access-control.ts +63 -0
  91. package/src/server/auth/adapter.ts +11 -0
  92. package/src/server/auth/endpoints.ts +5 -1
  93. package/src/server/auth/factory.ts +30 -3
  94. package/src/server/auth/index.ts +10 -1
  95. package/src/server/cleanup/configured.ts +12 -5
  96. package/src/server/cleanup/documents.ts +165 -71
  97. package/src/server/cleanup/objects.ts +10 -2
  98. package/src/server/cleanup/revisions.ts +24 -7
  99. package/src/server/cleanup/uploads.ts +8 -2
  100. package/src/server/cleanup/worker.ts +5 -2
  101. package/src/server/collaboration/index.ts +3 -0
  102. package/src/server/collaboration/relay/connection.ts +196 -0
  103. package/src/server/collaboration/relay/documents.ts +13 -10
  104. package/src/server/collaboration/relay/hub.ts +279 -195
  105. package/src/server/collaboration/relay/index.ts +3 -1
  106. package/src/server/collaboration/relay/log.ts +79 -0
  107. package/src/server/collaboration/relay/persistence.ts +147 -22
  108. package/src/server/collaboration/relay/resume.ts +28 -0
  109. package/src/server/collaboration/routes.ts +27 -16
  110. package/src/server/collaboration/service.ts +27 -31
  111. package/src/server/config/deployment.ts +35 -4
  112. package/src/server/config/public.ts +5 -4
  113. package/src/server/config/schema.ts +104 -16
  114. package/src/server/db/expressions.ts +15 -1
  115. package/src/server/db/migrate.ts +34 -114
  116. package/src/server/db/migrations/001_accounts.ts +79 -0
  117. package/src/server/db/migrations/002_workspaces.ts +48 -0
  118. package/src/server/db/migrations/003_storage.ts +44 -0
  119. package/src/server/db/migrations/004_documents.ts +193 -0
  120. package/src/server/db/migrations/005_sharing.ts +95 -0
  121. package/src/server/db/migrations/006_operations.ts +110 -0
  122. package/src/server/db/migrations/007_trash.ts +33 -0
  123. package/src/server/db/migrations/008_terms.ts +18 -0
  124. package/src/server/db/migrations/schema.ts +71 -0
  125. package/src/server/db/tables.ts +70 -7
  126. package/src/server/documents/access.ts +66 -55
  127. package/src/server/documents/errors.ts +7 -0
  128. package/src/server/documents/images.ts +2 -1
  129. package/src/server/documents/limits.ts +9 -9
  130. package/src/server/documents/purge.ts +75 -0
  131. package/src/server/documents/repository.ts +2 -0
  132. package/src/server/documents/routes.ts +132 -80
  133. package/src/server/documents/service.ts +18 -4
  134. package/src/server/documents/summary.ts +1 -0
  135. package/src/server/documents/trash.ts +203 -0
  136. package/src/server/documents/versions/checkpoint.ts +2 -1
  137. package/src/server/documents/versions/history.ts +11 -7
  138. package/src/server/documents/versions/routes.ts +36 -25
  139. package/src/server/documents/versions/service.ts +22 -4
  140. package/src/server/errors/index.ts +99 -0
  141. package/src/server/errors/posthog.ts +80 -0
  142. package/src/server/index.ts +56 -16
  143. package/src/server/internal/index.ts +3 -3
  144. package/src/server/internal/routes.ts +7 -15
  145. package/src/server/internal/schema.ts +1 -1
  146. package/src/server/internal/service.ts +17 -23
  147. package/src/server/invitations/budget.ts +66 -0
  148. package/src/server/invitations/index.ts +1 -0
  149. package/src/server/logging.ts +76 -0
  150. package/src/server/objects/types.ts +12 -0
  151. package/src/server/pictures/routes.ts +4 -2
  152. package/src/server/pictures/service.ts +2 -1
  153. package/src/server/policy/database.ts +39 -25
  154. package/src/server/policy/index.ts +2 -1
  155. package/src/server/policy/keys.ts +5 -2
  156. package/src/server/policy/policy.ts +1 -0
  157. package/src/server/policy/schema.ts +2 -2
  158. package/src/server/policy/service.ts +12 -9
  159. package/src/server/policy/static.ts +5 -5
  160. package/src/server/policy/workspace.ts +41 -0
  161. package/src/server/quota/reconcile.ts +2 -0
  162. package/src/server/quota/storage.ts +69 -31
  163. package/src/server/rate-limit/cleanup.ts +17 -5
  164. package/src/server/rate-limit/index.ts +4 -1
  165. package/src/server/rate-limit/middleware.ts +100 -12
  166. package/src/server/reports/index.ts +8 -0
  167. package/src/server/reports/routes.ts +37 -0
  168. package/src/server/reports/service.ts +123 -0
  169. package/src/server/services/errors.ts +7 -0
  170. package/src/server/services/index.ts +8 -0
  171. package/src/server/{ai → services}/keys.ts +11 -11
  172. package/src/server/services/routes.ts +47 -0
  173. package/src/server/{ai/service.ts → services/tokens.ts} +23 -20
  174. package/src/server/sharing/capabilities/service.ts +68 -22
  175. package/src/server/sharing/document-routes.ts +30 -22
  176. package/src/server/sharing/grants/service.ts +11 -18
  177. package/src/server/sharing/http.ts +4 -0
  178. package/src/server/sharing/invitations/service.ts +21 -11
  179. package/src/server/sharing/policy.ts +9 -14
  180. package/src/server/sharing/routes.ts +108 -85
  181. package/src/server/sharing/service.ts +1 -17
  182. package/src/server/validation.ts +59 -2
  183. package/src/server/workspaces/index.ts +1 -0
  184. package/src/server/workspaces/members.ts +89 -38
  185. package/src/server/workspaces/repository.ts +2 -1
  186. package/src/server/workspaces/routes.ts +67 -45
  187. package/src/{ai-relay → services}/app.ts +48 -44
  188. package/src/{ai-relay → services}/config.ts +25 -19
  189. package/src/services/db/index.ts +10 -0
  190. package/src/services/db/migrate.ts +37 -0
  191. package/src/{ai-relay/db/migrations/001_usage.ts → services/db/migrations/001_baseline.ts} +18 -7
  192. package/src/services/db/schema.ts +22 -0
  193. package/src/services/index.ts +18 -0
  194. package/src/{ai-relay → services}/metering.ts +2 -2
  195. package/src/{ai-relay → services}/usage.ts +43 -19
  196. package/dist/bin/ai-relay.mjs.map +0 -1
  197. package/src/ai-relay/db/index.ts +0 -10
  198. package/src/ai-relay/db/migrate.ts +0 -32
  199. package/src/ai-relay/db/schema.ts +0 -20
  200. package/src/ai-relay/index.ts +0 -18
  201. package/src/contract/ai.ts +0 -150
  202. package/src/runtime/cloudflare/relay.ts +0 -136
  203. package/src/server/ai/errors.ts +0 -7
  204. package/src/server/ai/index.ts +0 -4
  205. package/src/server/ai/routes.ts +0 -43
  206. package/src/server/auth/workspace-access.ts +0 -23
  207. package/src/server/db/migrations/001_foundation.ts +0 -164
  208. package/src/server/db/migrations/002_upload_cleanup_claims.ts +0 -34
  209. package/src/server/db/migrations/003_document_cleanup_claims.ts +0 -23
  210. package/src/server/db/migrations/004_document_sharing.ts +0 -82
  211. package/src/server/db/migrations/005_document_collaboration_epoch.ts +0 -13
  212. package/src/server/db/migrations/006_invitation_continuation.ts +0 -27
  213. package/src/server/db/migrations/007_collaboration_state.ts +0 -25
  214. package/src/server/db/migrations/008_storage_reservations.ts +0 -37
  215. package/src/server/db/migrations/009_workspace_entitlements.ts +0 -22
  216. package/src/server/db/migrations/010_upload_finalization.ts +0 -22
  217. package/src/server/db/migrations/011_upload_finalization_lease.ts +0 -12
  218. package/src/server/db/migrations/012_transactional_email.ts +0 -48
  219. package/src/server/db/migrations/013_admin_enrollment.ts +0 -54
  220. package/src/server/db/migrations/014_enrollment_email_kinds.ts +0 -30
  221. package/src/server/db/migrations/015_cloud_rate_limit.ts +0 -22
  222. package/src/server/db/migrations/016_auth_schema.ts +0 -16
  223. package/src/server/db/migrations/017_authentication_email_kinds.ts +0 -32
  224. package/src/server/db/migrations/018_authentication_mfa.ts +0 -78
  225. package/src/server/db/migrations/019_better_auth_account_issuer.ts +0 -42
  226. package/src/server/db/migrations/020_workspace_invitations.ts +0 -57
  227. package/src/server/db/migrations/021_document_versions.ts +0 -66
  228. package/src/server/db/migrations/022_entitlement_external_version.ts +0 -17
  229. package/src/server/db/migrations/023_document_images.ts +0 -61
  230. package/src/server/db/migrations/024_account_activity.ts +0 -66
  231. package/src/server/db/migrations/025_table_names.ts +0 -58
  232. package/src/server/db/migrations/026_user_picture.ts +0 -30
  233. package/src/server/db/migrations/027_document_kind.ts +0 -19
  234. package/src/server/db/migrations/helpers.ts +0 -78
  235. /package/dist/bin/{ai-relay.d.mts → services.d.mts} +0 -0
@@ -1,40 +1,43 @@
1
1
  import { readFile } from 'node:fs/promises'
2
2
 
3
- import { createWorkspaceEntitlementRepository, parseStaticEntitlementsTOML } from '#cloud/server'
3
+ import { createAccountEntitlementRepository, parseStaticEntitlementsTOML } from '#cloud/server'
4
4
  import { parse as parseTOML } from 'smol-toml'
5
5
 
6
6
  import { withNodeCloudDatabase } from '../command'
7
7
  import { entitlementWriteRefusal } from './refusal'
8
8
 
9
9
  const operation = process.argv[2]
10
- const workspaceIndex = process.argv.indexOf('--workspace')
10
+ const accountIndex = process.argv.indexOf('--account')
11
11
  const fileIndex = process.argv.indexOf('--file')
12
12
  const dryRun = process.argv.includes('--dry-run')
13
- const workspaceId = workspaceIndex !== -1 ? process.argv[workspaceIndex + 1] : undefined
14
- if (!workspaceId) throw new Error('Expected --workspace <uuid>')
13
+ const accountArgument = accountIndex !== -1 ? process.argv[accountIndex + 1] : undefined
14
+ if (!accountArgument) throw new Error('Expected --account <user id>')
15
15
 
16
16
  await withNodeCloudDatabase(async (database, config) => {
17
- const repository = createWorkspaceEntitlementRepository(database)
17
+ const repository = createAccountEntitlementRepository(database)
18
18
  const refusal = entitlementWriteRefusal(config, operation, dryRun)
19
19
  if (refusal) throw new Error(refusal)
20
+ // A plan is stored under the account as the user table spells it, which policy reads.
21
+ const userId = await repository.account(accountArgument)
22
+ if (!userId) throw new Error(`No account ${accountArgument}`)
20
23
  if (operation === 'get') {
21
- console.log(JSON.stringify({ entitlement: await repository.get(workspaceId) }))
24
+ console.log(JSON.stringify({ entitlement: await repository.get(userId) }))
22
25
  } else if (operation === 'clear') {
23
26
  if (dryRun) {
24
- console.log(JSON.stringify({ dryRun, cleared: false, workspaceId }))
27
+ console.log(JSON.stringify({ dryRun, cleared: false, userId }))
25
28
  return
26
29
  }
27
- await repository.clear(workspaceId)
28
- console.log(JSON.stringify({ cleared: true, workspaceId }))
30
+ await repository.clear(userId)
31
+ console.log(JSON.stringify({ cleared: true, userId }))
29
32
  } else if (operation === 'set') {
30
33
  const file = fileIndex !== -1 ? process.argv[fileIndex + 1] : undefined
31
34
  if (!file) throw new Error('Expected --file <path>')
32
35
  const values = parseStaticEntitlementsTOML(parseTOML(await readFile(file, 'utf8')))
33
36
  if (dryRun) {
34
- console.log(JSON.stringify({ dryRun, workspaceId, values }))
37
+ console.log(JSON.stringify({ dryRun, userId, values }))
35
38
  return
36
39
  }
37
- console.log(JSON.stringify({ entitlement: await repository.set(workspaceId, values, 'cli') }))
40
+ console.log(JSON.stringify({ entitlement: await repository.set(userId, values, 'cli') }))
38
41
  } else {
39
42
  throw new Error('Expected operation: get, set, or clear')
40
43
  }
@@ -1,4 +1,4 @@
1
- export { startNodeAIRelay, type NodeAIRelayOptions } from './ai-relay/server'
1
+ export { startNodeServices, type NodeServicesOptions } from './services/server'
2
2
  export { createMigratedNodeCloudDatabase } from './bootstrap'
3
3
  export { createNodeCloudApplication, type NodeCloudApplicationOptions } from './application'
4
4
  export { loadNodeCloudServerConfig } from './config'
@@ -4,7 +4,11 @@ import { createBetterAuthAdapter, migrateCloudDatabase } from '#cloud/server'
4
4
  import { loadNodeCloudServerConfig } from './config'
5
5
 
6
6
  const config = await loadNodeCloudServerConfig(process.env)
7
- const database = createPostgresCloudDatabase({ connectionString: config.databaseURL })
7
+ // A migration may rewrite a large table, which takes as long as it takes.
8
+ const database = createPostgresCloudDatabase({
9
+ connectionString: config.databaseURL,
10
+ statementTimeoutMs: null
11
+ })
8
12
  try {
9
13
  const auth = createBetterAuthAdapter(config, database)
10
14
  await migrateCloudDatabase(database, { run: auth.migrate, schemaVersion: auth.schemaVersion })
@@ -1,7 +1,7 @@
1
1
  import type { IncomingMessage, Server } from 'node:http'
2
2
  import type { Duplex } from 'node:stream'
3
3
 
4
- import { RELAY_PATH } from '#cloud/contract'
4
+ import { RELAY_HEARTBEAT, RELAY_PATH } from '#cloud/contract'
5
5
  import type { CollaborationRelay } from '#cloud/server'
6
6
  import { WebSocketServer, type RawData } from 'ws'
7
7
 
@@ -34,7 +34,13 @@ export function attachCollaborationRelay(
34
34
  send: (data) => webSocket.send(data),
35
35
  close: (code, reason) => webSocket.close(code, reason)
36
36
  })
37
- webSocket.on('message', (data) => connection.receive(bytes(data)))
37
+ webSocket.on('message', (data, isBinary) => {
38
+ if (!isBinary && new TextDecoder().decode(bytes(data)) === RELAY_HEARTBEAT.ping) {
39
+ webSocket.send(RELAY_HEARTBEAT.pong)
40
+ return
41
+ }
42
+ connection.receive(bytes(data))
43
+ })
38
44
  webSocket.on('close', () => connection.closed())
39
45
  webSocket.on('error', () => webSocket.terminate())
40
46
  })
@@ -0,0 +1,13 @@
1
+ import type { ForwardedRequestId } from '#cloud/server'
2
+
3
+ /** What a proxy's `X-Request-Id` may hold to be kept. */
4
+ const PROXY_REQUEST_ID = /^[\w-]{1,128}$/
5
+
6
+ /**
7
+ * Names a request by the `X-Request-Id` a proxy in front set, such as nginx's `$request_id`, so
8
+ * the proxy's log and the server's agree.
9
+ */
10
+ export const proxyRequestId: ForwardedRequestId = (headers) => {
11
+ const id = headers.get('x-request-id')
12
+ return id && PROXY_REQUEST_ID.test(id) ? id : undefined
13
+ }
@@ -1,9 +1,12 @@
1
1
  import { defaultRelayURL } from '#cloud/contract'
2
+ import { createPostgresCloudDatabase } from '#cloud/runtime/postgres/database'
3
+ import { createCloudErrorReporter } from '#cloud/runtime/posthog/errors'
2
4
  import { createS3ObjectStore } from '#cloud/runtime/s3/objects'
3
5
  import { createCloudServer, type SiteFiles } from '#cloud/runtime/site/server'
4
6
  import {
5
7
  createCloudApp,
6
- loadAISigningKey,
8
+ createLoggingErrorReporter,
9
+ loadServicesSigningKey,
7
10
  createCloudAuthenticationRuntime,
8
11
  createCollaborationRelay,
9
12
  createCollaborationStateStore,
@@ -11,14 +14,20 @@ import {
11
14
  createCleanupServices,
12
15
  createConfiguredCloudPolicy,
13
16
  relayDocumentLookups,
17
+ reportCloudError,
14
18
  startCleanupWorker,
15
- startTransactionalEmailWorker
19
+ startTransactionalEmailWorker,
20
+ writeJSONLine,
21
+ type CloudErrorReporter,
22
+ type CloudServerConfig
16
23
  } from '#cloud/server'
17
24
 
18
- import { createMigratedNodeCloudDatabase } from './bootstrap'
25
+ import { migratedNodeCloudDatabase } from './bootstrap'
26
+ import { loadNodeCloudServerConfig } from './config'
19
27
  import { createNodeTransactionalEmailRuntime } from './email/runtime'
20
28
  import { listen } from './listen'
21
29
  import { attachCollaborationRelay } from './relay'
30
+ import { proxyRequestId } from './request-id'
22
31
  import { developmentSiteFiles, nodeSiteFiles } from './site'
23
32
 
24
33
  export type NodeCloudServerOptions = {
@@ -40,11 +49,35 @@ function siteFiles(
40
49
  return undefined
41
50
  }
42
51
 
52
+ /**
53
+ * Starts the server. Once the config parses, a startup that fails, such as a migration, a signing
54
+ * key, or a port that cannot be bound, is reported before it is thrown; a config that does not
55
+ * parse has no error tracking to report to.
56
+ */
43
57
  export async function startNodeCloudServer(options: NodeCloudServerOptions = {}) {
44
58
  const environment = options.environment ?? process.env
45
- const { config, database } = await createMigratedNodeCloudDatabase(environment)
59
+ const config = await loadNodeCloudServerConfig(environment)
60
+ const reportError = createLoggingErrorReporter(
61
+ writeJSONLine,
62
+ createCloudErrorReporter(config, 'node')
63
+ )
64
+ try {
65
+ return await startConfiguredServer(config, environment, options, reportError)
66
+ } catch (error) {
67
+ await reportCloudError(reportError, 'Starting the server failed', error, { source: 'setup' })
68
+ throw error
69
+ }
70
+ }
71
+
72
+ async function startConfiguredServer(
73
+ config: CloudServerConfig,
74
+ environment: Readonly<Record<string, string | undefined>>,
75
+ options: NodeCloudServerOptions,
76
+ reportError: CloudErrorReporter | undefined
77
+ ) {
78
+ const { database } = await migratedNodeCloudDatabase(config)
46
79
  // Before any worker starts, so a key that cannot sign leaves nothing running.
47
- const aiSigningKey = await loadAISigningKey(config).catch(async (error: unknown) => {
80
+ const servicesSigningKey = await loadServicesSigningKey(config).catch(async (error: unknown) => {
48
81
  await database.destroy()
49
82
  throw error
50
83
  })
@@ -56,18 +89,34 @@ export async function startNodeCloudServer(options: NodeCloudServerOptions = {})
56
89
  } = createNodeTransactionalEmailRuntime(config, database)
57
90
  const { auth, enrollment } = createCloudAuthenticationRuntime(config, database, email)
58
91
  const policy = createConfiguredCloudPolicy(config, database)
59
- const cleanup = config.cleanupEnabled
60
- ? startCleanupWorker(createCleanupServices(config, database, objects, policy), {
61
- ...cleanupPassOptions(config),
62
- intervalMs: config.cleanupIntervalMs,
63
- onError: (error) => console.error('[Cloud] Cleanup worker failed:', error)
92
+ // Cleanup sweeps may run longer than a request should, so they have a connection of their own
93
+ // without the statement timeout.
94
+ const cleanupDatabase = config.cleanupEnabled
95
+ ? createPostgresCloudDatabase({
96
+ connectionString: config.databaseURL,
97
+ statementTimeoutMs: null
64
98
  })
65
99
  : undefined
100
+ const cleanup = cleanupDatabase
101
+ ? startCleanupWorker(
102
+ createCleanupServices(config, cleanupDatabase, objects, policy, reportError),
103
+ {
104
+ ...cleanupPassOptions(config),
105
+ intervalMs: config.cleanupIntervalMs,
106
+ onError: (error) =>
107
+ void reportCloudError(reportError, 'Cleanup worker failed', error, {
108
+ source: 'cleanup'
109
+ })
110
+ }
111
+ )
112
+ : undefined
66
113
  const relay = createCollaborationRelay({
67
114
  authSecret: config.authSecret,
68
- store: createCollaborationStateStore(database),
115
+ store: createCollaborationStateStore(database, objects),
69
116
  maximumMessageBytes: config.technicalLimits.maximumCollaborationMessageBytes,
70
117
  maximumConnectionsPerRoom: config.technicalLimits.maximumConnectionsPerRoom,
118
+ onError: (message, error) =>
119
+ void reportCloudError(reportError, message, error, { source: 'relay' }),
71
120
  ...relayDocumentLookups(database, policy, config.deployment)
72
121
  })
73
122
  const app = createCloudApp({
@@ -78,10 +127,13 @@ export async function startNodeCloudServer(options: NodeCloudServerOptions = {})
78
127
  invitationOutbox,
79
128
  transactionalEmail: email,
80
129
  enrollment,
81
- aiSigningKey,
130
+ servicesSigningKey,
82
131
  relayURL: config.relayURL ?? defaultRelayURL(config.publicURL),
83
132
  onDocumentRestored: (documentId) => relay.startOver(documentId),
84
- onDocumentVersionsChanged: (documentId) => relay.announceVersions(documentId)
133
+ onDocumentTakenDown: (documentId) => relay.closeDocument(documentId),
134
+ onDocumentVersionsChanged: (documentId) => relay.announceVersions(documentId),
135
+ reportError,
136
+ forwardedRequestId: proxyRequestId
85
137
  })
86
138
  const emailWorker = emailTransport
87
139
  ? startTransactionalEmailWorker(email, {
@@ -89,12 +141,15 @@ export async function startNodeCloudServer(options: NodeCloudServerOptions = {})
89
141
  intervalMs: config.emailIntervalMs,
90
142
  leaseDurationMs: config.emailLeaseDurationMs,
91
143
  maximumAttempts: config.emailMaximumAttempts,
92
- onError: (error) => console.error('[Cloud] Email worker failed:', error)
144
+ onError: (error) =>
145
+ void reportCloudError(reportError, 'Email worker failed', error, { source: 'email' })
93
146
  })
94
147
  : undefined
95
148
  const cloudServer = createCloudServer(app, {
96
149
  site: siteFiles(environment),
97
- indexingPolicy: config.indexingPolicy
150
+ indexingPolicy: config.indexingPolicy,
151
+ reportError,
152
+ forwardedRequestId: proxyRequestId
98
153
  })
99
154
  const server = await listen(
100
155
  async (request) => cloudServer.fetch(request),
@@ -114,6 +169,7 @@ export async function startNodeCloudServer(options: NodeCloudServerOptions = {})
114
169
  await relayListener.close()
115
170
  await server.close()
116
171
  await cleanup?.stop()
172
+ await cleanupDatabase?.destroy()
117
173
  await emailWorker?.stop()
118
174
  await database.destroy()
119
175
  }
@@ -1,23 +1,23 @@
1
1
  #!/usr/bin/env node
2
2
  import dedent from 'dedent'
3
3
 
4
- import { startNodeAIRelay } from './server'
4
+ import { startNodeServices } from './server'
5
5
 
6
6
  if (process.argv.includes('--help') || process.argv.includes('-h')) {
7
7
  console.log(dedent`
8
- Usage: openpencil-ai-relay
8
+ Usage: openpencil-services
9
9
 
10
10
  Runs the reference AI service for a self-hosted OpenPencil Cloud. Settings come from the
11
11
  environment:
12
- OPENPENCIL_AI_RELAY_CONFIG the relay TOML file (required)
12
+ OPENPENCIL_SERVICES_CONFIG the services TOML file (required)
13
13
  PORT, HOST where to listen (default 8788 on 0.0.0.0)
14
14
  plus the secrets the TOML names. See https://openpencil.dev/reference/self-hosting-cloud
15
15
  `)
16
16
  process.exit(0)
17
17
  }
18
18
 
19
- const relay = await startNodeAIRelay()
20
- console.log(`OpenPencil AI relay listening on ${relay.url}`)
19
+ const relay = await startNodeServices()
20
+ console.log(`OpenPencil services listening on ${relay.url}`)
21
21
 
22
22
  let stopping: Promise<void> | undefined
23
23
 
@@ -25,7 +25,7 @@ function shutdown() {
25
25
  stopping ??= relay.stop().then(
26
26
  () => process.exit(0),
27
27
  (error: unknown) => {
28
- console.error('[AI relay] Shutdown failed:', error)
28
+ console.error('[Services] Shutdown failed:', error)
29
29
  process.exit(1)
30
30
  }
31
31
  )
@@ -1,47 +1,47 @@
1
1
  import { readFile } from 'node:fs/promises'
2
2
 
3
+ import { CLOUD_SERVICES_JWKS_PATH } from '#cloud/contract'
3
4
  import {
4
- AIRelayConfigError,
5
- createAIRelayApp,
6
- createAIRelayDatabase,
7
- migrateAIRelayDatabase,
8
- parseAIRelayTOML,
9
- type AIRelayEnvironment
10
- } from '#cloud/ai-relay'
11
- import { CLOUD_AI_JWKS_PATH } from '#cloud/contract'
5
+ ServicesConfigError,
6
+ createServicesApp,
7
+ createServicesDatabase,
8
+ migrateServicesDatabase,
9
+ parseServicesTOML,
10
+ type ServicesEnvironment
11
+ } from '#cloud/services'
12
12
  import { createRemoteJWKSet } from 'jose'
13
13
  import { PostgresDialect } from 'kysely'
14
14
  import { Pool } from 'pg'
15
15
 
16
16
  import { listen } from '../listen'
17
17
 
18
- export type NodeAIRelayOptions = {
19
- environment?: AIRelayEnvironment
18
+ export type NodeServicesOptions = {
19
+ environment?: ServicesEnvironment
20
20
  port?: number
21
21
  }
22
22
 
23
- export async function startNodeAIRelay(options: NodeAIRelayOptions = {}) {
23
+ export async function startNodeServices(options: NodeServicesOptions = {}) {
24
24
  const environment = options.environment ?? process.env
25
- const path = environment.OPENPENCIL_AI_RELAY_CONFIG
25
+ const path = environment.OPENPENCIL_SERVICES_CONFIG
26
26
  if (!path) {
27
- throw new AIRelayConfigError('OPENPENCIL_AI_RELAY_CONFIG must name a relay TOML file')
27
+ throw new ServicesConfigError('OPENPENCIL_SERVICES_CONFIG must name a services TOML file')
28
28
  }
29
- const config = parseAIRelayTOML(await readFile(path, 'utf8'), environment)
30
- const database = createAIRelayDatabase(
29
+ const config = parseServicesTOML(await readFile(path, 'utf8'), environment)
30
+ const database = createServicesDatabase(
31
31
  new PostgresDialect({ pool: new Pool({ connectionString: config.databaseURL }) })
32
32
  )
33
33
  try {
34
- await migrateAIRelayDatabase(database)
34
+ await migrateServicesDatabase(database)
35
35
  } catch (error) {
36
36
  await database.destroy()
37
37
  throw error
38
38
  }
39
39
  // Metering outlives its answer; shutdown waits for it so no request goes uncharged.
40
40
  const metering = new Set<Promise<unknown>>()
41
- const app = createAIRelayApp({
41
+ const app = createServicesApp({
42
42
  config,
43
43
  database,
44
- keys: createRemoteJWKSet(new URL(CLOUD_AI_JWKS_PATH, config.cloudURL)),
44
+ keys: createRemoteJWKSet(new URL(CLOUD_SERVICES_JWKS_PATH, config.cloudURL)),
45
45
  waitUntil: (promise) => {
46
46
  metering.add(promise)
47
47
  void promise.finally(() => metering.delete(promise))
@@ -2,16 +2,35 @@ import { createCloudDatabase } from '#cloud/server'
2
2
  import { PostgresDialect } from 'kysely'
3
3
  import { Pool, type PoolConfig } from 'pg'
4
4
 
5
+ /**
6
+ * How long a query serving a request may run before PostgreSQL cancels it. Rate limits and
7
+ * sessions live in PostgreSQL, so one stuck query would otherwise hold its connection, and every
8
+ * request waiting for one, indefinitely.
9
+ */
10
+ export const POSTGRES_STATEMENT_TIMEOUT_MS = 30_000
11
+
5
12
  export type PostgresCloudDatabaseOptions = {
6
13
  connectionString: string
7
- pool?: Omit<PoolConfig, 'connectionString'>
14
+ pool?: Omit<PoolConfig, 'connectionString' | 'statement_timeout' | 'query_timeout'>
15
+ /**
16
+ * How long one statement may run, `POSTGRES_STATEMENT_TIMEOUT_MS` unless set; null for no limit
17
+ * of this server's own, as migrations need and as Hyperdrive takes, which keeps no session
18
+ * settings and ends queries past its own limit. PostgreSQL cancels a statement past it, so its
19
+ * connection is free again for the next query.
20
+ */
21
+ statementTimeoutMs?: number | null
8
22
  }
9
23
 
10
24
  export function createPostgresCloudDatabase(options: PostgresCloudDatabaseOptions) {
25
+ const statementTimeoutMs =
26
+ options.statementTimeoutMs === undefined
27
+ ? POSTGRES_STATEMENT_TIMEOUT_MS
28
+ : options.statementTimeoutMs
11
29
  return createCloudDatabase({
12
30
  dialect: new PostgresDialect({
13
31
  pool: new Pool({
14
32
  ...options.pool,
33
+ ...(statementTimeoutMs === null ? {} : { statement_timeout: statementTimeoutMs }),
15
34
  connectionString: options.connectionString
16
35
  })
17
36
  })
@@ -0,0 +1,32 @@
1
+ import type { CloudServerConfig } from '#cloud/server'
2
+ import { createPostHogErrorReporter, type CloudErrorReporter } from '#cloud/server/errors'
3
+ import type { PostHogOptions } from 'posthog-node'
4
+
5
+ /** Which runtime serves the deployment, as its error reports name it. */
6
+ export type CloudRuntimeName = 'node' | 'cloudflare'
7
+
8
+ /**
9
+ * Reports server errors to the PostHog project the deployment's `[error_tracking]` names, or
10
+ * returns nothing when it names none. Reports carry the runtime, the deployment, and where the
11
+ * error was caught, never who made the request or what it carried.
12
+ */
13
+ export function createCloudErrorReporter(
14
+ config: CloudServerConfig,
15
+ runtime: CloudRuntimeName,
16
+ options: Pick<PostHogOptions, 'fetch'> = {}
17
+ ): CloudErrorReporter | undefined {
18
+ const tracking = config.errorTracking
19
+ if (!tracking) return undefined
20
+ return createPostHogErrorReporter({
21
+ key: tracking.projectAPIKey,
22
+ host: tracking.host,
23
+ properties: {
24
+ cloud_runtime: runtime,
25
+ cloud_deployment: config.deployment,
26
+ cloud_host: new URL(config.publicURL).host
27
+ },
28
+ sourceProperty: 'cloud_source',
29
+ logName: 'Cloud',
30
+ fetch: options.fetch
31
+ })
32
+ }
@@ -3,6 +3,7 @@ import type { ObjectStore } from '#cloud/server/objects'
3
3
  import {
4
4
  AbortMultipartUploadCommand,
5
5
  CompleteMultipartUploadCommand,
6
+ CopyObjectCommand,
6
7
  CreateMultipartUploadCommand,
7
8
  DeleteObjectCommand,
8
9
  GetObjectCommand,
@@ -11,7 +12,7 @@ import {
11
12
  S3Client,
12
13
  UploadPartCommand
13
14
  } from '@aws-sdk/client-s3'
14
- // eslint-disable-next-line open-pencil/no-mixed-case-acronym-identifiers -- Upstream export spelling.
15
+ // eslint-disable-next-line @open-pencil/structure/no-mixed-case-acronym-identifiers -- Upstream export spelling.
15
16
  import { getSignedUrl as getSignedURL } from '@aws-sdk/s3-request-presigner'
16
17
 
17
18
  const CHECKSUM_METADATA_KEY = 'openpencil-sha256'
@@ -70,6 +71,27 @@ export function createS3ObjectStore(config: CloudServerConfig): ObjectStore {
70
71
  )
71
72
  }
72
73
 
74
+ /**
75
+ * Copies a file under the deleted prefix before it goes, so it outlives the delete until the
76
+ * bucket's lifecycle rule expires it. A file already gone, as when a cleanup pass retries, has
77
+ * nothing to keep.
78
+ */
79
+ async function keepDeleted(key: string, prefix: string): Promise<void> {
80
+ try {
81
+ await client.send(
82
+ new CopyObjectCommand({
83
+ Bucket: config.s3Bucket,
84
+ Key: `${prefix}${key}`,
85
+ CopySource: `${config.s3Bucket}/${key.split('/').map(encodeURIComponent).join('/')}`,
86
+ MetadataDirective: 'COPY',
87
+ ...encryption(config)
88
+ })
89
+ )
90
+ } catch (error) {
91
+ if (!isNotFoundError(error)) throw error
92
+ }
93
+ }
94
+
73
95
  return {
74
96
  capabilities: {
75
97
  nativeSHA256,
@@ -238,7 +260,34 @@ export function createS3ObjectStore(config: CloudServerConfig): ObjectStore {
238
260
  },
239
261
 
240
262
  async delete(key) {
263
+ if (config.s3DeletedPrefix) await keepDeleted(key, config.s3DeletedPrefix)
241
264
  await client.send(new DeleteObjectCommand({ Bucket: config.s3Bucket, Key: key }))
265
+ },
266
+
267
+ async write(input) {
268
+ await client.send(
269
+ new PutObjectCommand({
270
+ Bucket: config.s3Bucket,
271
+ Key: input.key,
272
+ Body: input.bytes,
273
+ ContentType: input.contentType,
274
+ ChecksumSHA256: nativeSHA256 ? input.checksum : undefined,
275
+ Metadata: { [CHECKSUM_METADATA_KEY]: input.checksum },
276
+ ...encryption(config)
277
+ })
278
+ )
279
+ },
280
+
281
+ async read(key) {
282
+ try {
283
+ const object = await client.send(
284
+ new GetObjectCommand({ Bucket: config.s3Bucket, Key: key })
285
+ )
286
+ return object.Body ? await object.Body.transformToByteArray() : null
287
+ } catch (error) {
288
+ if (isNotFoundError(error)) return null
289
+ throw error
290
+ }
242
291
  }
243
292
  }
244
293
  }
@@ -1,5 +1,11 @@
1
1
  import { CLOUD_PORTAL_PAGE_ROOTS } from '#cloud/contract'
2
- import { withIndexingPolicy } from '#cloud/server'
2
+ import {
3
+ cloudErrorHandler,
4
+ cloudRequestId,
5
+ withIndexingPolicy,
6
+ type CloudErrorReporter,
7
+ type ForwardedRequestId
8
+ } from '#cloud/server'
3
9
  import { Hono } from 'hono'
4
10
 
5
11
  /** A file of the Cloud site (`bun run build:cloud`): the editor and the portal. */
@@ -16,11 +22,28 @@ export type SiteFiles = (path: string, request: Request) => Promise<SiteFile | n
16
22
  const PORTAL_PAGE = '/cloud.html'
17
23
  const EDITOR_PAGE = '/index.html'
18
24
 
25
+ /**
26
+ * What the site's pages may run: only the site's own scripts and Turnstile's on the portal, so an
27
+ * injected script tag or one from another host never runs. The pages carry no inline script (the
28
+ * boot splash loads its own file). The editor compiles design JSX and code-tool snippets with
29
+ * `Function` and runs WebAssembly and workers from blobs, which the policy keeps; plugins and AI
30
+ * providers reach other hosts through `fetch`, which it leaves be. Desktop sign-in approval and
31
+ * the editor must never be framed by another site.
32
+ */
33
+ export const SITE_CONTENT_SECURITY_POLICY = [
34
+ "script-src 'self' 'unsafe-eval' 'wasm-unsafe-eval' blob: https://challenges.cloudflare.com",
35
+ "worker-src 'self' blob:",
36
+ "object-src 'none'",
37
+ "base-uri 'self'",
38
+ "frame-ancestors 'none'"
39
+ ].join('; ')
40
+
19
41
  const PAGE_HEADERS = {
20
42
  'Content-Type': 'text/html; charset=utf-8',
21
43
  'Cache-Control': 'no-cache',
22
- // Desktop sign-in approval and the editor must never be framed by another site.
23
- 'Content-Security-Policy': "frame-ancestors 'none'",
44
+ 'Content-Security-Policy': SITE_CONTENT_SECURITY_POLICY,
45
+ // Browsers take this only over HTTPS; a deployment on plain HTTP is unaffected.
46
+ 'Strict-Transport-Security': 'max-age=31536000',
24
47
  'Referrer-Policy': 'no-referrer',
25
48
  'X-Content-Type-Options': 'nosniff'
26
49
  }
@@ -29,6 +52,10 @@ export type CloudServerOptions = {
29
52
  /** The built site; without it, the server answers only its API. */
30
53
  site?: SiteFiles
31
54
  indexingPolicy: 'allow' | 'deny'
55
+ /** Reports errors reading the site's files, when the deployment configures error tracking. */
56
+ reportError?: CloudErrorReporter
57
+ /** The request ID the runtime trusts from what stands in front of it; see `cloudRequestId`. */
58
+ forwardedRequestId?: ForwardedRequestId
32
59
  }
33
60
 
34
61
  function cacheControl(path: string): string {
@@ -43,9 +70,11 @@ function cacheControl(path: string): string {
43
70
  */
44
71
  export function createCloudServer(
45
72
  api: { fetch(request: Request): Response | Promise<Response> },
46
- { site, indexingPolicy }: CloudServerOptions
73
+ { site, indexingPolicy, reportError, forwardedRequestId }: CloudServerOptions
47
74
  ) {
48
75
  const server = new Hono()
76
+ .onError(cloudErrorHandler(reportError))
77
+ .use(cloudRequestId(forwardedRequestId))
49
78
  server.use(async (c, next) => {
50
79
  await next()
51
80
  c.res = withIndexingPolicy(c.res, c.req.path, indexingPolicy)
@@ -9,6 +9,11 @@ const ACCOUNT_SECURITY_ERROR_CODES: Record<string, AccountSecurityErrorCode> = {
9
9
  PASSWORD_TOO_LONG: 'password_too_long',
10
10
  FAILED_TO_UNLINK_LAST_ACCOUNT: 'last_authentication_method',
11
11
  SESSION_NOT_FRESH: 'session_not_fresh',
12
+ // Better Auth's own word for a session too old to delete the account without its password.
13
+ SESSION_EXPIRED: 'session_not_fresh',
14
+ CREDENTIAL_ACCOUNT_NOT_FOUND: 'session_not_fresh',
15
+ ACCOUNT_SOLE_ADMIN: 'account_sole_admin',
16
+ LAST_ADMIN_REQUIRED: 'last_admin_required',
12
17
  CHANGE_EMAIL_DISABLED: 'email_change_unavailable',
13
18
  CURRENT_SESSION: 'current_session',
14
19
  EMAIL_UNCHANGED: 'email_unchanged'