@open-pencil/cloud 0.17.0-beta.2 → 0.17.0-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (339) hide show
  1. package/README.md +3 -2
  2. package/dist/bin/server.mjs +1 -1
  3. package/dist/bin/server.mjs.map +1 -1
  4. package/dist/bin/{ai-relay.mjs → services.mjs} +9 -9
  5. package/dist/bin/services.mjs.map +1 -0
  6. package/dist/client.d.ts +525 -66
  7. package/dist/client.d.ts.map +1 -1
  8. package/dist/client.js +1257 -654
  9. package/dist/client.js.map +1 -1
  10. package/dist/contract.d.ts +2 -2
  11. package/dist/contract.js +2 -2
  12. package/dist/contract2.js +1028 -481
  13. package/dist/contract2.js.map +1 -1
  14. package/dist/email.d.ts +1780 -31
  15. package/dist/email.d.ts.map +1 -1
  16. package/dist/email.js +3556 -325
  17. package/dist/email.js.map +1 -1
  18. package/dist/errors.d.ts +2 -0
  19. package/dist/errors.js +48 -0
  20. package/dist/errors.js.map +1 -0
  21. package/dist/index.d.ts +929 -72
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index2.d.ts +35 -0
  24. package/dist/index2.d.ts.map +1 -0
  25. package/dist/listen.mjs +802 -927
  26. package/dist/listen.mjs.map +1 -1
  27. package/dist/runtime-cloudflare.d.ts +89 -0
  28. package/dist/runtime-cloudflare.d.ts.map +1 -0
  29. package/dist/runtime-cloudflare.js +15667 -0
  30. package/dist/runtime-cloudflare.js.map +1 -0
  31. package/dist/runtime-node.d.mts +7166 -3101
  32. package/dist/runtime-node.d.mts.map +1 -1
  33. package/dist/runtime-node.mjs +6 -5
  34. package/dist/runtime-node.mjs.map +1 -1
  35. package/dist/server.d.ts +6525 -2205
  36. package/dist/server.d.ts.map +1 -1
  37. package/dist/server.js +12500 -7613
  38. package/dist/server.js.map +1 -1
  39. package/dist/server.mjs +8625 -2878
  40. package/dist/server.mjs.map +1 -1
  41. package/dist/server2.mjs +101 -79
  42. package/dist/server2.mjs.map +1 -1
  43. package/package.json +27 -9
  44. package/src/admin/api/routes.ts +59 -0
  45. package/src/admin/audit/service.ts +27 -8
  46. package/src/admin/documents/service.ts +431 -0
  47. package/src/admin/errors.ts +9 -1
  48. package/src/admin/index.ts +6 -0
  49. package/src/admin/operations/service.ts +7 -0
  50. package/src/admin/reports/service.ts +153 -0
  51. package/src/admin/schema.ts +6 -0
  52. package/src/client/api.ts +28 -461
  53. package/src/client/auth/client.ts +22 -3
  54. package/src/client/auth/navigation.ts +6 -5
  55. package/src/client/billing.ts +80 -0
  56. package/src/client/community.ts +40 -0
  57. package/src/client/discovery.ts +35 -9
  58. package/src/client/documents.ts +160 -0
  59. package/src/client/index.ts +14 -7
  60. package/src/client/portal/account.ts +79 -2
  61. package/src/client/portal/administration.ts +56 -0
  62. package/src/client/portal/request.ts +2 -0
  63. package/src/client/response.ts +21 -0
  64. package/src/client/schema.ts +8 -0
  65. package/src/client/services.ts +118 -0
  66. package/src/client/sharing.ts +262 -0
  67. package/src/client/versions.ts +104 -0
  68. package/src/contract/account.ts +108 -2
  69. package/src/contract/admin/index.ts +67 -3
  70. package/src/contract/billing.ts +166 -0
  71. package/src/contract/collaboration.ts +11 -1
  72. package/src/contract/community.ts +60 -0
  73. package/src/contract/discovery.ts +83 -4
  74. package/src/contract/documents.ts +27 -0
  75. package/src/contract/entitlements.ts +12 -2
  76. package/src/contract/index.ts +114 -39
  77. package/src/contract/links.ts +57 -4
  78. package/src/contract/locale.ts +27 -0
  79. package/src/contract/portal.ts +12 -1
  80. package/src/contract/relay.ts +6 -0
  81. package/src/contract/reports.ts +42 -0
  82. package/src/contract/routes.ts +10 -0
  83. package/src/contract/services.ts +235 -0
  84. package/src/contract/sharing.ts +3 -9
  85. package/src/email/i18n.ts +174 -0
  86. package/src/email/index.ts +13 -2
  87. package/src/email/locales/ar.json +141 -0
  88. package/src/email/locales/de.json +141 -0
  89. package/src/email/locales/es.json +141 -0
  90. package/src/email/locales/fr.json +141 -0
  91. package/src/email/locales/it.json +141 -0
  92. package/src/email/locales/ja.json +141 -0
  93. package/src/email/locales/pl.json +141 -0
  94. package/src/email/locales/pt-br.json +141 -0
  95. package/src/email/locales/ru.json +141 -0
  96. package/src/email/locales/zh-cn.json +141 -0
  97. package/src/email/messages.ts +200 -0
  98. package/src/email/render.ts +138 -246
  99. package/src/email/samples.ts +115 -0
  100. package/src/email/schema.ts +79 -5
  101. package/src/email/templates/BrandEmail.vue +54 -0
  102. package/src/email/templates/DocumentBlocked.vue +45 -0
  103. package/src/email/templates/DocumentInvitation.vue +24 -0
  104. package/src/email/templates/DocumentReport.vue +32 -0
  105. package/src/email/templates/DocumentReportDecided.vue +36 -0
  106. package/src/email/templates/DocumentReportReceived.vue +18 -0
  107. package/src/email/templates/DocumentRestored.vue +24 -0
  108. package/src/email/templates/EmailChangeRequested.vue +24 -0
  109. package/src/email/templates/EmailRoot.vue +19 -0
  110. package/src/email/templates/EmailText.vue +9 -0
  111. package/src/email/templates/EmailVerification.vue +38 -0
  112. package/src/email/templates/EnrollmentNotification.vue +24 -0
  113. package/src/email/templates/EnrollmentUpdate.vue +52 -0
  114. package/src/email/templates/NewSignIn.vue +37 -0
  115. package/src/email/templates/PasswordChanged.vue +22 -0
  116. package/src/email/templates/PasswordReset.vue +23 -0
  117. package/src/email/templates/StorageLimitReached.vue +28 -0
  118. package/src/email/templates/TwoStepChanged.vue +32 -0
  119. package/src/email/templates/WaitlistConfirmation.vue +29 -0
  120. package/src/email/templates/WorkspaceInvitation.vue +28 -0
  121. package/src/email/templates/brand.ts +9 -61
  122. package/src/email/templates/localization.ts +17 -0
  123. package/src/errors/index.ts +6 -0
  124. package/src/errors/posthog.ts +80 -0
  125. package/src/global.d.ts +7 -0
  126. package/src/runtime/cloudflare/environment.ts +22 -11
  127. package/src/runtime/cloudflare/index.ts +1 -1
  128. package/src/runtime/cloudflare/logging.ts +15 -0
  129. package/src/runtime/cloudflare/maintenance.ts +31 -0
  130. package/src/runtime/cloudflare/relay/index.ts +69 -15
  131. package/src/runtime/cloudflare/tsconfig.json +2 -1
  132. package/src/runtime/cloudflare/worker.ts +46 -13
  133. package/src/runtime/node/application.ts +5 -3
  134. package/src/runtime/node/bootstrap.ts +15 -4
  135. package/src/runtime/node/cleanup.ts +20 -3
  136. package/src/runtime/node/email/transport.ts +13 -10
  137. package/src/runtime/node/entitlements/main.ts +14 -11
  138. package/src/runtime/node/index.ts +1 -1
  139. package/src/runtime/node/main.ts +1 -1
  140. package/src/runtime/node/migrate.ts +5 -1
  141. package/src/runtime/node/relay.ts +8 -2
  142. package/src/runtime/node/request-id.ts +13 -0
  143. package/src/runtime/node/server.ts +50 -16
  144. package/src/runtime/node/{ai-relay → services}/main.ts +7 -7
  145. package/src/runtime/node/{ai-relay → services}/server.ts +18 -18
  146. package/src/runtime/postgres/database.ts +20 -1
  147. package/src/runtime/posthog/errors.ts +15 -30
  148. package/src/runtime/s3/objects.ts +41 -1
  149. package/src/runtime/site/community.ts +101 -0
  150. package/src/runtime/site/server.ts +49 -6
  151. package/src/server/account/actions.ts +5 -0
  152. package/src/server/account/deletion.ts +304 -0
  153. package/src/server/account/routes.ts +281 -206
  154. package/src/server/account/terms.ts +42 -0
  155. package/src/server/account/tokens.ts +57 -0
  156. package/src/server/api/router.ts +32 -5
  157. package/src/server/api/types.ts +2 -0
  158. package/src/server/app/checks.ts +84 -0
  159. package/src/server/app/editors.ts +20 -0
  160. package/src/server/app/index.ts +128 -0
  161. package/src/server/app/indexing.ts +40 -0
  162. package/src/server/app/limiters.ts +53 -0
  163. package/src/server/app/routes.ts +114 -0
  164. package/src/server/app/services.ts +185 -0
  165. package/src/server/auth/access-control.ts +63 -0
  166. package/src/server/auth/access-tokens.ts +62 -0
  167. package/src/server/auth/adapter.ts +32 -1
  168. package/src/server/auth/email.ts +4 -1
  169. package/src/server/auth/endpoints.ts +5 -1
  170. package/src/server/auth/enrollment.ts +116 -0
  171. package/src/server/auth/factory.ts +116 -132
  172. package/src/server/auth/index.ts +10 -1
  173. package/src/server/auth/logger.ts +23 -0
  174. package/src/server/auth/mfa-assurance.ts +51 -0
  175. package/src/server/auth/passwords.ts +43 -0
  176. package/src/server/auth/session.ts +2 -0
  177. package/src/server/billing/index.ts +6 -0
  178. package/src/server/billing/routes.ts +27 -0
  179. package/src/server/billing/tokens.ts +37 -0
  180. package/src/server/cleanup/configured.ts +27 -7
  181. package/src/server/cleanup/documents.ts +165 -71
  182. package/src/server/cleanup/revisions.ts +16 -5
  183. package/src/server/cleanup/uploads.ts +8 -2
  184. package/src/server/cleanup/worker.ts +15 -3
  185. package/src/server/collaboration/relay/connection.ts +4 -3
  186. package/src/server/collaboration/relay/documents.ts +13 -10
  187. package/src/server/collaboration/relay/hub.ts +77 -122
  188. package/src/server/collaboration/relay/log.ts +5 -2
  189. package/src/server/collaboration/relay/room.ts +154 -0
  190. package/src/server/collaboration/service.ts +48 -33
  191. package/src/server/community/index.ts +2 -0
  192. package/src/server/community/routes.ts +93 -0
  193. package/src/server/community/service.ts +134 -0
  194. package/src/server/config/deployment.ts +41 -9
  195. package/src/server/config/index.ts +1 -1
  196. package/src/server/config/public.ts +11 -4
  197. package/src/server/config/schema.ts +91 -277
  198. package/src/server/config/toml/schema.ts +307 -0
  199. package/src/server/db/expressions.ts +15 -1
  200. package/src/server/db/index.ts +1 -0
  201. package/src/server/db/migrate.ts +46 -116
  202. package/src/server/db/migrations/001_accounts.ts +79 -0
  203. package/src/server/db/migrations/002_workspaces.ts +48 -0
  204. package/src/server/db/migrations/003_storage.ts +44 -0
  205. package/src/server/db/migrations/004_documents.ts +193 -0
  206. package/src/server/db/migrations/005_sharing.ts +95 -0
  207. package/src/server/db/migrations/006_operations.ts +110 -0
  208. package/src/server/db/migrations/007_trash.ts +33 -0
  209. package/src/server/db/migrations/008_terms.ts +18 -0
  210. package/src/server/db/migrations/009_access_tokens.ts +47 -0
  211. package/src/server/db/migrations/010_storage_limit_notices.ts +50 -0
  212. package/src/server/db/migrations/011_account_ids.ts +30 -0
  213. package/src/server/db/migrations/012_email_locales.ts +34 -0
  214. package/src/server/db/migrations/013_document_block_emails.ts +40 -0
  215. package/src/server/db/migrations/014_report_replies.ts +28 -0
  216. package/src/server/db/migrations/schema.ts +96 -0
  217. package/src/server/db/tables.ts +82 -4
  218. package/src/server/db/users.ts +22 -0
  219. package/src/server/documents/access.ts +66 -55
  220. package/src/server/documents/errors.ts +7 -0
  221. package/src/server/documents/images.ts +2 -1
  222. package/src/server/documents/index.ts +1 -1
  223. package/src/server/documents/limits.ts +14 -8
  224. package/src/server/documents/purge.ts +75 -0
  225. package/src/server/documents/repository.ts +2 -0
  226. package/src/server/documents/routes.ts +26 -0
  227. package/src/server/documents/service.ts +30 -406
  228. package/src/server/documents/summary.ts +1 -0
  229. package/src/server/documents/trash.ts +203 -0
  230. package/src/server/documents/uploads.ts +450 -0
  231. package/src/server/documents/versions/checkpoint.ts +2 -1
  232. package/src/server/documents/versions/history.ts +11 -7
  233. package/src/server/documents/versions/service.ts +94 -28
  234. package/src/server/email/invitations.ts +1 -0
  235. package/src/server/email/service.ts +31 -4
  236. package/src/server/email/types.ts +12 -0
  237. package/src/server/errors/index.ts +51 -16
  238. package/src/server/index.ts +52 -16
  239. package/src/server/internal/index.ts +3 -3
  240. package/src/server/internal/routes.ts +7 -10
  241. package/src/server/internal/schema.ts +1 -1
  242. package/src/server/internal/service.ts +17 -23
  243. package/src/server/invitations/budget.ts +66 -0
  244. package/src/server/invitations/index.ts +1 -0
  245. package/src/server/invitations/types.ts +5 -1
  246. package/src/server/logging.ts +76 -0
  247. package/src/server/objects/types.ts +5 -0
  248. package/src/server/pictures/service.ts +2 -1
  249. package/src/server/policy/configured.ts +9 -3
  250. package/src/server/policy/database.ts +52 -27
  251. package/src/server/policy/index.ts +2 -1
  252. package/src/server/policy/keys.ts +5 -2
  253. package/src/server/policy/policy.ts +40 -12
  254. package/src/server/policy/schema.ts +2 -2
  255. package/src/server/policy/service.ts +78 -56
  256. package/src/server/policy/static.ts +5 -5
  257. package/src/server/policy/storage.ts +33 -0
  258. package/src/server/policy/workspace.ts +41 -0
  259. package/src/server/quota/index.ts +6 -0
  260. package/src/server/quota/notices.ts +121 -0
  261. package/src/server/quota/reconcile.ts +2 -0
  262. package/src/server/quota/storage.ts +74 -31
  263. package/src/server/rate-limit/cleanup.ts +17 -5
  264. package/src/server/rate-limit/middleware.ts +6 -2
  265. package/src/server/reports/index.ts +9 -0
  266. package/src/server/reports/replies.ts +55 -0
  267. package/src/server/reports/routes.ts +39 -0
  268. package/src/server/reports/service.ts +145 -0
  269. package/src/server/services/errors.ts +7 -0
  270. package/src/server/services/index.ts +8 -0
  271. package/src/server/{ai → services}/keys.ts +11 -11
  272. package/src/server/services/routes.ts +47 -0
  273. package/src/server/{ai/service.ts → services/tokens.ts} +23 -20
  274. package/src/server/sharing/capabilities/service.ts +68 -23
  275. package/src/server/sharing/grants/service.ts +11 -18
  276. package/src/server/sharing/http.ts +4 -0
  277. package/src/server/sharing/invitations/service.ts +35 -15
  278. package/src/server/sharing/policy.ts +9 -14
  279. package/src/server/sharing/routes.ts +0 -16
  280. package/src/server/sharing/service.ts +1 -17
  281. package/src/server/validation.ts +4 -0
  282. package/src/server/workspaces/index.ts +1 -0
  283. package/src/server/workspaces/members.ts +91 -52
  284. package/src/server/workspaces/repository.ts +2 -1
  285. package/src/server/workspaces/routes.ts +4 -0
  286. package/src/{ai-relay → services}/app.ts +46 -44
  287. package/src/{ai-relay → services}/config.ts +26 -19
  288. package/src/services/db/index.ts +10 -0
  289. package/src/services/db/migrate.ts +37 -0
  290. package/src/{ai-relay/db/migrations/001_usage.ts → services/db/migrations/001_baseline.ts} +18 -7
  291. package/src/services/db/schema.ts +22 -0
  292. package/src/services/index.ts +18 -0
  293. package/src/{ai-relay → services}/metering.ts +2 -2
  294. package/src/{ai-relay → services}/usage.ts +35 -19
  295. package/dist/bin/ai-relay.mjs.map +0 -1
  296. package/src/ai-relay/db/index.ts +0 -10
  297. package/src/ai-relay/db/migrate.ts +0 -32
  298. package/src/ai-relay/db/schema.ts +0 -20
  299. package/src/ai-relay/index.ts +0 -18
  300. package/src/client/ai.ts +0 -96
  301. package/src/contract/ai.ts +0 -150
  302. package/src/email/templates/document-invitation.ts +0 -35
  303. package/src/email/templates/enrollment.ts +0 -27
  304. package/src/email/templates/layout.ts +0 -18
  305. package/src/server/ai/errors.ts +0 -7
  306. package/src/server/ai/index.ts +0 -4
  307. package/src/server/ai/routes.ts +0 -43
  308. package/src/server/app.ts +0 -379
  309. package/src/server/auth/workspace-access.ts +0 -23
  310. package/src/server/db/migrations/001_foundation.ts +0 -164
  311. package/src/server/db/migrations/002_upload_cleanup_claims.ts +0 -34
  312. package/src/server/db/migrations/003_document_cleanup_claims.ts +0 -23
  313. package/src/server/db/migrations/004_document_sharing.ts +0 -82
  314. package/src/server/db/migrations/005_document_collaboration_epoch.ts +0 -13
  315. package/src/server/db/migrations/006_invitation_continuation.ts +0 -27
  316. package/src/server/db/migrations/007_collaboration_state.ts +0 -25
  317. package/src/server/db/migrations/008_storage_reservations.ts +0 -37
  318. package/src/server/db/migrations/009_workspace_entitlements.ts +0 -22
  319. package/src/server/db/migrations/010_upload_finalization.ts +0 -22
  320. package/src/server/db/migrations/011_upload_finalization_lease.ts +0 -12
  321. package/src/server/db/migrations/012_transactional_email.ts +0 -48
  322. package/src/server/db/migrations/013_admin_enrollment.ts +0 -54
  323. package/src/server/db/migrations/014_enrollment_email_kinds.ts +0 -30
  324. package/src/server/db/migrations/015_cloud_rate_limit.ts +0 -22
  325. package/src/server/db/migrations/016_auth_schema.ts +0 -16
  326. package/src/server/db/migrations/017_authentication_email_kinds.ts +0 -32
  327. package/src/server/db/migrations/018_authentication_mfa.ts +0 -78
  328. package/src/server/db/migrations/019_better_auth_account_issuer.ts +0 -42
  329. package/src/server/db/migrations/020_workspace_invitations.ts +0 -57
  330. package/src/server/db/migrations/021_document_versions.ts +0 -66
  331. package/src/server/db/migrations/022_entitlement_external_version.ts +0 -17
  332. package/src/server/db/migrations/023_document_images.ts +0 -61
  333. package/src/server/db/migrations/024_account_activity.ts +0 -66
  334. package/src/server/db/migrations/025_table_names.ts +0 -58
  335. package/src/server/db/migrations/026_user_picture.ts +0 -30
  336. package/src/server/db/migrations/027_document_kind.ts +0 -19
  337. package/src/server/db/migrations/028_collaboration_updates.ts +0 -84
  338. package/src/server/db/migrations/helpers.ts +0 -78
  339. /package/dist/bin/{ai-relay.d.mts → services.d.mts} +0 -0
@@ -3,7 +3,7 @@ import { createS3ObjectStore } from '#cloud/runtime/s3/objects'
3
3
  import {
4
4
  createCloudAuthenticationRuntime,
5
5
  createCloudApp,
6
- loadAISigningKey,
6
+ loadServicesSigningKey,
7
7
  type CloudApp,
8
8
  type CloudEnvironment,
9
9
  type CloudServerConfig,
@@ -12,6 +12,7 @@ import {
12
12
 
13
13
  import { loadNodeCloudServerConfig } from './config'
14
14
  import { createNodeTransactionalEmailRuntime } from './email/runtime'
15
+ import { proxyRequestId } from './request-id'
15
16
 
16
17
  export type NodeCloudApplicationOptions = {
17
18
  environment?: CloudEnvironment
@@ -29,7 +30,7 @@ export async function createNodeCloudApplication(
29
30
  }> {
30
31
  const environment = options.environment ?? process.env
31
32
  const config = await loadNodeCloudServerConfig(environment)
32
- const aiSigningKey = await loadAISigningKey(config)
33
+ const servicesSigningKey = await loadServicesSigningKey(config)
33
34
  const database = createPostgresCloudDatabase({
34
35
  connectionString: options.databaseURL ?? config.databaseURL
35
36
  })
@@ -44,7 +45,8 @@ export async function createNodeCloudApplication(
44
45
  invitationOutbox,
45
46
  transactionalEmail: email,
46
47
  enrollment,
47
- aiSigningKey
48
+ servicesSigningKey,
49
+ forwardedRequestId: proxyRequestId
48
50
  })
49
51
  return { app, config, database, objects, email }
50
52
  }
@@ -7,12 +7,23 @@ import {
7
7
 
8
8
  import { loadNodeCloudServerConfig } from './config'
9
9
 
10
- /** The deployment's database, migrated to this version before anything uses it. */
10
+ /**
11
+ * The deployment's database, migrated to this version before anything uses it. Migrations run
12
+ * on a connection of their own without the statement timeout, since one may rewrite a large table.
13
+ */
11
14
  export async function migratedNodeCloudDatabase(config: CloudServerConfig) {
15
+ const migrating = createPostgresCloudDatabase({
16
+ connectionString: config.databaseURL,
17
+ statementTimeoutMs: null
18
+ })
19
+ try {
20
+ const auth = createBetterAuthAdapter(config, migrating)
21
+ await migrateCloudDatabase(migrating, { run: auth.migrate, schemaVersion: auth.schemaVersion })
22
+ } finally {
23
+ await migrating.destroy()
24
+ }
12
25
  const database = createPostgresCloudDatabase({ connectionString: config.databaseURL })
13
- const auth = createBetterAuthAdapter(config, database)
14
- await migrateCloudDatabase(database, { run: auth.migrate, schemaVersion: auth.schemaVersion })
15
- return { database, auth }
26
+ return { database, auth: createBetterAuthAdapter(config, database) }
16
27
  }
17
28
 
18
29
  export async function createMigratedNodeCloudDatabase(
@@ -1,21 +1,38 @@
1
1
  import { createPostgresCloudDatabase } from '#cloud/runtime/postgres/database'
2
+ import { createCloudErrorReporter } from '#cloud/runtime/posthog/errors'
2
3
  import { createS3ObjectStore } from '#cloud/runtime/s3/objects'
3
4
  import {
4
5
  cleanupPassOptions,
5
6
  createCleanupServices,
6
7
  createConfiguredCloudPolicy,
7
- runCleanup
8
+ createLoggingErrorReporter,
9
+ runCleanup,
10
+ writeJSONLine
8
11
  } from '#cloud/server'
9
12
 
10
13
  import { loadNodeCloudServerConfig } from './config'
14
+ import { createNodeTransactionalEmailRuntime } from './email/runtime'
11
15
 
12
16
  const config = await loadNodeCloudServerConfig(process.env)
13
- const database = createPostgresCloudDatabase({ connectionString: config.databaseURL })
17
+ // A sweep may take longer than a request should.
18
+ const database = createPostgresCloudDatabase({
19
+ connectionString: config.databaseURL,
20
+ statementTimeoutMs: null
21
+ })
14
22
  const objects = createS3ObjectStore(config)
15
23
  try {
16
24
  const policy = createConfiguredCloudPolicy(config, database)
25
+ // Queued here, sent by the server's email worker.
26
+ const { email, transport } = createNodeTransactionalEmailRuntime(config, database)
17
27
  const result = await runCleanup(
18
- createCleanupServices(config, database, objects, policy),
28
+ createCleanupServices(
29
+ config,
30
+ database,
31
+ objects,
32
+ policy,
33
+ createLoggingErrorReporter(writeJSONLine, createCloudErrorReporter(config, 'node')),
34
+ transport ? email : undefined
35
+ ),
19
36
  cleanupPassOptions(config)
20
37
  )
21
38
  console.log(JSON.stringify(result))
@@ -77,16 +77,19 @@ export function createNodemailerInvitationDelivery(
77
77
  const transport = createNodemailerTransactionalEmailTransport(options)
78
78
  return {
79
79
  async sendDocumentInvitation(message: DocumentInvitationMessage): Promise<void> {
80
- const rendered = await renderTransactionalEmail({
81
- kind: 'document-invitation',
82
- payload: {
83
- inviterName: message.inviterName,
84
- documentName: message.documentName,
85
- permission: message.permission,
86
- expiresAt: message.expiresAt,
87
- acceptanceURL: message.acceptanceURL
88
- }
89
- })
80
+ const rendered = await renderTransactionalEmail(
81
+ {
82
+ kind: 'document-invitation',
83
+ payload: {
84
+ inviterName: message.inviterName,
85
+ documentName: message.documentName,
86
+ permission: message.permission,
87
+ expiresAt: message.expiresAt,
88
+ acceptanceURL: message.acceptanceURL
89
+ }
90
+ },
91
+ message.locale
92
+ )
90
93
  await transport.send({
91
94
  deliveryId: message.deliveryId,
92
95
  from: options.from,
@@ -1,40 +1,43 @@
1
1
  import { readFile } from 'node:fs/promises'
2
2
 
3
- import { createWorkspaceEntitlementRepository, parseStaticEntitlementsTOML } from '#cloud/server'
3
+ import { createAccountEntitlementRepository, parseStaticEntitlementsTOML } from '#cloud/server'
4
4
  import { parse as parseTOML } from 'smol-toml'
5
5
 
6
6
  import { withNodeCloudDatabase } from '../command'
7
7
  import { entitlementWriteRefusal } from './refusal'
8
8
 
9
9
  const operation = process.argv[2]
10
- const workspaceIndex = process.argv.indexOf('--workspace')
10
+ const accountIndex = process.argv.indexOf('--account')
11
11
  const fileIndex = process.argv.indexOf('--file')
12
12
  const dryRun = process.argv.includes('--dry-run')
13
- const workspaceId = workspaceIndex !== -1 ? process.argv[workspaceIndex + 1] : undefined
14
- if (!workspaceId) throw new Error('Expected --workspace <uuid>')
13
+ const accountArgument = accountIndex !== -1 ? process.argv[accountIndex + 1] : undefined
14
+ if (!accountArgument) throw new Error('Expected --account <user id>')
15
15
 
16
16
  await withNodeCloudDatabase(async (database, config) => {
17
- const repository = createWorkspaceEntitlementRepository(database)
17
+ const repository = createAccountEntitlementRepository(database)
18
18
  const refusal = entitlementWriteRefusal(config, operation, dryRun)
19
19
  if (refusal) throw new Error(refusal)
20
+ // A plan is stored under the account as the user table spells it, which policy reads.
21
+ const userId = await repository.account(accountArgument)
22
+ if (!userId) throw new Error(`No account ${accountArgument}`)
20
23
  if (operation === 'get') {
21
- console.log(JSON.stringify({ entitlement: await repository.get(workspaceId) }))
24
+ console.log(JSON.stringify({ entitlement: await repository.get(userId) }))
22
25
  } else if (operation === 'clear') {
23
26
  if (dryRun) {
24
- console.log(JSON.stringify({ dryRun, cleared: false, workspaceId }))
27
+ console.log(JSON.stringify({ dryRun, cleared: false, userId }))
25
28
  return
26
29
  }
27
- await repository.clear(workspaceId)
28
- console.log(JSON.stringify({ cleared: true, workspaceId }))
30
+ await repository.clear(userId)
31
+ console.log(JSON.stringify({ cleared: true, userId }))
29
32
  } else if (operation === 'set') {
30
33
  const file = fileIndex !== -1 ? process.argv[fileIndex + 1] : undefined
31
34
  if (!file) throw new Error('Expected --file <path>')
32
35
  const values = parseStaticEntitlementsTOML(parseTOML(await readFile(file, 'utf8')))
33
36
  if (dryRun) {
34
- console.log(JSON.stringify({ dryRun, workspaceId, values }))
37
+ console.log(JSON.stringify({ dryRun, userId, values }))
35
38
  return
36
39
  }
37
- console.log(JSON.stringify({ entitlement: await repository.set(workspaceId, values, 'cli') }))
40
+ console.log(JSON.stringify({ entitlement: await repository.set(userId, values, 'cli') }))
38
41
  } else {
39
42
  throw new Error('Expected operation: get, set, or clear')
40
43
  }
@@ -1,4 +1,4 @@
1
- export { startNodeAIRelay, type NodeAIRelayOptions } from './ai-relay/server'
1
+ export { startNodeServices, type NodeServicesOptions } from './services/server'
2
2
  export { createMigratedNodeCloudDatabase } from './bootstrap'
3
3
  export { createNodeCloudApplication, type NodeCloudApplicationOptions } from './application'
4
4
  export { loadNodeCloudServerConfig } from './config'
@@ -13,7 +13,7 @@ if (process.argv.includes('--help') || process.argv.includes('-h')) {
13
13
  OPENPENCIL_CLOUD_DEV_SITE_URL
14
14
  a Vite dev server to serve them from instead (cloud:dev)
15
15
  PORT, HOST where to listen (default 8787 on 0.0.0.0)
16
- plus the secrets the TOML names. See https://openpencil.dev/reference/self-hosting-cloud
16
+ plus the secrets the TOML names. See https://openpencil.dev/cloud/self-hosting/
17
17
  `)
18
18
  process.exit(0)
19
19
  }
@@ -4,7 +4,11 @@ import { createBetterAuthAdapter, migrateCloudDatabase } from '#cloud/server'
4
4
  import { loadNodeCloudServerConfig } from './config'
5
5
 
6
6
  const config = await loadNodeCloudServerConfig(process.env)
7
- const database = createPostgresCloudDatabase({ connectionString: config.databaseURL })
7
+ // A migration may rewrite a large table, which takes as long as it takes.
8
+ const database = createPostgresCloudDatabase({
9
+ connectionString: config.databaseURL,
10
+ statementTimeoutMs: null
11
+ })
8
12
  try {
9
13
  const auth = createBetterAuthAdapter(config, database)
10
14
  await migrateCloudDatabase(database, { run: auth.migrate, schemaVersion: auth.schemaVersion })
@@ -1,7 +1,7 @@
1
1
  import type { IncomingMessage, Server } from 'node:http'
2
2
  import type { Duplex } from 'node:stream'
3
3
 
4
- import { RELAY_PATH } from '#cloud/contract'
4
+ import { RELAY_HEARTBEAT, RELAY_PATH } from '#cloud/contract'
5
5
  import type { CollaborationRelay } from '#cloud/server'
6
6
  import { WebSocketServer, type RawData } from 'ws'
7
7
 
@@ -34,7 +34,13 @@ export function attachCollaborationRelay(
34
34
  send: (data) => webSocket.send(data),
35
35
  close: (code, reason) => webSocket.close(code, reason)
36
36
  })
37
- webSocket.on('message', (data) => connection.receive(bytes(data)))
37
+ webSocket.on('message', (data, isBinary) => {
38
+ if (!isBinary && new TextDecoder().decode(bytes(data)) === RELAY_HEARTBEAT.ping) {
39
+ webSocket.send(RELAY_HEARTBEAT.pong)
40
+ return
41
+ }
42
+ connection.receive(bytes(data))
43
+ })
38
44
  webSocket.on('close', () => connection.closed())
39
45
  webSocket.on('error', () => webSocket.terminate())
40
46
  })
@@ -0,0 +1,13 @@
1
+ import type { ForwardedRequestId } from '#cloud/server'
2
+
3
+ /** What a proxy's `X-Request-Id` may hold to be kept. */
4
+ const PROXY_REQUEST_ID = /^[\w-]{1,128}$/
5
+
6
+ /**
7
+ * Names a request by the `X-Request-Id` a proxy in front set, such as nginx's `$request_id`, so
8
+ * the proxy's log and the server's agree.
9
+ */
10
+ export const proxyRequestId: ForwardedRequestId = (headers) => {
11
+ const id = headers.get('x-request-id')
12
+ return id && PROXY_REQUEST_ID.test(id) ? id : undefined
13
+ }
@@ -1,10 +1,12 @@
1
1
  import { defaultRelayURL } from '#cloud/contract'
2
- import { createPostHogErrorReporter } from '#cloud/runtime/posthog/errors'
2
+ import { createPostgresCloudDatabase } from '#cloud/runtime/postgres/database'
3
+ import { createCloudErrorReporter } from '#cloud/runtime/posthog/errors'
3
4
  import { createS3ObjectStore } from '#cloud/runtime/s3/objects'
4
5
  import { createCloudServer, type SiteFiles } from '#cloud/runtime/site/server'
5
6
  import {
6
7
  createCloudApp,
7
- loadAISigningKey,
8
+ createLoggingErrorReporter,
9
+ loadServicesSigningKey,
8
10
  createCloudAuthenticationRuntime,
9
11
  createCollaborationRelay,
10
12
  createCollaborationStateStore,
@@ -15,6 +17,7 @@ import {
15
17
  reportCloudError,
16
18
  startCleanupWorker,
17
19
  startTransactionalEmailWorker,
20
+ writeJSONLine,
18
21
  type CloudErrorReporter,
19
22
  type CloudServerConfig
20
23
  } from '#cloud/server'
@@ -24,6 +27,7 @@ import { loadNodeCloudServerConfig } from './config'
24
27
  import { createNodeTransactionalEmailRuntime } from './email/runtime'
25
28
  import { listen } from './listen'
26
29
  import { attachCollaborationRelay } from './relay'
30
+ import { proxyRequestId } from './request-id'
27
31
  import { developmentSiteFiles, nodeSiteFiles } from './site'
28
32
 
29
33
  export type NodeCloudServerOptions = {
@@ -53,11 +57,14 @@ function siteFiles(
53
57
  export async function startNodeCloudServer(options: NodeCloudServerOptions = {}) {
54
58
  const environment = options.environment ?? process.env
55
59
  const config = await loadNodeCloudServerConfig(environment)
56
- const reportError = createPostHogErrorReporter(config, 'node')
60
+ const reportError = createLoggingErrorReporter(
61
+ writeJSONLine,
62
+ createCloudErrorReporter(config, 'node')
63
+ )
57
64
  try {
58
65
  return await startConfiguredServer(config, environment, options, reportError)
59
66
  } catch (error) {
60
- await reportCloudError(reportError, 'Starting the server failed:', error, { source: 'setup' })
67
+ await reportCloudError(reportError, 'Starting the server failed', error, { source: 'setup' })
61
68
  throw error
62
69
  }
63
70
  }
@@ -70,7 +77,7 @@ async function startConfiguredServer(
70
77
  ) {
71
78
  const { database } = await migratedNodeCloudDatabase(config)
72
79
  // Before any worker starts, so a key that cannot sign leaves nothing running.
73
- const aiSigningKey = await loadAISigningKey(config).catch(async (error: unknown) => {
80
+ const servicesSigningKey = await loadServicesSigningKey(config).catch(async (error: unknown) => {
74
81
  await database.destroy()
75
82
  throw error
76
83
  })
@@ -80,16 +87,39 @@ async function startConfiguredServer(
80
87
  invitationOutbox,
81
88
  transport: emailTransport
82
89
  } = createNodeTransactionalEmailRuntime(config, database)
83
- const { auth, enrollment } = createCloudAuthenticationRuntime(config, database, email)
90
+ // The adapter that serves for as long as the server runs checks the schema once it starts.
91
+ const { auth, enrollment } = createCloudAuthenticationRuntime(config, database, email, {
92
+ validateSchema: true
93
+ })
84
94
  const policy = createConfiguredCloudPolicy(config, database)
85
- const cleanup = config.cleanupEnabled
86
- ? startCleanupWorker(createCleanupServices(config, database, objects, policy), {
87
- ...cleanupPassOptions(config),
88
- intervalMs: config.cleanupIntervalMs,
89
- onError: (error) =>
90
- void reportCloudError(reportError, 'Cleanup worker failed:', error, { source: 'cleanup' })
95
+ // Cleanup sweeps may run longer than a request should, so they have a connection of their own
96
+ // without the statement timeout.
97
+ const cleanupDatabase = config.cleanupEnabled
98
+ ? createPostgresCloudDatabase({
99
+ connectionString: config.databaseURL,
100
+ statementTimeoutMs: null
91
101
  })
92
102
  : undefined
103
+ const cleanup = cleanupDatabase
104
+ ? startCleanupWorker(
105
+ createCleanupServices(
106
+ config,
107
+ cleanupDatabase,
108
+ objects,
109
+ policy,
110
+ reportError,
111
+ emailTransport ? email : undefined
112
+ ),
113
+ {
114
+ ...cleanupPassOptions(config),
115
+ intervalMs: config.cleanupIntervalMs,
116
+ onError: (error) =>
117
+ void reportCloudError(reportError, 'Cleanup worker failed', error, {
118
+ source: 'cleanup'
119
+ })
120
+ }
121
+ )
122
+ : undefined
93
123
  const relay = createCollaborationRelay({
94
124
  authSecret: config.authSecret,
95
125
  store: createCollaborationStateStore(database, objects),
@@ -107,11 +137,13 @@ async function startConfiguredServer(
107
137
  invitationOutbox,
108
138
  transactionalEmail: email,
109
139
  enrollment,
110
- aiSigningKey,
140
+ servicesSigningKey,
111
141
  relayURL: config.relayURL ?? defaultRelayURL(config.publicURL),
112
142
  onDocumentRestored: (documentId) => relay.startOver(documentId),
143
+ onDocumentTakenDown: (documentId) => relay.closeDocument(documentId),
113
144
  onDocumentVersionsChanged: (documentId) => relay.announceVersions(documentId),
114
- reportError
145
+ reportError,
146
+ forwardedRequestId: proxyRequestId
115
147
  })
116
148
  const emailWorker = emailTransport
117
149
  ? startTransactionalEmailWorker(email, {
@@ -120,13 +152,14 @@ async function startConfiguredServer(
120
152
  leaseDurationMs: config.emailLeaseDurationMs,
121
153
  maximumAttempts: config.emailMaximumAttempts,
122
154
  onError: (error) =>
123
- void reportCloudError(reportError, 'Email worker failed:', error, { source: 'email' })
155
+ void reportCloudError(reportError, 'Email worker failed', error, { source: 'email' })
124
156
  })
125
157
  : undefined
126
158
  const cloudServer = createCloudServer(app, {
127
159
  site: siteFiles(environment),
128
160
  indexingPolicy: config.indexingPolicy,
129
- reportError
161
+ reportError,
162
+ forwardedRequestId: proxyRequestId
130
163
  })
131
164
  const server = await listen(
132
165
  async (request) => cloudServer.fetch(request),
@@ -146,6 +179,7 @@ async function startConfiguredServer(
146
179
  await relayListener.close()
147
180
  await server.close()
148
181
  await cleanup?.stop()
182
+ await cleanupDatabase?.destroy()
149
183
  await emailWorker?.stop()
150
184
  await database.destroy()
151
185
  }
@@ -1,23 +1,23 @@
1
1
  #!/usr/bin/env node
2
2
  import dedent from 'dedent'
3
3
 
4
- import { startNodeAIRelay } from './server'
4
+ import { startNodeServices } from './server'
5
5
 
6
6
  if (process.argv.includes('--help') || process.argv.includes('-h')) {
7
7
  console.log(dedent`
8
- Usage: openpencil-ai-relay
8
+ Usage: openpencil-services
9
9
 
10
10
  Runs the reference AI service for a self-hosted OpenPencil Cloud. Settings come from the
11
11
  environment:
12
- OPENPENCIL_AI_RELAY_CONFIG the relay TOML file (required)
12
+ OPENPENCIL_SERVICES_CONFIG the services TOML file (required)
13
13
  PORT, HOST where to listen (default 8788 on 0.0.0.0)
14
- plus the secrets the TOML names. See https://openpencil.dev/reference/self-hosting-cloud
14
+ plus the secrets the TOML names. See https://openpencil.dev/cloud/self-hosting/
15
15
  `)
16
16
  process.exit(0)
17
17
  }
18
18
 
19
- const relay = await startNodeAIRelay()
20
- console.log(`OpenPencil AI relay listening on ${relay.url}`)
19
+ const relay = await startNodeServices()
20
+ console.log(`OpenPencil services listening on ${relay.url}`)
21
21
 
22
22
  let stopping: Promise<void> | undefined
23
23
 
@@ -25,7 +25,7 @@ function shutdown() {
25
25
  stopping ??= relay.stop().then(
26
26
  () => process.exit(0),
27
27
  (error: unknown) => {
28
- console.error('[AI relay] Shutdown failed:', error)
28
+ console.error('[Services] Shutdown failed:', error)
29
29
  process.exit(1)
30
30
  }
31
31
  )
@@ -1,47 +1,47 @@
1
1
  import { readFile } from 'node:fs/promises'
2
2
 
3
+ import { CLOUD_SERVICES_JWKS_PATH } from '#cloud/contract'
3
4
  import {
4
- AIRelayConfigError,
5
- createAIRelayApp,
6
- createAIRelayDatabase,
7
- migrateAIRelayDatabase,
8
- parseAIRelayTOML,
9
- type AIRelayEnvironment
10
- } from '#cloud/ai-relay'
11
- import { CLOUD_AI_JWKS_PATH } from '#cloud/contract'
5
+ ServicesConfigError,
6
+ createServicesApp,
7
+ createServicesDatabase,
8
+ migrateServicesDatabase,
9
+ parseServicesTOML,
10
+ type ServicesEnvironment
11
+ } from '#cloud/services'
12
12
  import { createRemoteJWKSet } from 'jose'
13
13
  import { PostgresDialect } from 'kysely'
14
14
  import { Pool } from 'pg'
15
15
 
16
16
  import { listen } from '../listen'
17
17
 
18
- export type NodeAIRelayOptions = {
19
- environment?: AIRelayEnvironment
18
+ export type NodeServicesOptions = {
19
+ environment?: ServicesEnvironment
20
20
  port?: number
21
21
  }
22
22
 
23
- export async function startNodeAIRelay(options: NodeAIRelayOptions = {}) {
23
+ export async function startNodeServices(options: NodeServicesOptions = {}) {
24
24
  const environment = options.environment ?? process.env
25
- const path = environment.OPENPENCIL_AI_RELAY_CONFIG
25
+ const path = environment.OPENPENCIL_SERVICES_CONFIG
26
26
  if (!path) {
27
- throw new AIRelayConfigError('OPENPENCIL_AI_RELAY_CONFIG must name a relay TOML file')
27
+ throw new ServicesConfigError('OPENPENCIL_SERVICES_CONFIG must name a services TOML file')
28
28
  }
29
- const config = parseAIRelayTOML(await readFile(path, 'utf8'), environment)
30
- const database = createAIRelayDatabase(
29
+ const config = parseServicesTOML(await readFile(path, 'utf8'), environment)
30
+ const database = createServicesDatabase(
31
31
  new PostgresDialect({ pool: new Pool({ connectionString: config.databaseURL }) })
32
32
  )
33
33
  try {
34
- await migrateAIRelayDatabase(database)
34
+ await migrateServicesDatabase(database)
35
35
  } catch (error) {
36
36
  await database.destroy()
37
37
  throw error
38
38
  }
39
39
  // Metering outlives its answer; shutdown waits for it so no request goes uncharged.
40
40
  const metering = new Set<Promise<unknown>>()
41
- const app = createAIRelayApp({
41
+ const app = createServicesApp({
42
42
  config,
43
43
  database,
44
- keys: createRemoteJWKSet(new URL(CLOUD_AI_JWKS_PATH, config.cloudURL)),
44
+ keys: createRemoteJWKSet(new URL(CLOUD_SERVICES_JWKS_PATH, config.cloudURL)),
45
45
  waitUntil: (promise) => {
46
46
  metering.add(promise)
47
47
  void promise.finally(() => metering.delete(promise))
@@ -2,16 +2,35 @@ import { createCloudDatabase } from '#cloud/server'
2
2
  import { PostgresDialect } from 'kysely'
3
3
  import { Pool, type PoolConfig } from 'pg'
4
4
 
5
+ /**
6
+ * How long a query serving a request may run before PostgreSQL cancels it. Rate limits and
7
+ * sessions live in PostgreSQL, so one stuck query would otherwise hold its connection, and every
8
+ * request waiting for one, indefinitely.
9
+ */
10
+ export const POSTGRES_STATEMENT_TIMEOUT_MS = 30_000
11
+
5
12
  export type PostgresCloudDatabaseOptions = {
6
13
  connectionString: string
7
- pool?: Omit<PoolConfig, 'connectionString'>
14
+ pool?: Omit<PoolConfig, 'connectionString' | 'statement_timeout' | 'query_timeout'>
15
+ /**
16
+ * How long one statement may run, `POSTGRES_STATEMENT_TIMEOUT_MS` unless set; null for no limit
17
+ * of this server's own, as migrations need and as Hyperdrive takes, which keeps no session
18
+ * settings and ends queries past its own limit. PostgreSQL cancels a statement past it, so its
19
+ * connection is free again for the next query.
20
+ */
21
+ statementTimeoutMs?: number | null
8
22
  }
9
23
 
10
24
  export function createPostgresCloudDatabase(options: PostgresCloudDatabaseOptions) {
25
+ const statementTimeoutMs =
26
+ options.statementTimeoutMs === undefined
27
+ ? POSTGRES_STATEMENT_TIMEOUT_MS
28
+ : options.statementTimeoutMs
11
29
  return createCloudDatabase({
12
30
  dialect: new PostgresDialect({
13
31
  pool: new Pool({
14
32
  ...options.pool,
33
+ ...(statementTimeoutMs === null ? {} : { statement_timeout: statementTimeoutMs }),
15
34
  connectionString: options.connectionString
16
35
  })
17
36
  })
@@ -1,48 +1,33 @@
1
+ import { createPostHogErrorReporter } from '#cloud/errors'
1
2
  import type { CloudServerConfig } from '#cloud/server'
2
3
  import type { CloudErrorReporter } from '#cloud/server/errors'
3
- import { PostHog, type PostHogOptions } from 'posthog-node'
4
+ import type { PostHogOptions } from 'posthog-node'
4
5
 
5
6
  /** Which runtime serves the deployment, as its error reports name it. */
6
7
  export type CloudRuntimeName = 'node' | 'cloudflare'
7
8
 
8
9
  /**
9
10
  * Reports server errors to the PostHog project the deployment's `[error_tracking]` names, or
10
- * returns nothing when it names none. Each error is sent as it is caught, as a `$exception` with
11
- * no person profile: reports carry the runtime, the deployment, and where the error was caught,
12
- * never who made the request or what it carried.
11
+ * returns nothing when it names none. Reports carry the runtime, the deployment, and where the
12
+ * error was caught, never who made the request or what it carried.
13
13
  */
14
- export function createPostHogErrorReporter(
14
+ export function createCloudErrorReporter(
15
15
  config: CloudServerConfig,
16
16
  runtime: CloudRuntimeName,
17
17
  options: Pick<PostHogOptions, 'fetch'> = {}
18
18
  ): CloudErrorReporter | undefined {
19
19
  const tracking = config.errorTracking
20
20
  if (!tracking) return undefined
21
- const client = new PostHog(tracking.projectAPIKey, {
21
+ return createPostHogErrorReporter({
22
+ key: tracking.projectAPIKey,
22
23
  host: tracking.host,
23
- disableGeoip: true,
24
- ...options
24
+ properties: {
25
+ cloud_runtime: runtime,
26
+ cloud_deployment: config.deployment,
27
+ cloud_host: new URL(config.publicURL).host
28
+ },
29
+ sourceProperty: 'cloud_source',
30
+ logName: 'Cloud',
31
+ fetch: options.fetch
25
32
  })
26
- // The SDK resolves a capture that PostHog refused and emits the refusal instead.
27
- client.on('error', (failure: unknown) => {
28
- console.error('[Cloud] PostHog refused an error report:', failure)
29
- })
30
- const deployment = {
31
- cloud_runtime: runtime,
32
- cloud_deployment: config.deployment,
33
- cloud_host: new URL(config.publicURL).host
34
- }
35
- return async (error, context) => {
36
- try {
37
- // Undefined request fields are left out of the event's JSON.
38
- await client.captureExceptionImmediate(error, undefined, {
39
- ...deployment,
40
- cloud_source: context.source,
41
- http_method: context.method,
42
- http_route: context.route
43
- })
44
- } catch (failure) {
45
- console.error('[Cloud] Reporting an error failed:', failure)
46
- }
47
- }
48
33
  }