@open-pencil/cloud 0.17.0-beta.2 → 0.17.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/{ai-relay.mjs → services.mjs} +8 -8
- package/dist/bin/services.mjs.map +1 -0
- package/dist/client.d.ts +236 -46
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +649 -400
- package/dist/client.js.map +1 -1
- package/dist/contract.d.ts +2 -2
- package/dist/contract.js +2 -2
- package/dist/contract2.js +680 -443
- package/dist/contract2.js.map +1 -1
- package/dist/email.d.ts +9 -1
- package/dist/email.d.ts.map +1 -1
- package/dist/email.js +99 -58
- package/dist/email.js.map +1 -1
- package/dist/index.d.ts +469 -65
- package/dist/index.d.ts.map +1 -1
- package/dist/listen.mjs +505 -787
- package/dist/listen.mjs.map +1 -1
- package/dist/runtime-node.d.mts +3920 -2524
- package/dist/runtime-node.d.mts.map +1 -1
- package/dist/runtime-node.mjs +6 -5
- package/dist/runtime-node.mjs.map +1 -1
- package/dist/server.d.ts +3586 -1897
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +5216 -4272
- package/dist/server.js.map +1 -1
- package/dist/server.mjs +5179 -3623
- package/dist/server.mjs.map +1 -1
- package/dist/server2.mjs +101 -79
- package/dist/server2.mjs.map +1 -1
- package/package.json +7 -6
- package/src/admin/api/routes.ts +59 -0
- package/src/admin/audit/service.ts +25 -6
- package/src/admin/documents/service.ts +361 -0
- package/src/admin/errors.ts +9 -1
- package/src/admin/index.ts +6 -0
- package/src/admin/operations/service.ts +7 -0
- package/src/admin/reports/service.ts +147 -0
- package/src/admin/schema.ts +6 -0
- package/src/client/api.ts +36 -21
- package/src/client/auth/navigation.ts +6 -5
- package/src/client/index.ts +7 -7
- package/src/client/portal/account.ts +25 -0
- package/src/client/portal/administration.ts +56 -0
- package/src/client/{ai.ts → services.ts} +44 -27
- package/src/contract/account.ts +58 -2
- package/src/contract/admin/index.ts +67 -3
- package/src/contract/collaboration.ts +6 -1
- package/src/contract/discovery.ts +22 -4
- package/src/contract/documents.ts +27 -0
- package/src/contract/entitlements.ts +2 -2
- package/src/contract/index.ts +63 -37
- package/src/contract/relay.ts +6 -0
- package/src/contract/reports.ts +42 -0
- package/src/contract/services.ts +233 -0
- package/src/contract/sharing.ts +3 -9
- package/src/email/render.ts +26 -0
- package/src/email/schema.ts +11 -1
- package/src/email/templates/brand.ts +1 -1
- package/src/runtime/cloudflare/environment.ts +22 -11
- package/src/runtime/cloudflare/index.ts +1 -1
- package/src/runtime/cloudflare/logging.ts +15 -0
- package/src/runtime/cloudflare/maintenance.ts +31 -0
- package/src/runtime/cloudflare/relay/index.ts +62 -14
- package/src/runtime/cloudflare/worker.ts +39 -13
- package/src/runtime/node/application.ts +5 -3
- package/src/runtime/node/bootstrap.ts +15 -4
- package/src/runtime/node/cleanup.ts +16 -3
- package/src/runtime/node/entitlements/main.ts +14 -11
- package/src/runtime/node/index.ts +1 -1
- package/src/runtime/node/migrate.ts +5 -1
- package/src/runtime/node/relay.ts +8 -2
- package/src/runtime/node/request-id.ts +13 -0
- package/src/runtime/node/server.ts +39 -15
- package/src/runtime/node/{ai-relay → services}/main.ts +6 -6
- package/src/runtime/node/{ai-relay → services}/server.ts +18 -18
- package/src/runtime/postgres/database.ts +20 -1
- package/src/runtime/posthog/errors.ts +15 -31
- package/src/runtime/s3/objects.ts +24 -1
- package/src/runtime/site/server.ts +32 -5
- package/src/server/account/actions.ts +5 -0
- package/src/server/account/deletion.ts +302 -0
- package/src/server/account/routes.ts +47 -3
- package/src/server/account/terms.ts +42 -0
- package/src/server/api/router.ts +9 -5
- package/src/server/api/types.ts +2 -0
- package/src/server/app.ts +90 -16
- package/src/server/auth/access-control.ts +63 -0
- package/src/server/auth/adapter.ts +11 -0
- package/src/server/auth/endpoints.ts +5 -1
- package/src/server/auth/factory.ts +30 -3
- package/src/server/auth/index.ts +10 -1
- package/src/server/cleanup/configured.ts +12 -5
- package/src/server/cleanup/documents.ts +165 -71
- package/src/server/cleanup/revisions.ts +16 -5
- package/src/server/cleanup/uploads.ts +8 -2
- package/src/server/cleanup/worker.ts +5 -2
- package/src/server/collaboration/relay/connection.ts +4 -3
- package/src/server/collaboration/relay/documents.ts +13 -10
- package/src/server/collaboration/relay/hub.ts +97 -25
- package/src/server/collaboration/relay/log.ts +5 -2
- package/src/server/collaboration/service.ts +27 -31
- package/src/server/config/deployment.ts +24 -4
- package/src/server/config/public.ts +5 -4
- package/src/server/config/schema.ts +82 -16
- package/src/server/db/expressions.ts +15 -1
- package/src/server/db/migrate.ts +34 -116
- package/src/server/db/migrations/001_accounts.ts +79 -0
- package/src/server/db/migrations/002_workspaces.ts +48 -0
- package/src/server/db/migrations/003_storage.ts +44 -0
- package/src/server/db/migrations/004_documents.ts +193 -0
- package/src/server/db/migrations/005_sharing.ts +95 -0
- package/src/server/db/migrations/006_operations.ts +110 -0
- package/src/server/db/migrations/007_trash.ts +33 -0
- package/src/server/db/migrations/008_terms.ts +18 -0
- package/src/server/db/migrations/schema.ts +71 -0
- package/src/server/db/tables.ts +52 -3
- package/src/server/documents/access.ts +66 -55
- package/src/server/documents/errors.ts +7 -0
- package/src/server/documents/images.ts +2 -1
- package/src/server/documents/limits.ts +9 -9
- package/src/server/documents/purge.ts +75 -0
- package/src/server/documents/repository.ts +2 -0
- package/src/server/documents/routes.ts +26 -0
- package/src/server/documents/service.ts +18 -4
- package/src/server/documents/summary.ts +1 -0
- package/src/server/documents/trash.ts +203 -0
- package/src/server/documents/versions/checkpoint.ts +2 -1
- package/src/server/documents/versions/history.ts +11 -7
- package/src/server/documents/versions/service.ts +13 -4
- package/src/server/errors/index.ts +59 -16
- package/src/server/errors/posthog.ts +80 -0
- package/src/server/index.ts +46 -17
- package/src/server/internal/index.ts +3 -3
- package/src/server/internal/routes.ts +7 -10
- package/src/server/internal/schema.ts +1 -1
- package/src/server/internal/service.ts +17 -23
- package/src/server/invitations/budget.ts +66 -0
- package/src/server/invitations/index.ts +1 -0
- package/src/server/logging.ts +76 -0
- package/src/server/pictures/service.ts +2 -1
- package/src/server/policy/database.ts +39 -25
- package/src/server/policy/index.ts +2 -1
- package/src/server/policy/keys.ts +5 -2
- package/src/server/policy/policy.ts +1 -0
- package/src/server/policy/schema.ts +2 -2
- package/src/server/policy/service.ts +12 -9
- package/src/server/policy/static.ts +5 -5
- package/src/server/policy/workspace.ts +41 -0
- package/src/server/quota/reconcile.ts +2 -0
- package/src/server/quota/storage.ts +69 -31
- package/src/server/rate-limit/cleanup.ts +17 -5
- package/src/server/rate-limit/middleware.ts +2 -2
- package/src/server/reports/index.ts +8 -0
- package/src/server/reports/routes.ts +37 -0
- package/src/server/reports/service.ts +123 -0
- package/src/server/services/errors.ts +7 -0
- package/src/server/services/index.ts +8 -0
- package/src/server/{ai → services}/keys.ts +11 -11
- package/src/server/services/routes.ts +47 -0
- package/src/server/{ai/service.ts → services/tokens.ts} +23 -20
- package/src/server/sharing/capabilities/service.ts +68 -22
- package/src/server/sharing/grants/service.ts +11 -18
- package/src/server/sharing/http.ts +4 -0
- package/src/server/sharing/invitations/service.ts +21 -11
- package/src/server/sharing/policy.ts +9 -14
- package/src/server/sharing/routes.ts +0 -16
- package/src/server/sharing/service.ts +1 -17
- package/src/server/validation.ts +1 -0
- package/src/server/workspaces/index.ts +1 -0
- package/src/server/workspaces/members.ts +89 -38
- package/src/server/workspaces/repository.ts +2 -1
- package/src/server/workspaces/routes.ts +4 -0
- package/src/{ai-relay → services}/app.ts +48 -44
- package/src/{ai-relay → services}/config.ts +25 -19
- package/src/services/db/index.ts +10 -0
- package/src/services/db/migrate.ts +37 -0
- package/src/{ai-relay/db/migrations/001_usage.ts → services/db/migrations/001_baseline.ts} +18 -7
- package/src/services/db/schema.ts +22 -0
- package/src/services/index.ts +18 -0
- package/src/{ai-relay → services}/metering.ts +2 -2
- package/src/{ai-relay → services}/usage.ts +43 -19
- package/dist/bin/ai-relay.mjs.map +0 -1
- package/src/ai-relay/db/index.ts +0 -10
- package/src/ai-relay/db/migrate.ts +0 -32
- package/src/ai-relay/db/schema.ts +0 -20
- package/src/ai-relay/index.ts +0 -18
- package/src/contract/ai.ts +0 -150
- package/src/server/ai/errors.ts +0 -7
- package/src/server/ai/index.ts +0 -4
- package/src/server/ai/routes.ts +0 -43
- package/src/server/auth/workspace-access.ts +0 -23
- package/src/server/db/migrations/001_foundation.ts +0 -164
- package/src/server/db/migrations/002_upload_cleanup_claims.ts +0 -34
- package/src/server/db/migrations/003_document_cleanup_claims.ts +0 -23
- package/src/server/db/migrations/004_document_sharing.ts +0 -82
- package/src/server/db/migrations/005_document_collaboration_epoch.ts +0 -13
- package/src/server/db/migrations/006_invitation_continuation.ts +0 -27
- package/src/server/db/migrations/007_collaboration_state.ts +0 -25
- package/src/server/db/migrations/008_storage_reservations.ts +0 -37
- package/src/server/db/migrations/009_workspace_entitlements.ts +0 -22
- package/src/server/db/migrations/010_upload_finalization.ts +0 -22
- package/src/server/db/migrations/011_upload_finalization_lease.ts +0 -12
- package/src/server/db/migrations/012_transactional_email.ts +0 -48
- package/src/server/db/migrations/013_admin_enrollment.ts +0 -54
- package/src/server/db/migrations/014_enrollment_email_kinds.ts +0 -30
- package/src/server/db/migrations/015_cloud_rate_limit.ts +0 -22
- package/src/server/db/migrations/016_auth_schema.ts +0 -16
- package/src/server/db/migrations/017_authentication_email_kinds.ts +0 -32
- package/src/server/db/migrations/018_authentication_mfa.ts +0 -78
- package/src/server/db/migrations/019_better_auth_account_issuer.ts +0 -42
- package/src/server/db/migrations/020_workspace_invitations.ts +0 -57
- package/src/server/db/migrations/021_document_versions.ts +0 -66
- package/src/server/db/migrations/022_entitlement_external_version.ts +0 -17
- package/src/server/db/migrations/023_document_images.ts +0 -61
- package/src/server/db/migrations/024_account_activity.ts +0 -66
- package/src/server/db/migrations/025_table_names.ts +0 -58
- package/src/server/db/migrations/026_user_picture.ts +0 -30
- package/src/server/db/migrations/027_document_kind.ts +0 -19
- package/src/server/db/migrations/028_collaboration_updates.ts +0 -84
- package/src/server/db/migrations/helpers.ts +0 -78
- /package/dist/bin/{ai-relay.d.mts → services.d.mts} +0 -0
|
@@ -1,40 +1,43 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises'
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { createAccountEntitlementRepository, parseStaticEntitlementsTOML } from '#cloud/server'
|
|
4
4
|
import { parse as parseTOML } from 'smol-toml'
|
|
5
5
|
|
|
6
6
|
import { withNodeCloudDatabase } from '../command'
|
|
7
7
|
import { entitlementWriteRefusal } from './refusal'
|
|
8
8
|
|
|
9
9
|
const operation = process.argv[2]
|
|
10
|
-
const
|
|
10
|
+
const accountIndex = process.argv.indexOf('--account')
|
|
11
11
|
const fileIndex = process.argv.indexOf('--file')
|
|
12
12
|
const dryRun = process.argv.includes('--dry-run')
|
|
13
|
-
const
|
|
14
|
-
if (!
|
|
13
|
+
const accountArgument = accountIndex !== -1 ? process.argv[accountIndex + 1] : undefined
|
|
14
|
+
if (!accountArgument) throw new Error('Expected --account <user id>')
|
|
15
15
|
|
|
16
16
|
await withNodeCloudDatabase(async (database, config) => {
|
|
17
|
-
const repository =
|
|
17
|
+
const repository = createAccountEntitlementRepository(database)
|
|
18
18
|
const refusal = entitlementWriteRefusal(config, operation, dryRun)
|
|
19
19
|
if (refusal) throw new Error(refusal)
|
|
20
|
+
// A plan is stored under the account as the user table spells it, which policy reads.
|
|
21
|
+
const userId = await repository.account(accountArgument)
|
|
22
|
+
if (!userId) throw new Error(`No account ${accountArgument}`)
|
|
20
23
|
if (operation === 'get') {
|
|
21
|
-
console.log(JSON.stringify({ entitlement: await repository.get(
|
|
24
|
+
console.log(JSON.stringify({ entitlement: await repository.get(userId) }))
|
|
22
25
|
} else if (operation === 'clear') {
|
|
23
26
|
if (dryRun) {
|
|
24
|
-
console.log(JSON.stringify({ dryRun, cleared: false,
|
|
27
|
+
console.log(JSON.stringify({ dryRun, cleared: false, userId }))
|
|
25
28
|
return
|
|
26
29
|
}
|
|
27
|
-
await repository.clear(
|
|
28
|
-
console.log(JSON.stringify({ cleared: true,
|
|
30
|
+
await repository.clear(userId)
|
|
31
|
+
console.log(JSON.stringify({ cleared: true, userId }))
|
|
29
32
|
} else if (operation === 'set') {
|
|
30
33
|
const file = fileIndex !== -1 ? process.argv[fileIndex + 1] : undefined
|
|
31
34
|
if (!file) throw new Error('Expected --file <path>')
|
|
32
35
|
const values = parseStaticEntitlementsTOML(parseTOML(await readFile(file, 'utf8')))
|
|
33
36
|
if (dryRun) {
|
|
34
|
-
console.log(JSON.stringify({ dryRun,
|
|
37
|
+
console.log(JSON.stringify({ dryRun, userId, values }))
|
|
35
38
|
return
|
|
36
39
|
}
|
|
37
|
-
console.log(JSON.stringify({ entitlement: await repository.set(
|
|
40
|
+
console.log(JSON.stringify({ entitlement: await repository.set(userId, values, 'cli') }))
|
|
38
41
|
} else {
|
|
39
42
|
throw new Error('Expected operation: get, set, or clear')
|
|
40
43
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { startNodeServices, type NodeServicesOptions } from './services/server'
|
|
2
2
|
export { createMigratedNodeCloudDatabase } from './bootstrap'
|
|
3
3
|
export { createNodeCloudApplication, type NodeCloudApplicationOptions } from './application'
|
|
4
4
|
export { loadNodeCloudServerConfig } from './config'
|
|
@@ -4,7 +4,11 @@ import { createBetterAuthAdapter, migrateCloudDatabase } from '#cloud/server'
|
|
|
4
4
|
import { loadNodeCloudServerConfig } from './config'
|
|
5
5
|
|
|
6
6
|
const config = await loadNodeCloudServerConfig(process.env)
|
|
7
|
-
|
|
7
|
+
// A migration may rewrite a large table, which takes as long as it takes.
|
|
8
|
+
const database = createPostgresCloudDatabase({
|
|
9
|
+
connectionString: config.databaseURL,
|
|
10
|
+
statementTimeoutMs: null
|
|
11
|
+
})
|
|
8
12
|
try {
|
|
9
13
|
const auth = createBetterAuthAdapter(config, database)
|
|
10
14
|
await migrateCloudDatabase(database, { run: auth.migrate, schemaVersion: auth.schemaVersion })
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { IncomingMessage, Server } from 'node:http'
|
|
2
2
|
import type { Duplex } from 'node:stream'
|
|
3
3
|
|
|
4
|
-
import { RELAY_PATH } from '#cloud/contract'
|
|
4
|
+
import { RELAY_HEARTBEAT, RELAY_PATH } from '#cloud/contract'
|
|
5
5
|
import type { CollaborationRelay } from '#cloud/server'
|
|
6
6
|
import { WebSocketServer, type RawData } from 'ws'
|
|
7
7
|
|
|
@@ -34,7 +34,13 @@ export function attachCollaborationRelay(
|
|
|
34
34
|
send: (data) => webSocket.send(data),
|
|
35
35
|
close: (code, reason) => webSocket.close(code, reason)
|
|
36
36
|
})
|
|
37
|
-
webSocket.on('message', (data) =>
|
|
37
|
+
webSocket.on('message', (data, isBinary) => {
|
|
38
|
+
if (!isBinary && new TextDecoder().decode(bytes(data)) === RELAY_HEARTBEAT.ping) {
|
|
39
|
+
webSocket.send(RELAY_HEARTBEAT.pong)
|
|
40
|
+
return
|
|
41
|
+
}
|
|
42
|
+
connection.receive(bytes(data))
|
|
43
|
+
})
|
|
38
44
|
webSocket.on('close', () => connection.closed())
|
|
39
45
|
webSocket.on('error', () => webSocket.terminate())
|
|
40
46
|
})
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { ForwardedRequestId } from '#cloud/server'
|
|
2
|
+
|
|
3
|
+
/** What a proxy's `X-Request-Id` may hold to be kept. */
|
|
4
|
+
const PROXY_REQUEST_ID = /^[\w-]{1,128}$/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Names a request by the `X-Request-Id` a proxy in front set, such as nginx's `$request_id`, so
|
|
8
|
+
* the proxy's log and the server's agree.
|
|
9
|
+
*/
|
|
10
|
+
export const proxyRequestId: ForwardedRequestId = (headers) => {
|
|
11
|
+
const id = headers.get('x-request-id')
|
|
12
|
+
return id && PROXY_REQUEST_ID.test(id) ? id : undefined
|
|
13
|
+
}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { defaultRelayURL } from '#cloud/contract'
|
|
2
|
-
import {
|
|
2
|
+
import { createPostgresCloudDatabase } from '#cloud/runtime/postgres/database'
|
|
3
|
+
import { createCloudErrorReporter } from '#cloud/runtime/posthog/errors'
|
|
3
4
|
import { createS3ObjectStore } from '#cloud/runtime/s3/objects'
|
|
4
5
|
import { createCloudServer, type SiteFiles } from '#cloud/runtime/site/server'
|
|
5
6
|
import {
|
|
6
7
|
createCloudApp,
|
|
7
|
-
|
|
8
|
+
createLoggingErrorReporter,
|
|
9
|
+
loadServicesSigningKey,
|
|
8
10
|
createCloudAuthenticationRuntime,
|
|
9
11
|
createCollaborationRelay,
|
|
10
12
|
createCollaborationStateStore,
|
|
@@ -15,6 +17,7 @@ import {
|
|
|
15
17
|
reportCloudError,
|
|
16
18
|
startCleanupWorker,
|
|
17
19
|
startTransactionalEmailWorker,
|
|
20
|
+
writeJSONLine,
|
|
18
21
|
type CloudErrorReporter,
|
|
19
22
|
type CloudServerConfig
|
|
20
23
|
} from '#cloud/server'
|
|
@@ -24,6 +27,7 @@ import { loadNodeCloudServerConfig } from './config'
|
|
|
24
27
|
import { createNodeTransactionalEmailRuntime } from './email/runtime'
|
|
25
28
|
import { listen } from './listen'
|
|
26
29
|
import { attachCollaborationRelay } from './relay'
|
|
30
|
+
import { proxyRequestId } from './request-id'
|
|
27
31
|
import { developmentSiteFiles, nodeSiteFiles } from './site'
|
|
28
32
|
|
|
29
33
|
export type NodeCloudServerOptions = {
|
|
@@ -53,11 +57,14 @@ function siteFiles(
|
|
|
53
57
|
export async function startNodeCloudServer(options: NodeCloudServerOptions = {}) {
|
|
54
58
|
const environment = options.environment ?? process.env
|
|
55
59
|
const config = await loadNodeCloudServerConfig(environment)
|
|
56
|
-
const reportError =
|
|
60
|
+
const reportError = createLoggingErrorReporter(
|
|
61
|
+
writeJSONLine,
|
|
62
|
+
createCloudErrorReporter(config, 'node')
|
|
63
|
+
)
|
|
57
64
|
try {
|
|
58
65
|
return await startConfiguredServer(config, environment, options, reportError)
|
|
59
66
|
} catch (error) {
|
|
60
|
-
await reportCloudError(reportError, 'Starting the server failed
|
|
67
|
+
await reportCloudError(reportError, 'Starting the server failed', error, { source: 'setup' })
|
|
61
68
|
throw error
|
|
62
69
|
}
|
|
63
70
|
}
|
|
@@ -70,7 +77,7 @@ async function startConfiguredServer(
|
|
|
70
77
|
) {
|
|
71
78
|
const { database } = await migratedNodeCloudDatabase(config)
|
|
72
79
|
// Before any worker starts, so a key that cannot sign leaves nothing running.
|
|
73
|
-
const
|
|
80
|
+
const servicesSigningKey = await loadServicesSigningKey(config).catch(async (error: unknown) => {
|
|
74
81
|
await database.destroy()
|
|
75
82
|
throw error
|
|
76
83
|
})
|
|
@@ -82,14 +89,27 @@ async function startConfiguredServer(
|
|
|
82
89
|
} = createNodeTransactionalEmailRuntime(config, database)
|
|
83
90
|
const { auth, enrollment } = createCloudAuthenticationRuntime(config, database, email)
|
|
84
91
|
const policy = createConfiguredCloudPolicy(config, database)
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
92
|
+
// Cleanup sweeps may run longer than a request should, so they have a connection of their own
|
|
93
|
+
// without the statement timeout.
|
|
94
|
+
const cleanupDatabase = config.cleanupEnabled
|
|
95
|
+
? createPostgresCloudDatabase({
|
|
96
|
+
connectionString: config.databaseURL,
|
|
97
|
+
statementTimeoutMs: null
|
|
91
98
|
})
|
|
92
99
|
: undefined
|
|
100
|
+
const cleanup = cleanupDatabase
|
|
101
|
+
? startCleanupWorker(
|
|
102
|
+
createCleanupServices(config, cleanupDatabase, objects, policy, reportError),
|
|
103
|
+
{
|
|
104
|
+
...cleanupPassOptions(config),
|
|
105
|
+
intervalMs: config.cleanupIntervalMs,
|
|
106
|
+
onError: (error) =>
|
|
107
|
+
void reportCloudError(reportError, 'Cleanup worker failed', error, {
|
|
108
|
+
source: 'cleanup'
|
|
109
|
+
})
|
|
110
|
+
}
|
|
111
|
+
)
|
|
112
|
+
: undefined
|
|
93
113
|
const relay = createCollaborationRelay({
|
|
94
114
|
authSecret: config.authSecret,
|
|
95
115
|
store: createCollaborationStateStore(database, objects),
|
|
@@ -107,11 +127,13 @@ async function startConfiguredServer(
|
|
|
107
127
|
invitationOutbox,
|
|
108
128
|
transactionalEmail: email,
|
|
109
129
|
enrollment,
|
|
110
|
-
|
|
130
|
+
servicesSigningKey,
|
|
111
131
|
relayURL: config.relayURL ?? defaultRelayURL(config.publicURL),
|
|
112
132
|
onDocumentRestored: (documentId) => relay.startOver(documentId),
|
|
133
|
+
onDocumentTakenDown: (documentId) => relay.closeDocument(documentId),
|
|
113
134
|
onDocumentVersionsChanged: (documentId) => relay.announceVersions(documentId),
|
|
114
|
-
reportError
|
|
135
|
+
reportError,
|
|
136
|
+
forwardedRequestId: proxyRequestId
|
|
115
137
|
})
|
|
116
138
|
const emailWorker = emailTransport
|
|
117
139
|
? startTransactionalEmailWorker(email, {
|
|
@@ -120,13 +142,14 @@ async function startConfiguredServer(
|
|
|
120
142
|
leaseDurationMs: config.emailLeaseDurationMs,
|
|
121
143
|
maximumAttempts: config.emailMaximumAttempts,
|
|
122
144
|
onError: (error) =>
|
|
123
|
-
void reportCloudError(reportError, 'Email worker failed
|
|
145
|
+
void reportCloudError(reportError, 'Email worker failed', error, { source: 'email' })
|
|
124
146
|
})
|
|
125
147
|
: undefined
|
|
126
148
|
const cloudServer = createCloudServer(app, {
|
|
127
149
|
site: siteFiles(environment),
|
|
128
150
|
indexingPolicy: config.indexingPolicy,
|
|
129
|
-
reportError
|
|
151
|
+
reportError,
|
|
152
|
+
forwardedRequestId: proxyRequestId
|
|
130
153
|
})
|
|
131
154
|
const server = await listen(
|
|
132
155
|
async (request) => cloudServer.fetch(request),
|
|
@@ -146,6 +169,7 @@ async function startConfiguredServer(
|
|
|
146
169
|
await relayListener.close()
|
|
147
170
|
await server.close()
|
|
148
171
|
await cleanup?.stop()
|
|
172
|
+
await cleanupDatabase?.destroy()
|
|
149
173
|
await emailWorker?.stop()
|
|
150
174
|
await database.destroy()
|
|
151
175
|
}
|
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import dedent from 'dedent'
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import { startNodeServices } from './server'
|
|
5
5
|
|
|
6
6
|
if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
|
7
7
|
console.log(dedent`
|
|
8
|
-
Usage: openpencil-
|
|
8
|
+
Usage: openpencil-services
|
|
9
9
|
|
|
10
10
|
Runs the reference AI service for a self-hosted OpenPencil Cloud. Settings come from the
|
|
11
11
|
environment:
|
|
12
|
-
|
|
12
|
+
OPENPENCIL_SERVICES_CONFIG the services TOML file (required)
|
|
13
13
|
PORT, HOST where to listen (default 8788 on 0.0.0.0)
|
|
14
14
|
plus the secrets the TOML names. See https://openpencil.dev/reference/self-hosting-cloud
|
|
15
15
|
`)
|
|
16
16
|
process.exit(0)
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
const relay = await
|
|
20
|
-
console.log(`OpenPencil
|
|
19
|
+
const relay = await startNodeServices()
|
|
20
|
+
console.log(`OpenPencil services listening on ${relay.url}`)
|
|
21
21
|
|
|
22
22
|
let stopping: Promise<void> | undefined
|
|
23
23
|
|
|
@@ -25,7 +25,7 @@ function shutdown() {
|
|
|
25
25
|
stopping ??= relay.stop().then(
|
|
26
26
|
() => process.exit(0),
|
|
27
27
|
(error: unknown) => {
|
|
28
|
-
console.error('[
|
|
28
|
+
console.error('[Services] Shutdown failed:', error)
|
|
29
29
|
process.exit(1)
|
|
30
30
|
}
|
|
31
31
|
)
|
|
@@ -1,47 +1,47 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises'
|
|
2
2
|
|
|
3
|
+
import { CLOUD_SERVICES_JWKS_PATH } from '#cloud/contract'
|
|
3
4
|
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
type
|
|
10
|
-
} from '#cloud/
|
|
11
|
-
import { CLOUD_AI_JWKS_PATH } from '#cloud/contract'
|
|
5
|
+
ServicesConfigError,
|
|
6
|
+
createServicesApp,
|
|
7
|
+
createServicesDatabase,
|
|
8
|
+
migrateServicesDatabase,
|
|
9
|
+
parseServicesTOML,
|
|
10
|
+
type ServicesEnvironment
|
|
11
|
+
} from '#cloud/services'
|
|
12
12
|
import { createRemoteJWKSet } from 'jose'
|
|
13
13
|
import { PostgresDialect } from 'kysely'
|
|
14
14
|
import { Pool } from 'pg'
|
|
15
15
|
|
|
16
16
|
import { listen } from '../listen'
|
|
17
17
|
|
|
18
|
-
export type
|
|
19
|
-
environment?:
|
|
18
|
+
export type NodeServicesOptions = {
|
|
19
|
+
environment?: ServicesEnvironment
|
|
20
20
|
port?: number
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
export async function
|
|
23
|
+
export async function startNodeServices(options: NodeServicesOptions = {}) {
|
|
24
24
|
const environment = options.environment ?? process.env
|
|
25
|
-
const path = environment.
|
|
25
|
+
const path = environment.OPENPENCIL_SERVICES_CONFIG
|
|
26
26
|
if (!path) {
|
|
27
|
-
throw new
|
|
27
|
+
throw new ServicesConfigError('OPENPENCIL_SERVICES_CONFIG must name a services TOML file')
|
|
28
28
|
}
|
|
29
|
-
const config =
|
|
30
|
-
const database =
|
|
29
|
+
const config = parseServicesTOML(await readFile(path, 'utf8'), environment)
|
|
30
|
+
const database = createServicesDatabase(
|
|
31
31
|
new PostgresDialect({ pool: new Pool({ connectionString: config.databaseURL }) })
|
|
32
32
|
)
|
|
33
33
|
try {
|
|
34
|
-
await
|
|
34
|
+
await migrateServicesDatabase(database)
|
|
35
35
|
} catch (error) {
|
|
36
36
|
await database.destroy()
|
|
37
37
|
throw error
|
|
38
38
|
}
|
|
39
39
|
// Metering outlives its answer; shutdown waits for it so no request goes uncharged.
|
|
40
40
|
const metering = new Set<Promise<unknown>>()
|
|
41
|
-
const app =
|
|
41
|
+
const app = createServicesApp({
|
|
42
42
|
config,
|
|
43
43
|
database,
|
|
44
|
-
keys: createRemoteJWKSet(new URL(
|
|
44
|
+
keys: createRemoteJWKSet(new URL(CLOUD_SERVICES_JWKS_PATH, config.cloudURL)),
|
|
45
45
|
waitUntil: (promise) => {
|
|
46
46
|
metering.add(promise)
|
|
47
47
|
void promise.finally(() => metering.delete(promise))
|
|
@@ -2,16 +2,35 @@ import { createCloudDatabase } from '#cloud/server'
|
|
|
2
2
|
import { PostgresDialect } from 'kysely'
|
|
3
3
|
import { Pool, type PoolConfig } from 'pg'
|
|
4
4
|
|
|
5
|
+
/**
|
|
6
|
+
* How long a query serving a request may run before PostgreSQL cancels it. Rate limits and
|
|
7
|
+
* sessions live in PostgreSQL, so one stuck query would otherwise hold its connection, and every
|
|
8
|
+
* request waiting for one, indefinitely.
|
|
9
|
+
*/
|
|
10
|
+
export const POSTGRES_STATEMENT_TIMEOUT_MS = 30_000
|
|
11
|
+
|
|
5
12
|
export type PostgresCloudDatabaseOptions = {
|
|
6
13
|
connectionString: string
|
|
7
|
-
pool?: Omit<PoolConfig, 'connectionString'>
|
|
14
|
+
pool?: Omit<PoolConfig, 'connectionString' | 'statement_timeout' | 'query_timeout'>
|
|
15
|
+
/**
|
|
16
|
+
* How long one statement may run, `POSTGRES_STATEMENT_TIMEOUT_MS` unless set; null for no limit
|
|
17
|
+
* of this server's own, as migrations need and as Hyperdrive takes, which keeps no session
|
|
18
|
+
* settings and ends queries past its own limit. PostgreSQL cancels a statement past it, so its
|
|
19
|
+
* connection is free again for the next query.
|
|
20
|
+
*/
|
|
21
|
+
statementTimeoutMs?: number | null
|
|
8
22
|
}
|
|
9
23
|
|
|
10
24
|
export function createPostgresCloudDatabase(options: PostgresCloudDatabaseOptions) {
|
|
25
|
+
const statementTimeoutMs =
|
|
26
|
+
options.statementTimeoutMs === undefined
|
|
27
|
+
? POSTGRES_STATEMENT_TIMEOUT_MS
|
|
28
|
+
: options.statementTimeoutMs
|
|
11
29
|
return createCloudDatabase({
|
|
12
30
|
dialect: new PostgresDialect({
|
|
13
31
|
pool: new Pool({
|
|
14
32
|
...options.pool,
|
|
33
|
+
...(statementTimeoutMs === null ? {} : { statement_timeout: statementTimeoutMs }),
|
|
15
34
|
connectionString: options.connectionString
|
|
16
35
|
})
|
|
17
36
|
})
|
|
@@ -1,48 +1,32 @@
|
|
|
1
1
|
import type { CloudServerConfig } from '#cloud/server'
|
|
2
|
-
import type
|
|
3
|
-
import {
|
|
2
|
+
import { createPostHogErrorReporter, type CloudErrorReporter } from '#cloud/server/errors'
|
|
3
|
+
import type { PostHogOptions } from 'posthog-node'
|
|
4
4
|
|
|
5
5
|
/** Which runtime serves the deployment, as its error reports name it. */
|
|
6
6
|
export type CloudRuntimeName = 'node' | 'cloudflare'
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Reports server errors to the PostHog project the deployment's `[error_tracking]` names, or
|
|
10
|
-
* returns nothing when it names none.
|
|
11
|
-
*
|
|
12
|
-
* never who made the request or what it carried.
|
|
10
|
+
* returns nothing when it names none. Reports carry the runtime, the deployment, and where the
|
|
11
|
+
* error was caught, never who made the request or what it carried.
|
|
13
12
|
*/
|
|
14
|
-
export function
|
|
13
|
+
export function createCloudErrorReporter(
|
|
15
14
|
config: CloudServerConfig,
|
|
16
15
|
runtime: CloudRuntimeName,
|
|
17
16
|
options: Pick<PostHogOptions, 'fetch'> = {}
|
|
18
17
|
): CloudErrorReporter | undefined {
|
|
19
18
|
const tracking = config.errorTracking
|
|
20
19
|
if (!tracking) return undefined
|
|
21
|
-
|
|
20
|
+
return createPostHogErrorReporter({
|
|
21
|
+
key: tracking.projectAPIKey,
|
|
22
22
|
host: tracking.host,
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
properties: {
|
|
24
|
+
cloud_runtime: runtime,
|
|
25
|
+
cloud_deployment: config.deployment,
|
|
26
|
+
cloud_host: new URL(config.publicURL).host
|
|
27
|
+
},
|
|
28
|
+
sourceProperty: 'cloud_source',
|
|
29
|
+
logName: 'Cloud',
|
|
30
|
+
fetch: options.fetch
|
|
25
31
|
})
|
|
26
|
-
// The SDK resolves a capture that PostHog refused and emits the refusal instead.
|
|
27
|
-
client.on('error', (failure: unknown) => {
|
|
28
|
-
console.error('[Cloud] PostHog refused an error report:', failure)
|
|
29
|
-
})
|
|
30
|
-
const deployment = {
|
|
31
|
-
cloud_runtime: runtime,
|
|
32
|
-
cloud_deployment: config.deployment,
|
|
33
|
-
cloud_host: new URL(config.publicURL).host
|
|
34
|
-
}
|
|
35
|
-
return async (error, context) => {
|
|
36
|
-
try {
|
|
37
|
-
// Undefined request fields are left out of the event's JSON.
|
|
38
|
-
await client.captureExceptionImmediate(error, undefined, {
|
|
39
|
-
...deployment,
|
|
40
|
-
cloud_source: context.source,
|
|
41
|
-
http_method: context.method,
|
|
42
|
-
http_route: context.route
|
|
43
|
-
})
|
|
44
|
-
} catch (failure) {
|
|
45
|
-
console.error('[Cloud] Reporting an error failed:', failure)
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
32
|
}
|
|
@@ -3,6 +3,7 @@ import type { ObjectStore } from '#cloud/server/objects'
|
|
|
3
3
|
import {
|
|
4
4
|
AbortMultipartUploadCommand,
|
|
5
5
|
CompleteMultipartUploadCommand,
|
|
6
|
+
CopyObjectCommand,
|
|
6
7
|
CreateMultipartUploadCommand,
|
|
7
8
|
DeleteObjectCommand,
|
|
8
9
|
GetObjectCommand,
|
|
@@ -11,7 +12,7 @@ import {
|
|
|
11
12
|
S3Client,
|
|
12
13
|
UploadPartCommand
|
|
13
14
|
} from '@aws-sdk/client-s3'
|
|
14
|
-
// eslint-disable-next-line open-pencil/no-mixed-case-acronym-identifiers -- Upstream export spelling.
|
|
15
|
+
// eslint-disable-next-line @open-pencil/structure/no-mixed-case-acronym-identifiers -- Upstream export spelling.
|
|
15
16
|
import { getSignedUrl as getSignedURL } from '@aws-sdk/s3-request-presigner'
|
|
16
17
|
|
|
17
18
|
const CHECKSUM_METADATA_KEY = 'openpencil-sha256'
|
|
@@ -70,6 +71,27 @@ export function createS3ObjectStore(config: CloudServerConfig): ObjectStore {
|
|
|
70
71
|
)
|
|
71
72
|
}
|
|
72
73
|
|
|
74
|
+
/**
|
|
75
|
+
* Copies a file under the deleted prefix before it goes, so it outlives the delete until the
|
|
76
|
+
* bucket's lifecycle rule expires it. A file already gone, as when a cleanup pass retries, has
|
|
77
|
+
* nothing to keep.
|
|
78
|
+
*/
|
|
79
|
+
async function keepDeleted(key: string, prefix: string): Promise<void> {
|
|
80
|
+
try {
|
|
81
|
+
await client.send(
|
|
82
|
+
new CopyObjectCommand({
|
|
83
|
+
Bucket: config.s3Bucket,
|
|
84
|
+
Key: `${prefix}${key}`,
|
|
85
|
+
CopySource: `${config.s3Bucket}/${key.split('/').map(encodeURIComponent).join('/')}`,
|
|
86
|
+
MetadataDirective: 'COPY',
|
|
87
|
+
...encryption(config)
|
|
88
|
+
})
|
|
89
|
+
)
|
|
90
|
+
} catch (error) {
|
|
91
|
+
if (!isNotFoundError(error)) throw error
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
73
95
|
return {
|
|
74
96
|
capabilities: {
|
|
75
97
|
nativeSHA256,
|
|
@@ -238,6 +260,7 @@ export function createS3ObjectStore(config: CloudServerConfig): ObjectStore {
|
|
|
238
260
|
},
|
|
239
261
|
|
|
240
262
|
async delete(key) {
|
|
263
|
+
if (config.s3DeletedPrefix) await keepDeleted(key, config.s3DeletedPrefix)
|
|
241
264
|
await client.send(new DeleteObjectCommand({ Bucket: config.s3Bucket, Key: key }))
|
|
242
265
|
},
|
|
243
266
|
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { CLOUD_PORTAL_PAGE_ROOTS } from '#cloud/contract'
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
cloudErrorHandler,
|
|
4
|
+
cloudRequestId,
|
|
5
|
+
withIndexingPolicy,
|
|
6
|
+
type CloudErrorReporter,
|
|
7
|
+
type ForwardedRequestId
|
|
8
|
+
} from '#cloud/server'
|
|
3
9
|
import { Hono } from 'hono'
|
|
4
10
|
|
|
5
11
|
/** A file of the Cloud site (`bun run build:cloud`): the editor and the portal. */
|
|
@@ -16,11 +22,28 @@ export type SiteFiles = (path: string, request: Request) => Promise<SiteFile | n
|
|
|
16
22
|
const PORTAL_PAGE = '/cloud.html'
|
|
17
23
|
const EDITOR_PAGE = '/index.html'
|
|
18
24
|
|
|
25
|
+
/**
|
|
26
|
+
* What the site's pages may run: only the site's own scripts and Turnstile's on the portal, so an
|
|
27
|
+
* injected script tag or one from another host never runs. The pages carry no inline script (the
|
|
28
|
+
* boot splash loads its own file). The editor compiles design JSX and code-tool snippets with
|
|
29
|
+
* `Function` and runs WebAssembly and workers from blobs, which the policy keeps; plugins and AI
|
|
30
|
+
* providers reach other hosts through `fetch`, which it leaves be. Desktop sign-in approval and
|
|
31
|
+
* the editor must never be framed by another site.
|
|
32
|
+
*/
|
|
33
|
+
export const SITE_CONTENT_SECURITY_POLICY = [
|
|
34
|
+
"script-src 'self' 'unsafe-eval' 'wasm-unsafe-eval' blob: https://challenges.cloudflare.com",
|
|
35
|
+
"worker-src 'self' blob:",
|
|
36
|
+
"object-src 'none'",
|
|
37
|
+
"base-uri 'self'",
|
|
38
|
+
"frame-ancestors 'none'"
|
|
39
|
+
].join('; ')
|
|
40
|
+
|
|
19
41
|
const PAGE_HEADERS = {
|
|
20
42
|
'Content-Type': 'text/html; charset=utf-8',
|
|
21
43
|
'Cache-Control': 'no-cache',
|
|
22
|
-
|
|
23
|
-
|
|
44
|
+
'Content-Security-Policy': SITE_CONTENT_SECURITY_POLICY,
|
|
45
|
+
// Browsers take this only over HTTPS; a deployment on plain HTTP is unaffected.
|
|
46
|
+
'Strict-Transport-Security': 'max-age=31536000',
|
|
24
47
|
'Referrer-Policy': 'no-referrer',
|
|
25
48
|
'X-Content-Type-Options': 'nosniff'
|
|
26
49
|
}
|
|
@@ -31,6 +54,8 @@ export type CloudServerOptions = {
|
|
|
31
54
|
indexingPolicy: 'allow' | 'deny'
|
|
32
55
|
/** Reports errors reading the site's files, when the deployment configures error tracking. */
|
|
33
56
|
reportError?: CloudErrorReporter
|
|
57
|
+
/** The request ID the runtime trusts from what stands in front of it; see `cloudRequestId`. */
|
|
58
|
+
forwardedRequestId?: ForwardedRequestId
|
|
34
59
|
}
|
|
35
60
|
|
|
36
61
|
function cacheControl(path: string): string {
|
|
@@ -45,9 +70,11 @@ function cacheControl(path: string): string {
|
|
|
45
70
|
*/
|
|
46
71
|
export function createCloudServer(
|
|
47
72
|
api: { fetch(request: Request): Response | Promise<Response> },
|
|
48
|
-
{ site, indexingPolicy, reportError }: CloudServerOptions
|
|
73
|
+
{ site, indexingPolicy, reportError, forwardedRequestId }: CloudServerOptions
|
|
49
74
|
) {
|
|
50
|
-
const server = new Hono()
|
|
75
|
+
const server = new Hono()
|
|
76
|
+
.onError(cloudErrorHandler(reportError))
|
|
77
|
+
.use(cloudRequestId(forwardedRequestId))
|
|
51
78
|
server.use(async (c, next) => {
|
|
52
79
|
await next()
|
|
53
80
|
c.res = withIndexingPolicy(c.res, c.req.path, indexingPolicy)
|
|
@@ -9,6 +9,11 @@ const ACCOUNT_SECURITY_ERROR_CODES: Record<string, AccountSecurityErrorCode> = {
|
|
|
9
9
|
PASSWORD_TOO_LONG: 'password_too_long',
|
|
10
10
|
FAILED_TO_UNLINK_LAST_ACCOUNT: 'last_authentication_method',
|
|
11
11
|
SESSION_NOT_FRESH: 'session_not_fresh',
|
|
12
|
+
// Better Auth's own word for a session too old to delete the account without its password.
|
|
13
|
+
SESSION_EXPIRED: 'session_not_fresh',
|
|
14
|
+
CREDENTIAL_ACCOUNT_NOT_FOUND: 'session_not_fresh',
|
|
15
|
+
ACCOUNT_SOLE_ADMIN: 'account_sole_admin',
|
|
16
|
+
LAST_ADMIN_REQUIRED: 'last_admin_required',
|
|
12
17
|
CHANGE_EMAIL_DISABLED: 'email_change_unavailable',
|
|
13
18
|
CURRENT_SESSION: 'current_session',
|
|
14
19
|
EMAIL_UNCHANGED: 'email_unchanged'
|