@open-pencil/cloud 0.17.0-beta.1 → 0.17.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/{ai-relay.mjs → services.mjs} +8 -8
- package/dist/bin/services.mjs.map +1 -0
- package/dist/client.d.ts +236 -46
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +649 -400
- package/dist/client.js.map +1 -1
- package/dist/contract.d.ts +2 -2
- package/dist/contract.js +2 -2
- package/dist/contract2.js +680 -443
- package/dist/contract2.js.map +1 -1
- package/dist/email.d.ts +9 -1
- package/dist/email.d.ts.map +1 -1
- package/dist/email.js +99 -58
- package/dist/email.js.map +1 -1
- package/dist/index.d.ts +468 -64
- package/dist/index.d.ts.map +1 -1
- package/dist/listen.mjs +575 -738
- package/dist/listen.mjs.map +1 -1
- package/dist/runtime-node.d.mts +6564 -2934
- package/dist/runtime-node.d.mts.map +1 -1
- package/dist/runtime-node.mjs +6 -5
- package/dist/runtime-node.mjs.map +1 -1
- package/dist/server.d.ts +8931 -4198
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +5260 -3859
- package/dist/server.js.map +1 -1
- package/dist/server.mjs +5686 -3736
- package/dist/server.mjs.map +1 -1
- package/dist/server2.mjs +101 -79
- package/dist/server2.mjs.map +1 -1
- package/package.json +9 -6
- package/src/admin/api/routes.ts +103 -33
- package/src/admin/audit/service.ts +25 -6
- package/src/admin/documents/service.ts +361 -0
- package/src/admin/errors.ts +9 -1
- package/src/admin/index.ts +6 -0
- package/src/admin/operations/service.ts +7 -0
- package/src/admin/reports/service.ts +147 -0
- package/src/admin/schema.ts +6 -0
- package/src/client/api.ts +36 -21
- package/src/client/auth/navigation.ts +6 -5
- package/src/client/index.ts +7 -7
- package/src/client/portal/account.ts +25 -0
- package/src/client/portal/administration.ts +56 -0
- package/src/client/{ai.ts → services.ts} +44 -27
- package/src/contract/account.ts +58 -2
- package/src/contract/admin/index.ts +67 -3
- package/src/contract/collaboration.ts +6 -1
- package/src/contract/discovery.ts +22 -4
- package/src/contract/documents.ts +27 -0
- package/src/contract/entitlements.ts +2 -2
- package/src/contract/index.ts +63 -37
- package/src/contract/relay.ts +6 -0
- package/src/contract/reports.ts +42 -0
- package/src/contract/services.ts +233 -0
- package/src/contract/sharing.ts +3 -9
- package/src/email/render.ts +26 -0
- package/src/email/schema.ts +11 -1
- package/src/email/templates/brand.ts +1 -1
- package/src/runtime/cloudflare/environment.ts +31 -6
- package/src/runtime/cloudflare/index.ts +1 -1
- package/src/runtime/cloudflare/logging.ts +15 -0
- package/src/runtime/cloudflare/maintenance.ts +31 -0
- package/src/runtime/cloudflare/relay/index.ts +176 -0
- package/src/runtime/cloudflare/relay/sockets.ts +112 -0
- package/src/runtime/cloudflare/worker.ts +89 -26
- package/src/runtime/node/application.ts +5 -3
- package/src/runtime/node/bootstrap.ts +20 -4
- package/src/runtime/node/cleanup.ts +16 -3
- package/src/runtime/node/entitlements/main.ts +14 -11
- package/src/runtime/node/index.ts +1 -1
- package/src/runtime/node/migrate.ts +5 -1
- package/src/runtime/node/relay.ts +8 -2
- package/src/runtime/node/request-id.ts +13 -0
- package/src/runtime/node/server.ts +71 -15
- package/src/runtime/node/{ai-relay → services}/main.ts +6 -6
- package/src/runtime/node/{ai-relay → services}/server.ts +18 -18
- package/src/runtime/postgres/database.ts +20 -1
- package/src/runtime/posthog/errors.ts +32 -0
- package/src/runtime/s3/objects.ts +50 -1
- package/src/runtime/site/server.ts +33 -4
- package/src/server/account/actions.ts +5 -0
- package/src/server/account/deletion.ts +302 -0
- package/src/server/account/routes.ts +47 -3
- package/src/server/account/terms.ts +42 -0
- package/src/server/api/router.ts +12 -7
- package/src/server/api/types.ts +9 -0
- package/src/server/app.ts +93 -15
- package/src/server/auth/access-control.ts +63 -0
- package/src/server/auth/adapter.ts +11 -0
- package/src/server/auth/endpoints.ts +5 -1
- package/src/server/auth/factory.ts +30 -3
- package/src/server/auth/index.ts +10 -1
- package/src/server/cleanup/configured.ts +12 -5
- package/src/server/cleanup/documents.ts +165 -71
- package/src/server/cleanup/objects.ts +10 -2
- package/src/server/cleanup/revisions.ts +24 -7
- package/src/server/cleanup/uploads.ts +8 -2
- package/src/server/cleanup/worker.ts +5 -2
- package/src/server/collaboration/index.ts +3 -0
- package/src/server/collaboration/relay/connection.ts +196 -0
- package/src/server/collaboration/relay/documents.ts +13 -10
- package/src/server/collaboration/relay/hub.ts +279 -195
- package/src/server/collaboration/relay/index.ts +3 -1
- package/src/server/collaboration/relay/log.ts +79 -0
- package/src/server/collaboration/relay/persistence.ts +147 -22
- package/src/server/collaboration/relay/resume.ts +28 -0
- package/src/server/collaboration/routes.ts +27 -16
- package/src/server/collaboration/service.ts +27 -31
- package/src/server/config/deployment.ts +35 -4
- package/src/server/config/public.ts +5 -4
- package/src/server/config/schema.ts +104 -16
- package/src/server/db/expressions.ts +15 -1
- package/src/server/db/migrate.ts +34 -114
- package/src/server/db/migrations/001_accounts.ts +79 -0
- package/src/server/db/migrations/002_workspaces.ts +48 -0
- package/src/server/db/migrations/003_storage.ts +44 -0
- package/src/server/db/migrations/004_documents.ts +193 -0
- package/src/server/db/migrations/005_sharing.ts +95 -0
- package/src/server/db/migrations/006_operations.ts +110 -0
- package/src/server/db/migrations/007_trash.ts +33 -0
- package/src/server/db/migrations/008_terms.ts +18 -0
- package/src/server/db/migrations/schema.ts +71 -0
- package/src/server/db/tables.ts +70 -7
- package/src/server/documents/access.ts +66 -55
- package/src/server/documents/errors.ts +7 -0
- package/src/server/documents/images.ts +2 -1
- package/src/server/documents/limits.ts +9 -9
- package/src/server/documents/purge.ts +75 -0
- package/src/server/documents/repository.ts +2 -0
- package/src/server/documents/routes.ts +132 -80
- package/src/server/documents/service.ts +18 -4
- package/src/server/documents/summary.ts +1 -0
- package/src/server/documents/trash.ts +203 -0
- package/src/server/documents/versions/checkpoint.ts +2 -1
- package/src/server/documents/versions/history.ts +11 -7
- package/src/server/documents/versions/routes.ts +36 -25
- package/src/server/documents/versions/service.ts +22 -4
- package/src/server/errors/index.ts +99 -0
- package/src/server/errors/posthog.ts +80 -0
- package/src/server/index.ts +56 -16
- package/src/server/internal/index.ts +3 -3
- package/src/server/internal/routes.ts +7 -15
- package/src/server/internal/schema.ts +1 -1
- package/src/server/internal/service.ts +17 -23
- package/src/server/invitations/budget.ts +66 -0
- package/src/server/invitations/index.ts +1 -0
- package/src/server/logging.ts +76 -0
- package/src/server/objects/types.ts +12 -0
- package/src/server/pictures/routes.ts +4 -2
- package/src/server/pictures/service.ts +2 -1
- package/src/server/policy/database.ts +39 -25
- package/src/server/policy/index.ts +2 -1
- package/src/server/policy/keys.ts +5 -2
- package/src/server/policy/policy.ts +1 -0
- package/src/server/policy/schema.ts +2 -2
- package/src/server/policy/service.ts +12 -9
- package/src/server/policy/static.ts +5 -5
- package/src/server/policy/workspace.ts +41 -0
- package/src/server/quota/reconcile.ts +2 -0
- package/src/server/quota/storage.ts +69 -31
- package/src/server/rate-limit/cleanup.ts +17 -5
- package/src/server/rate-limit/index.ts +4 -1
- package/src/server/rate-limit/middleware.ts +100 -12
- package/src/server/reports/index.ts +8 -0
- package/src/server/reports/routes.ts +37 -0
- package/src/server/reports/service.ts +123 -0
- package/src/server/services/errors.ts +7 -0
- package/src/server/services/index.ts +8 -0
- package/src/server/{ai → services}/keys.ts +11 -11
- package/src/server/services/routes.ts +47 -0
- package/src/server/{ai/service.ts → services/tokens.ts} +23 -20
- package/src/server/sharing/capabilities/service.ts +68 -22
- package/src/server/sharing/document-routes.ts +30 -22
- package/src/server/sharing/grants/service.ts +11 -18
- package/src/server/sharing/http.ts +4 -0
- package/src/server/sharing/invitations/service.ts +21 -11
- package/src/server/sharing/policy.ts +9 -14
- package/src/server/sharing/routes.ts +108 -85
- package/src/server/sharing/service.ts +1 -17
- package/src/server/validation.ts +59 -2
- package/src/server/workspaces/index.ts +1 -0
- package/src/server/workspaces/members.ts +89 -38
- package/src/server/workspaces/repository.ts +2 -1
- package/src/server/workspaces/routes.ts +67 -45
- package/src/{ai-relay → services}/app.ts +48 -44
- package/src/{ai-relay → services}/config.ts +25 -19
- package/src/services/db/index.ts +10 -0
- package/src/services/db/migrate.ts +37 -0
- package/src/{ai-relay/db/migrations/001_usage.ts → services/db/migrations/001_baseline.ts} +18 -7
- package/src/services/db/schema.ts +22 -0
- package/src/services/index.ts +18 -0
- package/src/{ai-relay → services}/metering.ts +2 -2
- package/src/{ai-relay → services}/usage.ts +43 -19
- package/dist/bin/ai-relay.mjs.map +0 -1
- package/src/ai-relay/db/index.ts +0 -10
- package/src/ai-relay/db/migrate.ts +0 -32
- package/src/ai-relay/db/schema.ts +0 -20
- package/src/ai-relay/index.ts +0 -18
- package/src/contract/ai.ts +0 -150
- package/src/runtime/cloudflare/relay.ts +0 -136
- package/src/server/ai/errors.ts +0 -7
- package/src/server/ai/index.ts +0 -4
- package/src/server/ai/routes.ts +0 -43
- package/src/server/auth/workspace-access.ts +0 -23
- package/src/server/db/migrations/001_foundation.ts +0 -164
- package/src/server/db/migrations/002_upload_cleanup_claims.ts +0 -34
- package/src/server/db/migrations/003_document_cleanup_claims.ts +0 -23
- package/src/server/db/migrations/004_document_sharing.ts +0 -82
- package/src/server/db/migrations/005_document_collaboration_epoch.ts +0 -13
- package/src/server/db/migrations/006_invitation_continuation.ts +0 -27
- package/src/server/db/migrations/007_collaboration_state.ts +0 -25
- package/src/server/db/migrations/008_storage_reservations.ts +0 -37
- package/src/server/db/migrations/009_workspace_entitlements.ts +0 -22
- package/src/server/db/migrations/010_upload_finalization.ts +0 -22
- package/src/server/db/migrations/011_upload_finalization_lease.ts +0 -12
- package/src/server/db/migrations/012_transactional_email.ts +0 -48
- package/src/server/db/migrations/013_admin_enrollment.ts +0 -54
- package/src/server/db/migrations/014_enrollment_email_kinds.ts +0 -30
- package/src/server/db/migrations/015_cloud_rate_limit.ts +0 -22
- package/src/server/db/migrations/016_auth_schema.ts +0 -16
- package/src/server/db/migrations/017_authentication_email_kinds.ts +0 -32
- package/src/server/db/migrations/018_authentication_mfa.ts +0 -78
- package/src/server/db/migrations/019_better_auth_account_issuer.ts +0 -42
- package/src/server/db/migrations/020_workspace_invitations.ts +0 -57
- package/src/server/db/migrations/021_document_versions.ts +0 -66
- package/src/server/db/migrations/022_entitlement_external_version.ts +0 -17
- package/src/server/db/migrations/023_document_images.ts +0 -61
- package/src/server/db/migrations/024_account_activity.ts +0 -66
- package/src/server/db/migrations/025_table_names.ts +0 -58
- package/src/server/db/migrations/026_user_picture.ts +0 -30
- package/src/server/db/migrations/027_document_kind.ts +0 -19
- package/src/server/db/migrations/helpers.ts +0 -78
- /package/dist/bin/{ai-relay.d.mts → services.d.mts} +0 -0
|
@@ -1,40 +1,43 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises'
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { createAccountEntitlementRepository, parseStaticEntitlementsTOML } from '#cloud/server'
|
|
4
4
|
import { parse as parseTOML } from 'smol-toml'
|
|
5
5
|
|
|
6
6
|
import { withNodeCloudDatabase } from '../command'
|
|
7
7
|
import { entitlementWriteRefusal } from './refusal'
|
|
8
8
|
|
|
9
9
|
const operation = process.argv[2]
|
|
10
|
-
const
|
|
10
|
+
const accountIndex = process.argv.indexOf('--account')
|
|
11
11
|
const fileIndex = process.argv.indexOf('--file')
|
|
12
12
|
const dryRun = process.argv.includes('--dry-run')
|
|
13
|
-
const
|
|
14
|
-
if (!
|
|
13
|
+
const accountArgument = accountIndex !== -1 ? process.argv[accountIndex + 1] : undefined
|
|
14
|
+
if (!accountArgument) throw new Error('Expected --account <user id>')
|
|
15
15
|
|
|
16
16
|
await withNodeCloudDatabase(async (database, config) => {
|
|
17
|
-
const repository =
|
|
17
|
+
const repository = createAccountEntitlementRepository(database)
|
|
18
18
|
const refusal = entitlementWriteRefusal(config, operation, dryRun)
|
|
19
19
|
if (refusal) throw new Error(refusal)
|
|
20
|
+
// A plan is stored under the account as the user table spells it, which policy reads.
|
|
21
|
+
const userId = await repository.account(accountArgument)
|
|
22
|
+
if (!userId) throw new Error(`No account ${accountArgument}`)
|
|
20
23
|
if (operation === 'get') {
|
|
21
|
-
console.log(JSON.stringify({ entitlement: await repository.get(
|
|
24
|
+
console.log(JSON.stringify({ entitlement: await repository.get(userId) }))
|
|
22
25
|
} else if (operation === 'clear') {
|
|
23
26
|
if (dryRun) {
|
|
24
|
-
console.log(JSON.stringify({ dryRun, cleared: false,
|
|
27
|
+
console.log(JSON.stringify({ dryRun, cleared: false, userId }))
|
|
25
28
|
return
|
|
26
29
|
}
|
|
27
|
-
await repository.clear(
|
|
28
|
-
console.log(JSON.stringify({ cleared: true,
|
|
30
|
+
await repository.clear(userId)
|
|
31
|
+
console.log(JSON.stringify({ cleared: true, userId }))
|
|
29
32
|
} else if (operation === 'set') {
|
|
30
33
|
const file = fileIndex !== -1 ? process.argv[fileIndex + 1] : undefined
|
|
31
34
|
if (!file) throw new Error('Expected --file <path>')
|
|
32
35
|
const values = parseStaticEntitlementsTOML(parseTOML(await readFile(file, 'utf8')))
|
|
33
36
|
if (dryRun) {
|
|
34
|
-
console.log(JSON.stringify({ dryRun,
|
|
37
|
+
console.log(JSON.stringify({ dryRun, userId, values }))
|
|
35
38
|
return
|
|
36
39
|
}
|
|
37
|
-
console.log(JSON.stringify({ entitlement: await repository.set(
|
|
40
|
+
console.log(JSON.stringify({ entitlement: await repository.set(userId, values, 'cli') }))
|
|
38
41
|
} else {
|
|
39
42
|
throw new Error('Expected operation: get, set, or clear')
|
|
40
43
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { startNodeServices, type NodeServicesOptions } from './services/server'
|
|
2
2
|
export { createMigratedNodeCloudDatabase } from './bootstrap'
|
|
3
3
|
export { createNodeCloudApplication, type NodeCloudApplicationOptions } from './application'
|
|
4
4
|
export { loadNodeCloudServerConfig } from './config'
|
|
@@ -4,7 +4,11 @@ import { createBetterAuthAdapter, migrateCloudDatabase } from '#cloud/server'
|
|
|
4
4
|
import { loadNodeCloudServerConfig } from './config'
|
|
5
5
|
|
|
6
6
|
const config = await loadNodeCloudServerConfig(process.env)
|
|
7
|
-
|
|
7
|
+
// A migration may rewrite a large table, which takes as long as it takes.
|
|
8
|
+
const database = createPostgresCloudDatabase({
|
|
9
|
+
connectionString: config.databaseURL,
|
|
10
|
+
statementTimeoutMs: null
|
|
11
|
+
})
|
|
8
12
|
try {
|
|
9
13
|
const auth = createBetterAuthAdapter(config, database)
|
|
10
14
|
await migrateCloudDatabase(database, { run: auth.migrate, schemaVersion: auth.schemaVersion })
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { IncomingMessage, Server } from 'node:http'
|
|
2
2
|
import type { Duplex } from 'node:stream'
|
|
3
3
|
|
|
4
|
-
import { RELAY_PATH } from '#cloud/contract'
|
|
4
|
+
import { RELAY_HEARTBEAT, RELAY_PATH } from '#cloud/contract'
|
|
5
5
|
import type { CollaborationRelay } from '#cloud/server'
|
|
6
6
|
import { WebSocketServer, type RawData } from 'ws'
|
|
7
7
|
|
|
@@ -34,7 +34,13 @@ export function attachCollaborationRelay(
|
|
|
34
34
|
send: (data) => webSocket.send(data),
|
|
35
35
|
close: (code, reason) => webSocket.close(code, reason)
|
|
36
36
|
})
|
|
37
|
-
webSocket.on('message', (data) =>
|
|
37
|
+
webSocket.on('message', (data, isBinary) => {
|
|
38
|
+
if (!isBinary && new TextDecoder().decode(bytes(data)) === RELAY_HEARTBEAT.ping) {
|
|
39
|
+
webSocket.send(RELAY_HEARTBEAT.pong)
|
|
40
|
+
return
|
|
41
|
+
}
|
|
42
|
+
connection.receive(bytes(data))
|
|
43
|
+
})
|
|
38
44
|
webSocket.on('close', () => connection.closed())
|
|
39
45
|
webSocket.on('error', () => webSocket.terminate())
|
|
40
46
|
})
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { ForwardedRequestId } from '#cloud/server'
|
|
2
|
+
|
|
3
|
+
/** What a proxy's `X-Request-Id` may hold to be kept. */
|
|
4
|
+
const PROXY_REQUEST_ID = /^[\w-]{1,128}$/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Names a request by the `X-Request-Id` a proxy in front set, such as nginx's `$request_id`, so
|
|
8
|
+
* the proxy's log and the server's agree.
|
|
9
|
+
*/
|
|
10
|
+
export const proxyRequestId: ForwardedRequestId = (headers) => {
|
|
11
|
+
const id = headers.get('x-request-id')
|
|
12
|
+
return id && PROXY_REQUEST_ID.test(id) ? id : undefined
|
|
13
|
+
}
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { defaultRelayURL } from '#cloud/contract'
|
|
2
|
+
import { createPostgresCloudDatabase } from '#cloud/runtime/postgres/database'
|
|
3
|
+
import { createCloudErrorReporter } from '#cloud/runtime/posthog/errors'
|
|
2
4
|
import { createS3ObjectStore } from '#cloud/runtime/s3/objects'
|
|
3
5
|
import { createCloudServer, type SiteFiles } from '#cloud/runtime/site/server'
|
|
4
6
|
import {
|
|
5
7
|
createCloudApp,
|
|
6
|
-
|
|
8
|
+
createLoggingErrorReporter,
|
|
9
|
+
loadServicesSigningKey,
|
|
7
10
|
createCloudAuthenticationRuntime,
|
|
8
11
|
createCollaborationRelay,
|
|
9
12
|
createCollaborationStateStore,
|
|
@@ -11,14 +14,20 @@ import {
|
|
|
11
14
|
createCleanupServices,
|
|
12
15
|
createConfiguredCloudPolicy,
|
|
13
16
|
relayDocumentLookups,
|
|
17
|
+
reportCloudError,
|
|
14
18
|
startCleanupWorker,
|
|
15
|
-
startTransactionalEmailWorker
|
|
19
|
+
startTransactionalEmailWorker,
|
|
20
|
+
writeJSONLine,
|
|
21
|
+
type CloudErrorReporter,
|
|
22
|
+
type CloudServerConfig
|
|
16
23
|
} from '#cloud/server'
|
|
17
24
|
|
|
18
|
-
import {
|
|
25
|
+
import { migratedNodeCloudDatabase } from './bootstrap'
|
|
26
|
+
import { loadNodeCloudServerConfig } from './config'
|
|
19
27
|
import { createNodeTransactionalEmailRuntime } from './email/runtime'
|
|
20
28
|
import { listen } from './listen'
|
|
21
29
|
import { attachCollaborationRelay } from './relay'
|
|
30
|
+
import { proxyRequestId } from './request-id'
|
|
22
31
|
import { developmentSiteFiles, nodeSiteFiles } from './site'
|
|
23
32
|
|
|
24
33
|
export type NodeCloudServerOptions = {
|
|
@@ -40,11 +49,35 @@ function siteFiles(
|
|
|
40
49
|
return undefined
|
|
41
50
|
}
|
|
42
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Starts the server. Once the config parses, a startup that fails, such as a migration, a signing
|
|
54
|
+
* key, or a port that cannot be bound, is reported before it is thrown; a config that does not
|
|
55
|
+
* parse has no error tracking to report to.
|
|
56
|
+
*/
|
|
43
57
|
export async function startNodeCloudServer(options: NodeCloudServerOptions = {}) {
|
|
44
58
|
const environment = options.environment ?? process.env
|
|
45
|
-
const
|
|
59
|
+
const config = await loadNodeCloudServerConfig(environment)
|
|
60
|
+
const reportError = createLoggingErrorReporter(
|
|
61
|
+
writeJSONLine,
|
|
62
|
+
createCloudErrorReporter(config, 'node')
|
|
63
|
+
)
|
|
64
|
+
try {
|
|
65
|
+
return await startConfiguredServer(config, environment, options, reportError)
|
|
66
|
+
} catch (error) {
|
|
67
|
+
await reportCloudError(reportError, 'Starting the server failed', error, { source: 'setup' })
|
|
68
|
+
throw error
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function startConfiguredServer(
|
|
73
|
+
config: CloudServerConfig,
|
|
74
|
+
environment: Readonly<Record<string, string | undefined>>,
|
|
75
|
+
options: NodeCloudServerOptions,
|
|
76
|
+
reportError: CloudErrorReporter | undefined
|
|
77
|
+
) {
|
|
78
|
+
const { database } = await migratedNodeCloudDatabase(config)
|
|
46
79
|
// Before any worker starts, so a key that cannot sign leaves nothing running.
|
|
47
|
-
const
|
|
80
|
+
const servicesSigningKey = await loadServicesSigningKey(config).catch(async (error: unknown) => {
|
|
48
81
|
await database.destroy()
|
|
49
82
|
throw error
|
|
50
83
|
})
|
|
@@ -56,18 +89,34 @@ export async function startNodeCloudServer(options: NodeCloudServerOptions = {})
|
|
|
56
89
|
} = createNodeTransactionalEmailRuntime(config, database)
|
|
57
90
|
const { auth, enrollment } = createCloudAuthenticationRuntime(config, database, email)
|
|
58
91
|
const policy = createConfiguredCloudPolicy(config, database)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
92
|
+
// Cleanup sweeps may run longer than a request should, so they have a connection of their own
|
|
93
|
+
// without the statement timeout.
|
|
94
|
+
const cleanupDatabase = config.cleanupEnabled
|
|
95
|
+
? createPostgresCloudDatabase({
|
|
96
|
+
connectionString: config.databaseURL,
|
|
97
|
+
statementTimeoutMs: null
|
|
64
98
|
})
|
|
65
99
|
: undefined
|
|
100
|
+
const cleanup = cleanupDatabase
|
|
101
|
+
? startCleanupWorker(
|
|
102
|
+
createCleanupServices(config, cleanupDatabase, objects, policy, reportError),
|
|
103
|
+
{
|
|
104
|
+
...cleanupPassOptions(config),
|
|
105
|
+
intervalMs: config.cleanupIntervalMs,
|
|
106
|
+
onError: (error) =>
|
|
107
|
+
void reportCloudError(reportError, 'Cleanup worker failed', error, {
|
|
108
|
+
source: 'cleanup'
|
|
109
|
+
})
|
|
110
|
+
}
|
|
111
|
+
)
|
|
112
|
+
: undefined
|
|
66
113
|
const relay = createCollaborationRelay({
|
|
67
114
|
authSecret: config.authSecret,
|
|
68
|
-
store: createCollaborationStateStore(database),
|
|
115
|
+
store: createCollaborationStateStore(database, objects),
|
|
69
116
|
maximumMessageBytes: config.technicalLimits.maximumCollaborationMessageBytes,
|
|
70
117
|
maximumConnectionsPerRoom: config.technicalLimits.maximumConnectionsPerRoom,
|
|
118
|
+
onError: (message, error) =>
|
|
119
|
+
void reportCloudError(reportError, message, error, { source: 'relay' }),
|
|
71
120
|
...relayDocumentLookups(database, policy, config.deployment)
|
|
72
121
|
})
|
|
73
122
|
const app = createCloudApp({
|
|
@@ -78,10 +127,13 @@ export async function startNodeCloudServer(options: NodeCloudServerOptions = {})
|
|
|
78
127
|
invitationOutbox,
|
|
79
128
|
transactionalEmail: email,
|
|
80
129
|
enrollment,
|
|
81
|
-
|
|
130
|
+
servicesSigningKey,
|
|
82
131
|
relayURL: config.relayURL ?? defaultRelayURL(config.publicURL),
|
|
83
132
|
onDocumentRestored: (documentId) => relay.startOver(documentId),
|
|
84
|
-
|
|
133
|
+
onDocumentTakenDown: (documentId) => relay.closeDocument(documentId),
|
|
134
|
+
onDocumentVersionsChanged: (documentId) => relay.announceVersions(documentId),
|
|
135
|
+
reportError,
|
|
136
|
+
forwardedRequestId: proxyRequestId
|
|
85
137
|
})
|
|
86
138
|
const emailWorker = emailTransport
|
|
87
139
|
? startTransactionalEmailWorker(email, {
|
|
@@ -89,12 +141,15 @@ export async function startNodeCloudServer(options: NodeCloudServerOptions = {})
|
|
|
89
141
|
intervalMs: config.emailIntervalMs,
|
|
90
142
|
leaseDurationMs: config.emailLeaseDurationMs,
|
|
91
143
|
maximumAttempts: config.emailMaximumAttempts,
|
|
92
|
-
onError: (error) =>
|
|
144
|
+
onError: (error) =>
|
|
145
|
+
void reportCloudError(reportError, 'Email worker failed', error, { source: 'email' })
|
|
93
146
|
})
|
|
94
147
|
: undefined
|
|
95
148
|
const cloudServer = createCloudServer(app, {
|
|
96
149
|
site: siteFiles(environment),
|
|
97
|
-
indexingPolicy: config.indexingPolicy
|
|
150
|
+
indexingPolicy: config.indexingPolicy,
|
|
151
|
+
reportError,
|
|
152
|
+
forwardedRequestId: proxyRequestId
|
|
98
153
|
})
|
|
99
154
|
const server = await listen(
|
|
100
155
|
async (request) => cloudServer.fetch(request),
|
|
@@ -114,6 +169,7 @@ export async function startNodeCloudServer(options: NodeCloudServerOptions = {})
|
|
|
114
169
|
await relayListener.close()
|
|
115
170
|
await server.close()
|
|
116
171
|
await cleanup?.stop()
|
|
172
|
+
await cleanupDatabase?.destroy()
|
|
117
173
|
await emailWorker?.stop()
|
|
118
174
|
await database.destroy()
|
|
119
175
|
}
|
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import dedent from 'dedent'
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import { startNodeServices } from './server'
|
|
5
5
|
|
|
6
6
|
if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
|
7
7
|
console.log(dedent`
|
|
8
|
-
Usage: openpencil-
|
|
8
|
+
Usage: openpencil-services
|
|
9
9
|
|
|
10
10
|
Runs the reference AI service for a self-hosted OpenPencil Cloud. Settings come from the
|
|
11
11
|
environment:
|
|
12
|
-
|
|
12
|
+
OPENPENCIL_SERVICES_CONFIG the services TOML file (required)
|
|
13
13
|
PORT, HOST where to listen (default 8788 on 0.0.0.0)
|
|
14
14
|
plus the secrets the TOML names. See https://openpencil.dev/reference/self-hosting-cloud
|
|
15
15
|
`)
|
|
16
16
|
process.exit(0)
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
const relay = await
|
|
20
|
-
console.log(`OpenPencil
|
|
19
|
+
const relay = await startNodeServices()
|
|
20
|
+
console.log(`OpenPencil services listening on ${relay.url}`)
|
|
21
21
|
|
|
22
22
|
let stopping: Promise<void> | undefined
|
|
23
23
|
|
|
@@ -25,7 +25,7 @@ function shutdown() {
|
|
|
25
25
|
stopping ??= relay.stop().then(
|
|
26
26
|
() => process.exit(0),
|
|
27
27
|
(error: unknown) => {
|
|
28
|
-
console.error('[
|
|
28
|
+
console.error('[Services] Shutdown failed:', error)
|
|
29
29
|
process.exit(1)
|
|
30
30
|
}
|
|
31
31
|
)
|
|
@@ -1,47 +1,47 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises'
|
|
2
2
|
|
|
3
|
+
import { CLOUD_SERVICES_JWKS_PATH } from '#cloud/contract'
|
|
3
4
|
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
type
|
|
10
|
-
} from '#cloud/
|
|
11
|
-
import { CLOUD_AI_JWKS_PATH } from '#cloud/contract'
|
|
5
|
+
ServicesConfigError,
|
|
6
|
+
createServicesApp,
|
|
7
|
+
createServicesDatabase,
|
|
8
|
+
migrateServicesDatabase,
|
|
9
|
+
parseServicesTOML,
|
|
10
|
+
type ServicesEnvironment
|
|
11
|
+
} from '#cloud/services'
|
|
12
12
|
import { createRemoteJWKSet } from 'jose'
|
|
13
13
|
import { PostgresDialect } from 'kysely'
|
|
14
14
|
import { Pool } from 'pg'
|
|
15
15
|
|
|
16
16
|
import { listen } from '../listen'
|
|
17
17
|
|
|
18
|
-
export type
|
|
19
|
-
environment?:
|
|
18
|
+
export type NodeServicesOptions = {
|
|
19
|
+
environment?: ServicesEnvironment
|
|
20
20
|
port?: number
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
export async function
|
|
23
|
+
export async function startNodeServices(options: NodeServicesOptions = {}) {
|
|
24
24
|
const environment = options.environment ?? process.env
|
|
25
|
-
const path = environment.
|
|
25
|
+
const path = environment.OPENPENCIL_SERVICES_CONFIG
|
|
26
26
|
if (!path) {
|
|
27
|
-
throw new
|
|
27
|
+
throw new ServicesConfigError('OPENPENCIL_SERVICES_CONFIG must name a services TOML file')
|
|
28
28
|
}
|
|
29
|
-
const config =
|
|
30
|
-
const database =
|
|
29
|
+
const config = parseServicesTOML(await readFile(path, 'utf8'), environment)
|
|
30
|
+
const database = createServicesDatabase(
|
|
31
31
|
new PostgresDialect({ pool: new Pool({ connectionString: config.databaseURL }) })
|
|
32
32
|
)
|
|
33
33
|
try {
|
|
34
|
-
await
|
|
34
|
+
await migrateServicesDatabase(database)
|
|
35
35
|
} catch (error) {
|
|
36
36
|
await database.destroy()
|
|
37
37
|
throw error
|
|
38
38
|
}
|
|
39
39
|
// Metering outlives its answer; shutdown waits for it so no request goes uncharged.
|
|
40
40
|
const metering = new Set<Promise<unknown>>()
|
|
41
|
-
const app =
|
|
41
|
+
const app = createServicesApp({
|
|
42
42
|
config,
|
|
43
43
|
database,
|
|
44
|
-
keys: createRemoteJWKSet(new URL(
|
|
44
|
+
keys: createRemoteJWKSet(new URL(CLOUD_SERVICES_JWKS_PATH, config.cloudURL)),
|
|
45
45
|
waitUntil: (promise) => {
|
|
46
46
|
metering.add(promise)
|
|
47
47
|
void promise.finally(() => metering.delete(promise))
|
|
@@ -2,16 +2,35 @@ import { createCloudDatabase } from '#cloud/server'
|
|
|
2
2
|
import { PostgresDialect } from 'kysely'
|
|
3
3
|
import { Pool, type PoolConfig } from 'pg'
|
|
4
4
|
|
|
5
|
+
/**
|
|
6
|
+
* How long a query serving a request may run before PostgreSQL cancels it. Rate limits and
|
|
7
|
+
* sessions live in PostgreSQL, so one stuck query would otherwise hold its connection, and every
|
|
8
|
+
* request waiting for one, indefinitely.
|
|
9
|
+
*/
|
|
10
|
+
export const POSTGRES_STATEMENT_TIMEOUT_MS = 30_000
|
|
11
|
+
|
|
5
12
|
export type PostgresCloudDatabaseOptions = {
|
|
6
13
|
connectionString: string
|
|
7
|
-
pool?: Omit<PoolConfig, 'connectionString'>
|
|
14
|
+
pool?: Omit<PoolConfig, 'connectionString' | 'statement_timeout' | 'query_timeout'>
|
|
15
|
+
/**
|
|
16
|
+
* How long one statement may run, `POSTGRES_STATEMENT_TIMEOUT_MS` unless set; null for no limit
|
|
17
|
+
* of this server's own, as migrations need and as Hyperdrive takes, which keeps no session
|
|
18
|
+
* settings and ends queries past its own limit. PostgreSQL cancels a statement past it, so its
|
|
19
|
+
* connection is free again for the next query.
|
|
20
|
+
*/
|
|
21
|
+
statementTimeoutMs?: number | null
|
|
8
22
|
}
|
|
9
23
|
|
|
10
24
|
export function createPostgresCloudDatabase(options: PostgresCloudDatabaseOptions) {
|
|
25
|
+
const statementTimeoutMs =
|
|
26
|
+
options.statementTimeoutMs === undefined
|
|
27
|
+
? POSTGRES_STATEMENT_TIMEOUT_MS
|
|
28
|
+
: options.statementTimeoutMs
|
|
11
29
|
return createCloudDatabase({
|
|
12
30
|
dialect: new PostgresDialect({
|
|
13
31
|
pool: new Pool({
|
|
14
32
|
...options.pool,
|
|
33
|
+
...(statementTimeoutMs === null ? {} : { statement_timeout: statementTimeoutMs }),
|
|
15
34
|
connectionString: options.connectionString
|
|
16
35
|
})
|
|
17
36
|
})
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { CloudServerConfig } from '#cloud/server'
|
|
2
|
+
import { createPostHogErrorReporter, type CloudErrorReporter } from '#cloud/server/errors'
|
|
3
|
+
import type { PostHogOptions } from 'posthog-node'
|
|
4
|
+
|
|
5
|
+
/** Which runtime serves the deployment, as its error reports name it. */
|
|
6
|
+
export type CloudRuntimeName = 'node' | 'cloudflare'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Reports server errors to the PostHog project the deployment's `[error_tracking]` names, or
|
|
10
|
+
* returns nothing when it names none. Reports carry the runtime, the deployment, and where the
|
|
11
|
+
* error was caught, never who made the request or what it carried.
|
|
12
|
+
*/
|
|
13
|
+
export function createCloudErrorReporter(
|
|
14
|
+
config: CloudServerConfig,
|
|
15
|
+
runtime: CloudRuntimeName,
|
|
16
|
+
options: Pick<PostHogOptions, 'fetch'> = {}
|
|
17
|
+
): CloudErrorReporter | undefined {
|
|
18
|
+
const tracking = config.errorTracking
|
|
19
|
+
if (!tracking) return undefined
|
|
20
|
+
return createPostHogErrorReporter({
|
|
21
|
+
key: tracking.projectAPIKey,
|
|
22
|
+
host: tracking.host,
|
|
23
|
+
properties: {
|
|
24
|
+
cloud_runtime: runtime,
|
|
25
|
+
cloud_deployment: config.deployment,
|
|
26
|
+
cloud_host: new URL(config.publicURL).host
|
|
27
|
+
},
|
|
28
|
+
sourceProperty: 'cloud_source',
|
|
29
|
+
logName: 'Cloud',
|
|
30
|
+
fetch: options.fetch
|
|
31
|
+
})
|
|
32
|
+
}
|
|
@@ -3,6 +3,7 @@ import type { ObjectStore } from '#cloud/server/objects'
|
|
|
3
3
|
import {
|
|
4
4
|
AbortMultipartUploadCommand,
|
|
5
5
|
CompleteMultipartUploadCommand,
|
|
6
|
+
CopyObjectCommand,
|
|
6
7
|
CreateMultipartUploadCommand,
|
|
7
8
|
DeleteObjectCommand,
|
|
8
9
|
GetObjectCommand,
|
|
@@ -11,7 +12,7 @@ import {
|
|
|
11
12
|
S3Client,
|
|
12
13
|
UploadPartCommand
|
|
13
14
|
} from '@aws-sdk/client-s3'
|
|
14
|
-
// eslint-disable-next-line open-pencil/no-mixed-case-acronym-identifiers -- Upstream export spelling.
|
|
15
|
+
// eslint-disable-next-line @open-pencil/structure/no-mixed-case-acronym-identifiers -- Upstream export spelling.
|
|
15
16
|
import { getSignedUrl as getSignedURL } from '@aws-sdk/s3-request-presigner'
|
|
16
17
|
|
|
17
18
|
const CHECKSUM_METADATA_KEY = 'openpencil-sha256'
|
|
@@ -70,6 +71,27 @@ export function createS3ObjectStore(config: CloudServerConfig): ObjectStore {
|
|
|
70
71
|
)
|
|
71
72
|
}
|
|
72
73
|
|
|
74
|
+
/**
|
|
75
|
+
* Copies a file under the deleted prefix before it goes, so it outlives the delete until the
|
|
76
|
+
* bucket's lifecycle rule expires it. A file already gone, as when a cleanup pass retries, has
|
|
77
|
+
* nothing to keep.
|
|
78
|
+
*/
|
|
79
|
+
async function keepDeleted(key: string, prefix: string): Promise<void> {
|
|
80
|
+
try {
|
|
81
|
+
await client.send(
|
|
82
|
+
new CopyObjectCommand({
|
|
83
|
+
Bucket: config.s3Bucket,
|
|
84
|
+
Key: `${prefix}${key}`,
|
|
85
|
+
CopySource: `${config.s3Bucket}/${key.split('/').map(encodeURIComponent).join('/')}`,
|
|
86
|
+
MetadataDirective: 'COPY',
|
|
87
|
+
...encryption(config)
|
|
88
|
+
})
|
|
89
|
+
)
|
|
90
|
+
} catch (error) {
|
|
91
|
+
if (!isNotFoundError(error)) throw error
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
73
95
|
return {
|
|
74
96
|
capabilities: {
|
|
75
97
|
nativeSHA256,
|
|
@@ -238,7 +260,34 @@ export function createS3ObjectStore(config: CloudServerConfig): ObjectStore {
|
|
|
238
260
|
},
|
|
239
261
|
|
|
240
262
|
async delete(key) {
|
|
263
|
+
if (config.s3DeletedPrefix) await keepDeleted(key, config.s3DeletedPrefix)
|
|
241
264
|
await client.send(new DeleteObjectCommand({ Bucket: config.s3Bucket, Key: key }))
|
|
265
|
+
},
|
|
266
|
+
|
|
267
|
+
async write(input) {
|
|
268
|
+
await client.send(
|
|
269
|
+
new PutObjectCommand({
|
|
270
|
+
Bucket: config.s3Bucket,
|
|
271
|
+
Key: input.key,
|
|
272
|
+
Body: input.bytes,
|
|
273
|
+
ContentType: input.contentType,
|
|
274
|
+
ChecksumSHA256: nativeSHA256 ? input.checksum : undefined,
|
|
275
|
+
Metadata: { [CHECKSUM_METADATA_KEY]: input.checksum },
|
|
276
|
+
...encryption(config)
|
|
277
|
+
})
|
|
278
|
+
)
|
|
279
|
+
},
|
|
280
|
+
|
|
281
|
+
async read(key) {
|
|
282
|
+
try {
|
|
283
|
+
const object = await client.send(
|
|
284
|
+
new GetObjectCommand({ Bucket: config.s3Bucket, Key: key })
|
|
285
|
+
)
|
|
286
|
+
return object.Body ? await object.Body.transformToByteArray() : null
|
|
287
|
+
} catch (error) {
|
|
288
|
+
if (isNotFoundError(error)) return null
|
|
289
|
+
throw error
|
|
290
|
+
}
|
|
242
291
|
}
|
|
243
292
|
}
|
|
244
293
|
}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { CLOUD_PORTAL_PAGE_ROOTS } from '#cloud/contract'
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
cloudErrorHandler,
|
|
4
|
+
cloudRequestId,
|
|
5
|
+
withIndexingPolicy,
|
|
6
|
+
type CloudErrorReporter,
|
|
7
|
+
type ForwardedRequestId
|
|
8
|
+
} from '#cloud/server'
|
|
3
9
|
import { Hono } from 'hono'
|
|
4
10
|
|
|
5
11
|
/** A file of the Cloud site (`bun run build:cloud`): the editor and the portal. */
|
|
@@ -16,11 +22,28 @@ export type SiteFiles = (path: string, request: Request) => Promise<SiteFile | n
|
|
|
16
22
|
const PORTAL_PAGE = '/cloud.html'
|
|
17
23
|
const EDITOR_PAGE = '/index.html'
|
|
18
24
|
|
|
25
|
+
/**
|
|
26
|
+
* What the site's pages may run: only the site's own scripts and Turnstile's on the portal, so an
|
|
27
|
+
* injected script tag or one from another host never runs. The pages carry no inline script (the
|
|
28
|
+
* boot splash loads its own file). The editor compiles design JSX and code-tool snippets with
|
|
29
|
+
* `Function` and runs WebAssembly and workers from blobs, which the policy keeps; plugins and AI
|
|
30
|
+
* providers reach other hosts through `fetch`, which it leaves be. Desktop sign-in approval and
|
|
31
|
+
* the editor must never be framed by another site.
|
|
32
|
+
*/
|
|
33
|
+
export const SITE_CONTENT_SECURITY_POLICY = [
|
|
34
|
+
"script-src 'self' 'unsafe-eval' 'wasm-unsafe-eval' blob: https://challenges.cloudflare.com",
|
|
35
|
+
"worker-src 'self' blob:",
|
|
36
|
+
"object-src 'none'",
|
|
37
|
+
"base-uri 'self'",
|
|
38
|
+
"frame-ancestors 'none'"
|
|
39
|
+
].join('; ')
|
|
40
|
+
|
|
19
41
|
const PAGE_HEADERS = {
|
|
20
42
|
'Content-Type': 'text/html; charset=utf-8',
|
|
21
43
|
'Cache-Control': 'no-cache',
|
|
22
|
-
|
|
23
|
-
|
|
44
|
+
'Content-Security-Policy': SITE_CONTENT_SECURITY_POLICY,
|
|
45
|
+
// Browsers take this only over HTTPS; a deployment on plain HTTP is unaffected.
|
|
46
|
+
'Strict-Transport-Security': 'max-age=31536000',
|
|
24
47
|
'Referrer-Policy': 'no-referrer',
|
|
25
48
|
'X-Content-Type-Options': 'nosniff'
|
|
26
49
|
}
|
|
@@ -29,6 +52,10 @@ export type CloudServerOptions = {
|
|
|
29
52
|
/** The built site; without it, the server answers only its API. */
|
|
30
53
|
site?: SiteFiles
|
|
31
54
|
indexingPolicy: 'allow' | 'deny'
|
|
55
|
+
/** Reports errors reading the site's files, when the deployment configures error tracking. */
|
|
56
|
+
reportError?: CloudErrorReporter
|
|
57
|
+
/** The request ID the runtime trusts from what stands in front of it; see `cloudRequestId`. */
|
|
58
|
+
forwardedRequestId?: ForwardedRequestId
|
|
32
59
|
}
|
|
33
60
|
|
|
34
61
|
function cacheControl(path: string): string {
|
|
@@ -43,9 +70,11 @@ function cacheControl(path: string): string {
|
|
|
43
70
|
*/
|
|
44
71
|
export function createCloudServer(
|
|
45
72
|
api: { fetch(request: Request): Response | Promise<Response> },
|
|
46
|
-
{ site, indexingPolicy }: CloudServerOptions
|
|
73
|
+
{ site, indexingPolicy, reportError, forwardedRequestId }: CloudServerOptions
|
|
47
74
|
) {
|
|
48
75
|
const server = new Hono()
|
|
76
|
+
.onError(cloudErrorHandler(reportError))
|
|
77
|
+
.use(cloudRequestId(forwardedRequestId))
|
|
49
78
|
server.use(async (c, next) => {
|
|
50
79
|
await next()
|
|
51
80
|
c.res = withIndexingPolicy(c.res, c.req.path, indexingPolicy)
|
|
@@ -9,6 +9,11 @@ const ACCOUNT_SECURITY_ERROR_CODES: Record<string, AccountSecurityErrorCode> = {
|
|
|
9
9
|
PASSWORD_TOO_LONG: 'password_too_long',
|
|
10
10
|
FAILED_TO_UNLINK_LAST_ACCOUNT: 'last_authentication_method',
|
|
11
11
|
SESSION_NOT_FRESH: 'session_not_fresh',
|
|
12
|
+
// Better Auth's own word for a session too old to delete the account without its password.
|
|
13
|
+
SESSION_EXPIRED: 'session_not_fresh',
|
|
14
|
+
CREDENTIAL_ACCOUNT_NOT_FOUND: 'session_not_fresh',
|
|
15
|
+
ACCOUNT_SOLE_ADMIN: 'account_sole_admin',
|
|
16
|
+
LAST_ADMIN_REQUIRED: 'last_admin_required',
|
|
12
17
|
CHANGE_EMAIL_DISABLED: 'email_change_unavailable',
|
|
13
18
|
CURRENT_SESSION: 'current_session',
|
|
14
19
|
EMAIL_UNCHANGED: 'email_unchanged'
|