@open-pencil/cloud 0.17.0-beta.5 → 0.17.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +560 -93
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +1101 -186
- package/dist/client.js.map +1 -1
- package/dist/contract.d.ts +2 -2
- package/dist/contract.js +2 -2
- package/dist/contract2.js +390 -44
- package/dist/contract2.js.map +1 -1
- package/dist/email.d.ts +366 -3
- package/dist/email.d.ts.map +1 -1
- package/dist/email.js +1403 -556
- package/dist/email.js.map +1 -1
- package/dist/errors.js +16 -3
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +878 -37
- package/dist/index.d.ts.map +1 -1
- package/dist/index2.d.ts +12 -2
- package/dist/index2.d.ts.map +1 -1
- package/dist/listen.mjs +589 -128
- package/dist/listen.mjs.map +1 -1
- package/dist/runtime-cloudflare.d.ts +4 -0
- package/dist/runtime-cloudflare.d.ts.map +1 -1
- package/dist/runtime-cloudflare.js +8689 -6324
- package/dist/runtime-cloudflare.js.map +1 -1
- package/dist/runtime-node.d.mts +10098 -5401
- package/dist/runtime-node.d.mts.map +1 -1
- package/dist/server.d.ts +14078 -7909
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +11723 -9412
- package/dist/server.js.map +1 -1
- package/dist/server.mjs +10658 -8649
- package/dist/server.mjs.map +1 -1
- package/dist/server2.mjs +79 -57
- package/dist/server2.mjs.map +1 -1
- package/package.json +13 -4
- package/src/admin/api/routes.ts +263 -190
- package/src/admin/audit/service.ts +9 -1
- package/src/admin/bootstrap.ts +86 -0
- package/src/admin/documents/service.ts +119 -4
- package/src/admin/email/service.ts +2 -1
- package/src/admin/enrollment/email.ts +17 -3
- package/src/admin/enrollment/service.ts +71 -10
- package/src/admin/errors.ts +5 -0
- package/src/admin/index.ts +1 -0
- package/src/admin/reports/service.ts +9 -3
- package/src/admin/schema.ts +6 -5
- package/src/admin/users/service.ts +173 -16
- package/src/client/api.ts +29 -2
- package/src/client/comments.ts +205 -0
- package/src/client/device-authorization.ts +4 -20
- package/src/client/discovery.ts +4 -2
- package/src/client/files/archive.ts +60 -0
- package/src/client/files/checksum.ts +7 -0
- package/src/client/files/download.ts +122 -0
- package/src/client/files/images.ts +89 -0
- package/src/client/files/index.ts +15 -0
- package/src/client/files/objects.ts +173 -0
- package/src/client/files/save.ts +79 -0
- package/src/client/index.ts +3 -6
- package/src/client/portal/account.ts +20 -2
- package/src/client/portal/administration.ts +52 -5
- package/src/client/portal/request.ts +4 -2
- package/src/client/tour.ts +44 -0
- package/src/client/versions.ts +7 -1
- package/src/contract/account.ts +61 -11
- package/src/contract/admin/index.ts +48 -3
- package/src/contract/billing.ts +73 -4
- package/src/contract/comments.ts +223 -0
- package/src/contract/discovery.ts +47 -19
- package/src/contract/index.ts +10 -0
- package/src/contract/links.ts +5 -0
- package/src/contract/relay.ts +3 -1
- package/src/contract/services.ts +44 -4
- package/src/contract/workspaces.ts +5 -0
- package/src/email/locales/ar.json +28 -0
- package/src/email/locales/de.json +28 -0
- package/src/email/locales/es.json +28 -0
- package/src/email/locales/fr.json +28 -0
- package/src/email/locales/it.json +28 -0
- package/src/email/locales/ja.json +28 -0
- package/src/email/locales/pl.json +28 -0
- package/src/email/locales/pt-br.json +28 -0
- package/src/email/locales/ru.json +28 -0
- package/src/email/locales/zh-cn.json +28 -0
- package/src/email/messages.ts +35 -0
- package/src/email/render.ts +18 -0
- package/src/email/samples.ts +28 -1
- package/src/email/schema.ts +29 -3
- package/src/email/templates/BrandEmail.vue +3 -1
- package/src/email/templates/CommentNotification.vue +39 -0
- package/src/email/templates/EnrollmentInvitation.vue +53 -0
- package/src/email/templates/TwoStepChanged.vue +17 -2
- package/src/email/templates/brand.ts +6 -0
- package/src/errors/posthog.ts +28 -4
- package/src/runtime/cloudflare/email.ts +2 -0
- package/src/runtime/cloudflare/relay/index.ts +21 -6
- package/src/runtime/cloudflare/worker.ts +6 -0
- package/src/runtime/cloudflare/xml.ts +12 -0
- package/src/runtime/node/admin.ts +51 -47
- package/src/runtime/node/email/runtime.ts +1 -0
- package/src/runtime/node/email/transport.ts +1 -0
- package/src/runtime/node/server.ts +8 -2
- package/src/runtime/posthog/errors.ts +3 -1
- package/src/server/account/actions.ts +3 -1
- package/src/server/account/deletion.ts +38 -4
- package/src/server/account/routes.ts +67 -46
- package/src/server/account/tokens.ts +3 -3
- package/src/server/api/router.ts +22 -7
- package/src/server/app/checks.ts +6 -1
- package/src/server/app/cors.ts +44 -0
- package/src/server/app/index.ts +76 -76
- package/src/server/app/routes.ts +14 -5
- package/src/server/app/services.ts +49 -15
- package/src/server/auth/access-control.ts +13 -8
- package/src/server/auth/account.ts +13 -12
- package/src/server/auth/activity.ts +7 -2
- package/src/server/auth/adapter.ts +16 -3
- package/src/server/auth/email.ts +29 -17
- package/src/server/auth/endpoints.ts +4 -0
- package/src/server/auth/factory.ts +83 -64
- package/src/server/auth/index.ts +1 -1
- package/src/server/auth/methods.ts +162 -0
- package/src/server/auth/mfa-assurance.ts +147 -10
- package/src/server/auth/rate-limit.ts +31 -0
- package/src/server/auth/runtime.ts +19 -5
- package/src/server/auth/social-providers.ts +70 -24
- package/src/server/billing/index.ts +2 -0
- package/src/server/billing/notify.ts +55 -0
- package/src/server/billing/seats.ts +17 -0
- package/src/server/billing/tokens.ts +1 -1
- package/src/server/cleanup/documents.ts +5 -2
- package/src/server/cleanup/revisions.ts +6 -3
- package/src/server/cleanup/uploads.ts +2 -1
- package/src/server/collaboration/relay/connection.ts +7 -2
- package/src/server/collaboration/relay/hub.ts +55 -24
- package/src/server/collaboration/routes.ts +8 -5
- package/src/server/comments/index.ts +9 -0
- package/src/server/comments/notify.ts +277 -0
- package/src/server/comments/routes.ts +352 -0
- package/src/server/comments/rows.ts +249 -0
- package/src/server/comments/service.ts +570 -0
- package/src/server/community/routes.ts +6 -6
- package/src/server/config/deployment.ts +19 -6
- package/src/server/config/index.ts +1 -1
- package/src/server/config/public.ts +19 -6
- package/src/server/config/schema.ts +14 -2
- package/src/server/config/toml/schema.ts +27 -4
- package/src/server/db/index.ts +2 -1
- package/src/server/db/migrate.ts +21 -1
- package/src/server/db/migrations/015_comments.ts +57 -0
- package/src/server/db/migrations/016_comment_notifications.ts +76 -0
- package/src/server/db/migrations/017_invitation_emails.ts +19 -0
- package/src/server/db/migrations/018_enrollment_source.ts +17 -0
- package/src/server/db/tables.ts +92 -2
- package/src/server/documents/access.ts +17 -0
- package/src/server/documents/images.ts +6 -1
- package/src/server/documents/routes.ts +12 -12
- package/src/server/documents/service.ts +19 -5
- package/src/server/documents/summary.ts +15 -0
- package/src/server/documents/versions/routes.ts +7 -8
- package/src/server/documents/versions/service.ts +5 -2
- package/src/server/email/service.ts +8 -2
- package/src/server/email/types.ts +1 -0
- package/src/server/errors/index.ts +48 -3
- package/src/server/index.ts +15 -1
- package/src/server/internal/routes.ts +8 -2
- package/src/server/internal/service.ts +10 -0
- package/src/server/invitations/budget.ts +3 -1
- package/src/server/invitations/index.ts +1 -0
- package/src/server/invitations/links.ts +6 -0
- package/src/server/pictures/service.ts +3 -1
- package/src/server/policy/workspace.ts +5 -2
- package/src/server/reports/routes.ts +2 -2
- package/src/server/reports/service.ts +3 -1
- package/src/server/services/routes.ts +2 -2
- package/src/server/sharing/capabilities/service.ts +1 -1
- package/src/server/sharing/document-routes.ts +3 -3
- package/src/server/sharing/invitations/service.ts +3 -4
- package/src/server/sharing/routes.ts +15 -15
- package/src/server/validation.ts +33 -25
- package/src/server/workspaces/members.ts +50 -12
- package/src/server/workspaces/repository.ts +23 -12
- package/src/server/workspaces/routes.ts +8 -8
- package/src/services/app.ts +30 -19
- package/src/services/config.ts +47 -8
- package/src/services/metering.ts +15 -27
- package/src/client/billing.ts +0 -80
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { CLOUD_IMAGE_NAME_PATTERN } from '#cloud/contract'
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
readStreamBytes,
|
|
5
|
+
readZipDirectory,
|
|
6
|
+
zipEntryContents,
|
|
7
|
+
zipStream,
|
|
8
|
+
type ZipEntry,
|
|
9
|
+
type ZipStreamEntry
|
|
10
|
+
} from '@open-pencil/fig'
|
|
11
|
+
|
|
12
|
+
/** Where a `.fig` archive keeps its images, each under the name its paints refer to. */
|
|
13
|
+
const IMAGE_ENTRY_PREFIX = 'images/'
|
|
14
|
+
|
|
15
|
+
/** An image a `.fig` archive holds, as Cloud stores it apart from the file. */
|
|
16
|
+
export type CloudDocumentImage = { name: string; bytes: Uint8Array }
|
|
17
|
+
|
|
18
|
+
function archiveEntry(bytes: Uint8Array, entry: ZipEntry): ZipStreamEntry {
|
|
19
|
+
return { name: entry.name, data: zipEntryContents(bytes, entry), deflate: entry.method !== 0 }
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The archive without its images, and the images, as views of `bytes`. A file that is not a zip
|
|
24
|
+
* archive, or holds no images, comes back whole; an image under a name Cloud does not store
|
|
25
|
+
* apart stays inside.
|
|
26
|
+
*/
|
|
27
|
+
export async function splitCloudDocument(
|
|
28
|
+
bytes: Uint8Array
|
|
29
|
+
): Promise<{ lean: Uint8Array; images: CloudDocumentImage[] }> {
|
|
30
|
+
const entries = readZipDirectory(bytes)
|
|
31
|
+
if (!entries) return { lean: bytes, images: [] }
|
|
32
|
+
const images: CloudDocumentImage[] = []
|
|
33
|
+
const rest: ZipStreamEntry[] = []
|
|
34
|
+
for (const entry of entries) {
|
|
35
|
+
const name = entry.name.startsWith(IMAGE_ENTRY_PREFIX)
|
|
36
|
+
? entry.name.slice(IMAGE_ENTRY_PREFIX.length)
|
|
37
|
+
: null
|
|
38
|
+
// An empty image, or one under a name Cloud does not take, stays inside the file.
|
|
39
|
+
if (name && CLOUD_IMAGE_NAME_PATTERN.test(name) && entry.size > 0) {
|
|
40
|
+
images.push({ name, bytes: zipEntryContents(bytes, entry) })
|
|
41
|
+
} else rest.push(archiveEntry(bytes, entry))
|
|
42
|
+
}
|
|
43
|
+
if (images.length === 0) return { lean: bytes, images }
|
|
44
|
+
return { lean: await readStreamBytes(zipStream(rest)), images }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The archive with its images put back under their names; the lean archive when there are none. */
|
|
48
|
+
export async function assembleCloudDocument(
|
|
49
|
+
lean: Uint8Array,
|
|
50
|
+
images: CloudDocumentImage[]
|
|
51
|
+
): Promise<Uint8Array> {
|
|
52
|
+
const entries = images.length === 0 ? null : readZipDirectory(lean)
|
|
53
|
+
if (!entries) return lean
|
|
54
|
+
return readStreamBytes(
|
|
55
|
+
zipStream([
|
|
56
|
+
...entries.map((entry) => archiveEntry(lean, entry)),
|
|
57
|
+
...images.map((image) => ({ name: `${IMAGE_ENTRY_PREFIX}${image.name}`, data: image.bytes }))
|
|
58
|
+
])
|
|
59
|
+
)
|
|
60
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { sha256 } from '@noble/hashes/sha2.js'
|
|
2
|
+
import { fromUint8Array } from 'js-base64'
|
|
3
|
+
|
|
4
|
+
/** The SHA-256 of `bytes`, in Base64, as Cloud checks an upload against it. */
|
|
5
|
+
export async function cloudChecksum(bytes: Uint8Array): Promise<string> {
|
|
6
|
+
return fromUint8Array(sha256(bytes))
|
|
7
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import type { DocumentDownload, DocumentImageLink } from '#cloud/contract'
|
|
2
|
+
import { limitAsync, sumBy } from 'es-toolkit'
|
|
3
|
+
|
|
4
|
+
import { assembleCloudDocument } from './archive'
|
|
5
|
+
import { cloudChecksum } from './checksum'
|
|
6
|
+
import type { CloudObjectFetch as ObjectFetch, CloudTransferProgress } from './objects'
|
|
7
|
+
|
|
8
|
+
const IMAGE_DOWNLOAD_CONCURRENCY = 6
|
|
9
|
+
|
|
10
|
+
/** An object's bytes from the store, checked against the size and checksum the server recorded. */
|
|
11
|
+
async function fetchVerified(
|
|
12
|
+
target: DocumentDownload['download'],
|
|
13
|
+
expected: { byteSize: number; checksum: string },
|
|
14
|
+
objectFetch: ObjectFetch,
|
|
15
|
+
signal?: AbortSignal
|
|
16
|
+
): Promise<Uint8Array> {
|
|
17
|
+
const response = await objectFetch(target.url, {
|
|
18
|
+
method: target.method,
|
|
19
|
+
headers: target.headers,
|
|
20
|
+
signal
|
|
21
|
+
})
|
|
22
|
+
if (!response.ok) throw new Error(`Cloud document download failed with HTTP ${response.status}`)
|
|
23
|
+
const bytes = new Uint8Array(await response.arrayBuffer())
|
|
24
|
+
if (
|
|
25
|
+
bytes.byteLength !== expected.byteSize ||
|
|
26
|
+
(await cloudChecksum(bytes)) !== expected.checksum
|
|
27
|
+
) {
|
|
28
|
+
throw new Error('The downloaded Cloud document does not match what the server stored')
|
|
29
|
+
}
|
|
30
|
+
return bytes
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** An image a live room names, from where the server linked it, checked against its record. */
|
|
34
|
+
export function downloadCloudImage(
|
|
35
|
+
link: DocumentImageLink,
|
|
36
|
+
objectFetch: ObjectFetch,
|
|
37
|
+
signal?: AbortSignal
|
|
38
|
+
): Promise<Uint8Array> {
|
|
39
|
+
return fetchVerified(link.download, link, objectFetch, signal)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A revision as the editor opens it: its stored file with every image put back under its name. */
|
|
43
|
+
export async function downloadCloudDocument(
|
|
44
|
+
download: DocumentDownload,
|
|
45
|
+
objectFetch: ObjectFetch,
|
|
46
|
+
options: { signal?: AbortSignal; onProgress?: (progress: CloudTransferProgress) => void } = {}
|
|
47
|
+
): Promise<Uint8Array> {
|
|
48
|
+
const totalBytes = download.byteSize + sumBy(download.images, (image) => image.byteSize)
|
|
49
|
+
let transferredBytes = 0
|
|
50
|
+
const received = (bytes: Uint8Array) => {
|
|
51
|
+
transferredBytes += bytes.byteLength
|
|
52
|
+
options.onProgress?.({ transferredBytes, totalBytes })
|
|
53
|
+
return bytes
|
|
54
|
+
}
|
|
55
|
+
const fetchImage = limitAsync(
|
|
56
|
+
async (image: DocumentDownload['images'][number]) => ({
|
|
57
|
+
name: image.name,
|
|
58
|
+
bytes: received(await fetchVerified(image.download, image, objectFetch, options.signal))
|
|
59
|
+
}),
|
|
60
|
+
IMAGE_DOWNLOAD_CONCURRENCY
|
|
61
|
+
)
|
|
62
|
+
const [lean, ...images] = await Promise.all([
|
|
63
|
+
fetchVerified(download.download, download, objectFetch, options.signal).then(received),
|
|
64
|
+
...download.images.map(fetchImage)
|
|
65
|
+
])
|
|
66
|
+
return assembleCloudDocument(lean, images)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Reads exactly `length` bytes of a response, stopping a body that runs past them rather than
|
|
71
|
+
* receiving the rest.
|
|
72
|
+
*/
|
|
73
|
+
async function readExactly(response: Response, length: number): Promise<Uint8Array> {
|
|
74
|
+
const bytes = new Uint8Array(length)
|
|
75
|
+
let received = 0
|
|
76
|
+
const reader = response.body?.getReader()
|
|
77
|
+
if (reader) {
|
|
78
|
+
for (;;) {
|
|
79
|
+
const { done, value } = await reader.read()
|
|
80
|
+
if (done) break
|
|
81
|
+
if (received + value.byteLength > length) {
|
|
82
|
+
await reader.cancel()
|
|
83
|
+
throw new Error(`Cloud document range read sent more than the ${length} bytes asked for`)
|
|
84
|
+
}
|
|
85
|
+
bytes.set(value, received)
|
|
86
|
+
received += value.byteLength
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (received !== length) {
|
|
90
|
+
throw new Error(`Cloud document range read sent ${received} of the ${length} bytes asked for`)
|
|
91
|
+
}
|
|
92
|
+
return bytes
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Bytes `start` to `endExclusive` of a revision, for reading its preview without the rest. Only
|
|
97
|
+
* a partial response of exactly that range counts: a store that ignores the range, or answers
|
|
98
|
+
* another one, is stopped rather than left sending the file. `Content-Range` is checked when the
|
|
99
|
+
* store's CORS rules let the browser read it.
|
|
100
|
+
*/
|
|
101
|
+
export async function readCloudRevisionRange(
|
|
102
|
+
download: DocumentDownload,
|
|
103
|
+
objectFetch: (input: string, init: RequestInit) => Promise<Response>,
|
|
104
|
+
start: number,
|
|
105
|
+
endExclusive: number
|
|
106
|
+
): Promise<Uint8Array> {
|
|
107
|
+
const response = await objectFetch(download.download.url, {
|
|
108
|
+
method: download.download.method,
|
|
109
|
+
headers: { ...download.download.headers, Range: `bytes=${start}-${endExclusive - 1}` }
|
|
110
|
+
})
|
|
111
|
+
const range = response.headers.get('Content-Range')
|
|
112
|
+
if (
|
|
113
|
+
response.status !== 206 ||
|
|
114
|
+
(range && !range.startsWith(`bytes ${start}-${endExclusive - 1}/`))
|
|
115
|
+
) {
|
|
116
|
+
await response.body?.cancel()
|
|
117
|
+
throw new Error(
|
|
118
|
+
`Cloud document range read failed with HTTP ${response.status}${range ? ` (${range})` : ''}`
|
|
119
|
+
)
|
|
120
|
+
}
|
|
121
|
+
return readExactly(response, endExclusive - start)
|
|
122
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CLOUD_API_MAX_BODY_BYTES,
|
|
3
|
+
type CommitImageUploadsInput,
|
|
4
|
+
type ObjectUploadTarget
|
|
5
|
+
} from '#cloud/contract'
|
|
6
|
+
import { limitAsync, sumBy } from 'es-toolkit'
|
|
7
|
+
|
|
8
|
+
import type { CloudAPIClient } from '../api'
|
|
9
|
+
import { putCloudObject, type CloudObjectFetch } from './objects'
|
|
10
|
+
|
|
11
|
+
const IMAGE_UPLOAD_CONCURRENCY = 4
|
|
12
|
+
/** How much of a request one image commit may fill: each upload brings its parts' ETags. */
|
|
13
|
+
const IMAGE_COMMIT_BYTES = CLOUD_API_MAX_BODY_BYTES / 2
|
|
14
|
+
|
|
15
|
+
/** What one upload adds to a commit besides its parts: its keys, its ID, and its checksum. */
|
|
16
|
+
const UPLOAD_ENTRY_BYTES = 256
|
|
17
|
+
/** What one part adds besides its ETag: its keys, a part number up to 10,000, and escaped quotes. */
|
|
18
|
+
const PART_ENTRY_BYTES = 48
|
|
19
|
+
|
|
20
|
+
type ImageCommitUpload = CommitImageUploadsInput['uploads'][number]
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* At most what an upload adds to a commit's body: fixed room for its fields, and the lengths of
|
|
24
|
+
* the strings the object store chose, its multipart upload's ID and each part's ETag.
|
|
25
|
+
*/
|
|
26
|
+
function commitEntryBytes(upload: ImageCommitUpload): number {
|
|
27
|
+
const multipart = upload.multipart
|
|
28
|
+
if (!multipart) return UPLOAD_ENTRY_BYTES
|
|
29
|
+
return (
|
|
30
|
+
UPLOAD_ENTRY_BYTES +
|
|
31
|
+
multipart.uploadId.length +
|
|
32
|
+
sumBy(multipart.parts, (part) => PART_ENTRY_BYTES + part.etag.length)
|
|
33
|
+
)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Uploads in commits that each stay under `IMAGE_COMMIT_BYTES` once sent. */
|
|
37
|
+
export function imageCommitBatches<T extends ImageCommitUpload>(uploads: readonly T[]): T[][] {
|
|
38
|
+
const batches: T[][] = []
|
|
39
|
+
let batch: T[] = []
|
|
40
|
+
let bytes = 0
|
|
41
|
+
for (const upload of uploads) {
|
|
42
|
+
const size = commitEntryBytes(upload)
|
|
43
|
+
if (batch.length > 0 && bytes + size > IMAGE_COMMIT_BYTES) {
|
|
44
|
+
batches.push(batch)
|
|
45
|
+
batch = []
|
|
46
|
+
bytes = 0
|
|
47
|
+
}
|
|
48
|
+
batch.push(upload)
|
|
49
|
+
bytes += size
|
|
50
|
+
}
|
|
51
|
+
if (batch.length > 0) batches.push(batch)
|
|
52
|
+
return batches
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Uploads the images the document's workspace lacks: the server answers with uploads for those
|
|
57
|
+
* only, and they are committed together once their bytes are in place.
|
|
58
|
+
*/
|
|
59
|
+
export async function uploadCloudImages(input: {
|
|
60
|
+
cloud: Pick<CloudAPIClient, 'requestImageUploads' | 'commitImageUploads'>
|
|
61
|
+
documentId: string
|
|
62
|
+
images: { checksum: string; bytes: Uint8Array }[]
|
|
63
|
+
objectFetch: CloudObjectFetch
|
|
64
|
+
signal?: AbortSignal
|
|
65
|
+
}): Promise<void> {
|
|
66
|
+
if (input.images.length === 0) return
|
|
67
|
+
const byChecksum = new Map(input.images.map((image) => [image.checksum, image.bytes]))
|
|
68
|
+
const offers = await input.cloud.requestImageUploads(input.documentId, {
|
|
69
|
+
images: [...byChecksum].map(([checksum, bytes]) => ({ checksum, byteSize: bytes.byteLength }))
|
|
70
|
+
})
|
|
71
|
+
if (offers.length === 0) return
|
|
72
|
+
const put = limitAsync(
|
|
73
|
+
(target: ObjectUploadTarget, bytes: Uint8Array) =>
|
|
74
|
+
putCloudObject(target, { bytes, objectFetch: input.objectFetch, signal: input.signal }),
|
|
75
|
+
IMAGE_UPLOAD_CONCURRENCY
|
|
76
|
+
)
|
|
77
|
+
const uploaded = await Promise.all(
|
|
78
|
+
offers.map(async (offer) => {
|
|
79
|
+
const bytes = byChecksum.get(offer.checksum)
|
|
80
|
+
if (!bytes) throw new Error('The server offered an upload for an image this save lacks')
|
|
81
|
+
const multipart = await put(offer.upload, bytes)
|
|
82
|
+
return { uploadId: offer.uploadId, checksum: offer.checksum, multipart }
|
|
83
|
+
})
|
|
84
|
+
)
|
|
85
|
+
// As few commits as fit the server's body limit, in turn, however many images a save adds.
|
|
86
|
+
for (const batch of imageCommitBatches(uploaded)) {
|
|
87
|
+
await input.cloud.commitImageUploads(input.documentId, batch)
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export { assembleCloudDocument, splitCloudDocument, type CloudDocumentImage } from './archive'
|
|
2
|
+
export { cloudChecksum } from './checksum'
|
|
3
|
+
export { downloadCloudDocument, downloadCloudImage, readCloudRevisionRange } from './download'
|
|
4
|
+
export { imageCommitBatches, uploadCloudImages } from './images'
|
|
5
|
+
export {
|
|
6
|
+
putCloudObject,
|
|
7
|
+
uploadCloudObject,
|
|
8
|
+
type CloudMultipartCommit,
|
|
9
|
+
type CloudObjectFetch,
|
|
10
|
+
type CloudObjectTransfer,
|
|
11
|
+
type CloudObjectUpload,
|
|
12
|
+
type CloudObjectUploadResult,
|
|
13
|
+
type CloudTransferProgress
|
|
14
|
+
} from './objects'
|
|
15
|
+
export { prepareCloudDocument, saveCloudDocument, type PreparedCloudDocument } from './save'
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import type { ObjectUploadTarget } from '#cloud/contract'
|
|
2
|
+
import { limitAsync, retry } from 'es-toolkit'
|
|
3
|
+
|
|
4
|
+
import { backoffDelay, SECOND_MS, type Backoff } from '@open-pencil/std/timing'
|
|
5
|
+
|
|
6
|
+
import type { CloudAPIClient } from '../api'
|
|
7
|
+
|
|
8
|
+
const MULTIPART_CONCURRENCY = 4
|
|
9
|
+
const PART_RETRIES = 2
|
|
10
|
+
/** A part's upload is retried soon, since a part fails on a dropped connection more than a busy store. */
|
|
11
|
+
const PART_RETRY: Backoff = { baseMs: 200, capMs: 5 * SECOND_MS, jitter: true }
|
|
12
|
+
|
|
13
|
+
/** How bytes reach the object store, which carries no Cloud session: `fetch` or a native one. */
|
|
14
|
+
export type CloudObjectFetch = (input: string, init: RequestInit) => Promise<Response>
|
|
15
|
+
|
|
16
|
+
/** How much of a transfer is done; the total is unknown until the server says. */
|
|
17
|
+
export type CloudTransferProgress = { transferredBytes: number; totalBytes: number | null }
|
|
18
|
+
type MultipartUpload = Extract<ObjectUploadTarget, { kind: 'multipart' }>
|
|
19
|
+
|
|
20
|
+
/** Bytes on their way to the object store, and how to report and stop the transfer. */
|
|
21
|
+
export type CloudObjectTransfer = {
|
|
22
|
+
bytes: Uint8Array
|
|
23
|
+
objectFetch: CloudObjectFetch
|
|
24
|
+
onProgress?: (progress: CloudTransferProgress) => void
|
|
25
|
+
signal?: AbortSignal
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type CloudObjectUpload = CloudObjectTransfer & {
|
|
29
|
+
cloud: Pick<CloudAPIClient, 'createUpload'>
|
|
30
|
+
documentId: string
|
|
31
|
+
checksum: string
|
|
32
|
+
baseRevision: string | null
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The parts a multipart upload's commit names. */
|
|
36
|
+
export type CloudMultipartCommit = {
|
|
37
|
+
uploadId: string
|
|
38
|
+
parts: { partNumber: number; etag: string }[]
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type CloudObjectUploadResult = {
|
|
42
|
+
uploadId: string
|
|
43
|
+
multipart?: CloudMultipartCommit
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** An object-store request the presigned URL refused or the store failed. */
|
|
47
|
+
class ObjectUploadError extends Error {
|
|
48
|
+
override readonly name = 'ObjectUploadError'
|
|
49
|
+
constructor(readonly status: number) {
|
|
50
|
+
super(`Cloud object upload failed with HTTP ${status}`)
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const retryable = (status: number) => status === 408 || status === 429 || status >= 500
|
|
55
|
+
// Presigned URLs expire; a fresh upload session gets new ones.
|
|
56
|
+
const expired = (error: unknown) =>
|
|
57
|
+
error instanceof ObjectUploadError && (error.status === 401 || error.status === 403)
|
|
58
|
+
|
|
59
|
+
function body(bytes: Uint8Array): Uint8Array<ArrayBuffer> {
|
|
60
|
+
return bytes.buffer instanceof ArrayBuffer
|
|
61
|
+
? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
62
|
+
: Uint8Array.from(bytes)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function report(upload: CloudObjectTransfer, transferredBytes: number): void {
|
|
66
|
+
upload.onProgress?.({ transferredBytes, totalBytes: upload.bytes.byteLength })
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function putPart(
|
|
70
|
+
upload: CloudObjectTransfer,
|
|
71
|
+
part: MultipartUpload['parts'][number],
|
|
72
|
+
bytes: Uint8Array,
|
|
73
|
+
signal: AbortSignal
|
|
74
|
+
): Promise<string> {
|
|
75
|
+
return retry(
|
|
76
|
+
async () => {
|
|
77
|
+
const response = await upload.objectFetch(part.url, {
|
|
78
|
+
method: part.method,
|
|
79
|
+
headers: part.headers,
|
|
80
|
+
body: body(bytes),
|
|
81
|
+
signal
|
|
82
|
+
})
|
|
83
|
+
if (!response.ok) throw new ObjectUploadError(response.status)
|
|
84
|
+
const etag = response.headers.get('etag')
|
|
85
|
+
if (!etag) throw new Error('Cloud multipart upload response did not include an ETag')
|
|
86
|
+
return etag
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
retries: PART_RETRIES,
|
|
90
|
+
delay: (attempt) => backoffDelay(attempt, PART_RETRY),
|
|
91
|
+
signal,
|
|
92
|
+
shouldRetry: (error) => error instanceof ObjectUploadError && retryable(error.status)
|
|
93
|
+
}
|
|
94
|
+
)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async function putParts(upload: CloudObjectTransfer, multipart: MultipartUpload) {
|
|
98
|
+
const controller = new AbortController()
|
|
99
|
+
const stop = () => controller.abort(upload.signal?.reason)
|
|
100
|
+
upload.signal?.addEventListener('abort', stop, { once: true })
|
|
101
|
+
let transferred = 0
|
|
102
|
+
const putOne = limitAsync(async (part: MultipartUpload['parts'][number]) => {
|
|
103
|
+
try {
|
|
104
|
+
controller.signal.throwIfAborted()
|
|
105
|
+
const start = (part.partNumber - 1) * multipart.partSize
|
|
106
|
+
const bytes = upload.bytes.subarray(start, start + multipart.partSize)
|
|
107
|
+
const etag = await putPart(upload, part, bytes, controller.signal)
|
|
108
|
+
transferred += bytes.byteLength
|
|
109
|
+
report(upload, transferred)
|
|
110
|
+
return { partNumber: part.partNumber, etag }
|
|
111
|
+
} catch (error) {
|
|
112
|
+
// One failed part fails the upload; the others stop instead of finishing for nothing.
|
|
113
|
+
controller.abort(error)
|
|
114
|
+
throw error
|
|
115
|
+
}
|
|
116
|
+
}, MULTIPART_CONCURRENCY)
|
|
117
|
+
try {
|
|
118
|
+
return await Promise.all(multipart.parts.map(putOne))
|
|
119
|
+
} finally {
|
|
120
|
+
upload.signal?.removeEventListener('abort', stop)
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Puts bytes where the server said, in parallel parts for a multipart upload. What a commit names
|
|
126
|
+
* of a multipart upload, or nothing for a single one.
|
|
127
|
+
*/
|
|
128
|
+
export async function putCloudObject(
|
|
129
|
+
target: ObjectUploadTarget,
|
|
130
|
+
upload: CloudObjectTransfer
|
|
131
|
+
): Promise<CloudMultipartCommit | undefined> {
|
|
132
|
+
report(upload, 0)
|
|
133
|
+
if (target.kind === 'single') {
|
|
134
|
+
const response = await upload.objectFetch(target.url, {
|
|
135
|
+
method: target.method,
|
|
136
|
+
headers: target.headers,
|
|
137
|
+
body: body(upload.bytes),
|
|
138
|
+
signal: upload.signal
|
|
139
|
+
})
|
|
140
|
+
if (!response.ok) throw new ObjectUploadError(response.status)
|
|
141
|
+
report(upload, upload.bytes.byteLength)
|
|
142
|
+
return undefined
|
|
143
|
+
}
|
|
144
|
+
const parts = await putParts(upload, target)
|
|
145
|
+
return { uploadId: target.uploadId, parts }
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function transfer(upload: CloudObjectUpload): Promise<CloudObjectUploadResult> {
|
|
149
|
+
const pending = await upload.cloud.createUpload(upload.documentId, {
|
|
150
|
+
baseRevisionId: upload.baseRevision,
|
|
151
|
+
byteSize: upload.bytes.byteLength,
|
|
152
|
+
checksum: upload.checksum,
|
|
153
|
+
contentType: 'application/octet-stream'
|
|
154
|
+
})
|
|
155
|
+
const multipart = await putCloudObject(pending.upload, upload)
|
|
156
|
+
return multipart ? { uploadId: pending.id, multipart } : { uploadId: pending.id }
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Sends a document's bytes straight to the server's object store through presigned URLs, in
|
|
161
|
+
* parallel parts for large files, starting once more if the URLs expired on the way.
|
|
162
|
+
*/
|
|
163
|
+
export async function uploadCloudObject(
|
|
164
|
+
upload: CloudObjectUpload
|
|
165
|
+
): Promise<CloudObjectUploadResult> {
|
|
166
|
+
try {
|
|
167
|
+
return await transfer(upload)
|
|
168
|
+
} catch (error) {
|
|
169
|
+
if (!expired(error)) throw error
|
|
170
|
+
upload.signal?.throwIfAborted()
|
|
171
|
+
return transfer(upload)
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import type { DocumentSummary } from '#cloud/contract'
|
|
2
|
+
|
|
3
|
+
import { CloudAPIError, type CloudAPIClient } from '../api'
|
|
4
|
+
import { splitCloudDocument } from './archive'
|
|
5
|
+
import { cloudChecksum } from './checksum'
|
|
6
|
+
import { uploadCloudImages } from './images'
|
|
7
|
+
import { uploadCloudObject, type CloudObjectFetch, type CloudTransferProgress } from './objects'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A `.fig` as Cloud stores it: the file without its images, which the workspace keeps once, with
|
|
11
|
+
* the checksum of each. The lean file's checksum is the revision's, so an unchanged file is known
|
|
12
|
+
* before anything is sent.
|
|
13
|
+
*/
|
|
14
|
+
export type PreparedCloudDocument = {
|
|
15
|
+
lean: Uint8Array
|
|
16
|
+
checksum: string
|
|
17
|
+
images: { name: string; bytes: Uint8Array; checksum: string }[]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export async function prepareCloudDocument(bytes: Uint8Array): Promise<PreparedCloudDocument> {
|
|
21
|
+
const { lean, images } = await splitCloudDocument(bytes)
|
|
22
|
+
return {
|
|
23
|
+
lean,
|
|
24
|
+
checksum: await cloudChecksum(lean),
|
|
25
|
+
images: await Promise.all(
|
|
26
|
+
images.map(async (image) => ({ ...image, checksum: await cloudChecksum(image.bytes) }))
|
|
27
|
+
)
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** The server refused the save as declared, as when an image it named was released meanwhile. */
|
|
32
|
+
function invalidUpload(error: unknown): boolean {
|
|
33
|
+
return error instanceof CloudAPIError && error.code === 'invalid_upload'
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Saves a document as a new revision on top of `baseRevision`, or as its first: the images the
|
|
38
|
+
* workspace lacks, then the file, committed together. An image released between asking and
|
|
39
|
+
* committing is uploaded again by a second try. A save on top of a revision that is no longer the
|
|
40
|
+
* latest fails with a `revision_conflict` `CloudAPIError`.
|
|
41
|
+
*/
|
|
42
|
+
export async function saveCloudDocument(input: {
|
|
43
|
+
cloud: Pick<
|
|
44
|
+
CloudAPIClient,
|
|
45
|
+
'requestImageUploads' | 'commitImageUploads' | 'createUpload' | 'commitUpload'
|
|
46
|
+
>
|
|
47
|
+
documentId: string
|
|
48
|
+
document: PreparedCloudDocument
|
|
49
|
+
name: string
|
|
50
|
+
kind: DocumentSummary['kind']
|
|
51
|
+
baseRevision: string | null
|
|
52
|
+
objectFetch: CloudObjectFetch
|
|
53
|
+
onProgress?: (progress: CloudTransferProgress) => void
|
|
54
|
+
}): Promise<DocumentSummary> {
|
|
55
|
+
const { cloud, documentId, document, objectFetch } = input
|
|
56
|
+
const save = async () => {
|
|
57
|
+
await uploadCloudImages({ cloud, documentId, images: document.images, objectFetch })
|
|
58
|
+
const upload = await uploadCloudObject({
|
|
59
|
+
cloud,
|
|
60
|
+
documentId,
|
|
61
|
+
bytes: document.lean,
|
|
62
|
+
checksum: document.checksum,
|
|
63
|
+
baseRevision: input.baseRevision,
|
|
64
|
+
objectFetch,
|
|
65
|
+
onProgress: input.onProgress
|
|
66
|
+
})
|
|
67
|
+
return cloud.commitUpload(upload.uploadId, {
|
|
68
|
+
checksum: document.checksum,
|
|
69
|
+
name: input.name,
|
|
70
|
+
kind: input.kind,
|
|
71
|
+
multipart: upload.multipart,
|
|
72
|
+
images: document.images.map((image) => ({ name: image.name, checksum: image.checksum }))
|
|
73
|
+
})
|
|
74
|
+
}
|
|
75
|
+
return save().catch((error: unknown) => {
|
|
76
|
+
if (invalidUpload(error)) return save()
|
|
77
|
+
throw error
|
|
78
|
+
})
|
|
79
|
+
}
|
package/src/client/index.ts
CHANGED
|
@@ -7,13 +7,9 @@ export {
|
|
|
7
7
|
type CloudServicesClientOptions,
|
|
8
8
|
type CloudServicesTokenSource
|
|
9
9
|
} from './services'
|
|
10
|
-
export {
|
|
11
|
-
CloudBillingError,
|
|
12
|
-
createCloudBillingClient,
|
|
13
|
-
type CloudBillingClient,
|
|
14
|
-
type CloudBillingClientOptions
|
|
15
|
-
} from './billing'
|
|
16
10
|
export { cloudEditorReturnURL, cloudRedirectPath, cloudSignInURL } from './auth/navigation'
|
|
11
|
+
export * from './files'
|
|
12
|
+
export { fetchCommunityTour, type CommunityTour } from './tour'
|
|
17
13
|
export {
|
|
18
14
|
CloudAPIError,
|
|
19
15
|
createCloudAPIClient,
|
|
@@ -58,3 +54,4 @@ export {
|
|
|
58
54
|
type CloudPortalClient,
|
|
59
55
|
type CloudPortalClientOptions
|
|
60
56
|
} from './portal/client'
|
|
57
|
+
export type { CloudCommentRequests } from './comments'
|
|
@@ -17,13 +17,31 @@ import {
|
|
|
17
17
|
cloudProfileResponseSchema,
|
|
18
18
|
cloudSignInsResponseSchema,
|
|
19
19
|
cloudSocialLinkResponseSchema,
|
|
20
|
-
|
|
20
|
+
notificationSettingsSchema,
|
|
21
|
+
type CloudAccessTokenCreate,
|
|
22
|
+
type CloudSocialProvider,
|
|
23
|
+
type NotificationSettings
|
|
21
24
|
} from '#cloud/contract'
|
|
22
25
|
|
|
23
26
|
import type { CloudCall, PortalRouteClients } from './request'
|
|
24
27
|
|
|
25
28
|
export function createAccountAPI(call: CloudCall, { account, tokens, api }: PortalRouteClients) {
|
|
26
29
|
return {
|
|
30
|
+
/** Which comments the account hears about by email. */
|
|
31
|
+
notificationSettings(signal?: AbortSignal) {
|
|
32
|
+
return call(
|
|
33
|
+
(options) => api.account.notifications.$get(undefined, options),
|
|
34
|
+
notificationSettingsSchema,
|
|
35
|
+
signal
|
|
36
|
+
)
|
|
37
|
+
},
|
|
38
|
+
setNotificationSettings(settings: NotificationSettings, signal?: AbortSignal) {
|
|
39
|
+
return call(
|
|
40
|
+
(options) => api.account.notifications.$put({ json: settings }, options),
|
|
41
|
+
notificationSettingsSchema,
|
|
42
|
+
signal
|
|
43
|
+
)
|
|
44
|
+
},
|
|
27
45
|
/** The account's plan, and the storage its workspaces hold together. */
|
|
28
46
|
entitlements(signal?: AbortSignal) {
|
|
29
47
|
return call(
|
|
@@ -129,7 +147,7 @@ export function createAccountAPI(call: CloudCall, { account, tokens, api }: Port
|
|
|
129
147
|
signal
|
|
130
148
|
)
|
|
131
149
|
},
|
|
132
|
-
linkSocial(provider:
|
|
150
|
+
linkSocial(provider: CloudSocialProvider, callbackURL: string, signal?: AbortSignal) {
|
|
133
151
|
return call(
|
|
134
152
|
(options) =>
|
|
135
153
|
account.authentication['link-social'].$post({ json: { provider, callbackURL } }, options),
|