@uploadfilm/sdk 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +26 -6
  2. package/dist/copy/types.d.ts +29 -2
  3. package/dist/index.js +1 -1
  4. package/dist/index.mjs +1 -1
  5. package/dist/player.js +1 -1
  6. package/dist/player.mjs +1 -1
  7. package/dist/server/media/client.d.ts +34 -1
  8. package/dist/server/media/protocol.d.ts +242 -4
  9. package/dist/server/r2/client.d.ts +18 -4
  10. package/dist/server/storage/chunked.d.ts +6 -2
  11. package/dist/server/storage/s3.d.ts +51 -0
  12. package/dist/server/testing/fake-r2.d.ts +22 -2
  13. package/dist/server/transfer/handler.d.ts +50 -0
  14. package/dist/server/transfer/protocol.d.ts +53 -4
  15. package/dist/server-media.js +1 -1
  16. package/dist/server-media.mjs +1 -1
  17. package/dist/server-testing.js +1 -1
  18. package/dist/server-testing.mjs +1 -1
  19. package/dist/server.js +1 -1
  20. package/dist/server.mjs +1 -1
  21. package/dist/transfer/client.d.ts +13 -2
  22. package/dist/transfer/engine.d.ts +22 -2
  23. package/dist/transfer/growing/bytes.d.ts +17 -0
  24. package/dist/transfer/growing/index.d.ts +12 -0
  25. package/dist/transfer/growing/parts.d.ts +157 -0
  26. package/dist/transfer/growing/retry.d.ts +38 -0
  27. package/dist/transfer/growing/store.d.ts +66 -0
  28. package/dist/transfer/growing/upload.d.ts +126 -0
  29. package/dist/transfer/index.d.ts +1 -0
  30. package/dist/transfer/protocol.d.ts +5 -0
  31. package/dist/transfer/storage.d.ts +28 -3
  32. package/dist/workers/cli.mjs +115 -0
  33. package/dist/workers/media/config.mjs +53 -0
  34. package/dist/workers/media/container/Dockerfile +19 -0
  35. package/dist/workers/media/container/FFMPEG.md +129 -0
  36. package/dist/workers/media/container/runner.mjs +2690 -0
  37. package/dist/workers/media/index.mjs +1011 -0
  38. package/dist/workers/media/init/config.ts +52 -0
  39. package/dist/workers/media/init/worker.ts +8 -0
  40. package/dist/workers/media/init/wrangler.toml +44 -0
  41. package/dist/workers/media/runner.mjs +2656 -0
  42. package/dist/workers/media/types/build/entry/config.d.mts +6 -0
  43. package/dist/workers/media/types/build/entry/index.d.mts +43 -0
  44. package/dist/workers/media/types/build/entry/program.d.mts +13 -0
  45. package/dist/workers/media/types/build/entry/runner.d.mts +12 -0
  46. package/dist/workers/media/types/build/entry/settings.d.mts +40 -0
  47. package/dist/workers/media/types/src/config.d.mts +135 -0
  48. package/dist/workers/media/types/src/jobs.d.mts +73 -0
  49. package/dist/workers/media/types/src/runner/container.d.mts +32 -0
  50. package/dist/workers/media/types/src/runner/ffmpeg.d.mts +227 -0
  51. package/dist/workers/media/types/src/runner/job.d.mts +107 -0
  52. package/dist/workers/media/types/src/runner/node.d.mts +71 -0
  53. package/dist/workers/media/types/src/runner/plugins.d.mts +45 -0
  54. package/dist/workers/media/types/src/serve.d.mts +107 -0
  55. package/dist/workers/media/types/src/steps/audio.d.mts +33 -0
  56. package/dist/workers/media/types/src/steps/edit.d.mts +62 -0
  57. package/dist/workers/media/types/src/steps/frameCheck.d.mts +44 -0
  58. package/dist/workers/media/types/src/steps/gate.d.mts +18 -0
  59. package/dist/workers/media/types/src/steps/paths.d.mts +1 -0
  60. package/dist/workers/media/types/src/steps/prep.d.mts +60 -0
  61. package/dist/workers/media/types/src/steps/render.d.mts +40 -0
  62. package/dist/workers/media/types/src/steps/runner.d.mts +30 -0
  63. package/dist/workers/media/types/src/steps/transcribe.d.mts +89 -0
  64. package/dist/workers/media/types/src/storyboard/index.d.mts +61 -0
  65. package/dist/workers/media/types/src/transcoder.d.mts +30 -0
  66. package/dist/workers/media/types/src/worker.d.mts +30 -0
  67. package/dist/workers/zip/index.mjs +589 -0
  68. package/dist/workers/zip/init/config.ts +22 -0
  69. package/dist/workers/zip/init/worker.ts +5 -0
  70. package/dist/workers/zip/init/wrangler.toml +25 -0
  71. package/dist/workers/zip/types/build/entry/index.d.mts +5 -0
  72. package/dist/workers/zip/types/src/config.d.mts +35 -0
  73. package/dist/workers/zip/types/src/serve.d.mts +86 -0
  74. package/dist/workers/zip/types/src/worker.d.mts +12 -0
  75. package/package.json +51 -3
@@ -95,6 +95,8 @@ export interface TransferEngineOptions {
95
95
  };
96
96
  /** See EngineCoreOptions (gaps 2–4): "md5" (default) or "sha256" with storage receipts, failing closed. */
97
97
  integrity?: "md5" | "sha256";
98
+ /** See EngineCoreOptions.readBack: pre-existing copies proven by a read-back SHA-256 (opt-in; the server's `readable`). */
99
+ readBack?: boolean;
98
100
  partSizeFor?(size: number): number;
99
101
  singlePutMax?: number;
100
102
  maxParts?: number;
@@ -219,7 +221,12 @@ export declare function createTransferEngine(o: TransferEngineOptions): {
219
221
  completeMultipart: (scope: string, key: string, uploadId: string, parts: {
220
222
  n: number;
221
223
  etag: string;
222
- }[]) => Promise<import("./storage").Stat>;
224
+ }[]) => Promise<{
225
+ recovered?: true | undefined;
226
+ size: number | null;
227
+ etag: string | null;
228
+ weak?: boolean;
229
+ }>;
223
230
  abortMultipart: (scope: string, key: string, uploadId: string) => Promise<void | undefined>;
224
231
  head: (scope: string, key: string) => Promise<import("./storage").Stat>;
225
232
  listParts: (scope: string, key: string, uploadId: string) => Promise<Map<number, string>>;
@@ -228,7 +235,7 @@ export declare function createTransferEngine(o: TransferEngineOptions): {
228
235
  initiated?: string;
229
236
  }[]>;
230
237
  uploadPart: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, onProgress: (sent: number) => void, timeoutMs: number, y?: import("./protocol").Yieldable) => Promise<string>;
231
- putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: import("./protocol").Yieldable, sha256?: string) => Promise<string>;
238
+ putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: import("./protocol").Yieldable, sha256?: string, w?: import("./storage").WriteMark) => Promise<string>;
232
239
  transport: Transport;
233
240
  createSession: (scope: string, key: string, contentType: string, size: number, counts: boolean, x: {
234
241
  sha256?: string;
@@ -251,6 +258,10 @@ export declare function createTransferEngine(o: TransferEngineOptions): {
251
258
  checksumSha256: string;
252
259
  }[]) => Promise<import("./storage").CheckedStat>;
253
260
  headWithChecksum: (scope: string, key: string) => Promise<import("./storage").CheckedStat>;
261
+ readBack: (scope: string, key: string, want: number, sha256Stream: () => {
262
+ update(b: Uint8Array): unknown;
263
+ digest(): Uint8Array;
264
+ }, crc32: (b: Uint8Array, prev?: number) => number) => Promise<import("./storage").ReadBack | null>;
254
265
  };
255
266
  };
256
267
  export {};
@@ -185,6 +185,17 @@ export interface EngineCoreOptions<I extends UploadItem> {
185
185
  * The stitched object is verified by storage's composite SHA-256 (or `confirmStored`).
186
186
  */
187
187
  integrity?: "md5" | "sha256";
188
+ /**
189
+ * Opt-in (default off): a copy found at the key (another uploader's, an earlier install's, anyone's who can write
190
+ * there, or an earlier attempt of this engine's) is adopted only after reading it back whole through the server's
191
+ * `get` op (createSignHandler `readable`) and matching its SHA-256 against the file; a server that refuses the read
192
+ * then gives `Unverified`, never `verified`. Every copy found at the key is proven that way, an earlier attempt of this
193
+ * engine's whose answer was lost included (it is these bytes: adopted, at the cost of one read; review round 2: an
194
+ * "it may be ours" marker let foreign copies through on MD5). Off: such an object is adopted on its MD5 ETag (cheap; a collision crafted
195
+ * against the file's MD5 would pass: see SECURITY.md "MD5 adoption"). Ignored with integrity "sha256" (storage's SHA-256
196
+ * proves it already).
197
+ */
198
+ readBack?: boolean;
188
199
  /** The client's part size for a new multipart (default partSizeFor). Given → also proposed to the server on create. */
189
200
  partSizeFor?(size: number): number;
190
201
  /** Below this size an object goes as one PUT (default SMALL, 8 MiB). */
@@ -266,7 +277,12 @@ export declare function createEngineCore<I extends UploadItem>(o: EngineCoreOpti
266
277
  completeMultipart: (scope: string, key: string, uploadId: string, parts: {
267
278
  n: number;
268
279
  etag: string;
269
- }[]) => Promise<import("./storage").Stat>;
280
+ }[]) => Promise<{
281
+ recovered?: true | undefined;
282
+ size: number | null;
283
+ etag: string | null;
284
+ weak?: boolean;
285
+ }>;
270
286
  abortMultipart: (scope: string, key: string, uploadId: string) => Promise<void | undefined>;
271
287
  head: (scope: string, key: string) => Promise<import("./storage").Stat>;
272
288
  listParts: (scope: string, key: string, uploadId: string) => Promise<Map<number, string>>;
@@ -275,7 +291,7 @@ export declare function createEngineCore<I extends UploadItem>(o: EngineCoreOpti
275
291
  initiated?: string;
276
292
  }[]>;
277
293
  uploadPart: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, onProgress: (sent: number) => void, timeoutMs: number, y?: Yieldable) => Promise<string>;
278
- putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: Yieldable, sha256?: string) => Promise<string>;
294
+ putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: Yieldable, sha256?: string, w?: import("./storage").WriteMark) => Promise<string>;
279
295
  transport: import("./protocol").Transport;
280
296
  createSession: (scope: string, key: string, contentType: string, size: number, counts: boolean, x: {
281
297
  sha256?: string;
@@ -298,6 +314,10 @@ export declare function createEngineCore<I extends UploadItem>(o: EngineCoreOpti
298
314
  checksumSha256: string;
299
315
  }[]) => Promise<import("./storage").CheckedStat>;
300
316
  headWithChecksum: (scope: string, key: string) => Promise<import("./storage").CheckedStat>;
317
+ readBack: (scope: string, key: string, want: number, sha256Stream: () => {
318
+ update(b: Uint8Array): unknown;
319
+ digest(): Uint8Array;
320
+ }, crc32: (b: Uint8Array, prev?: number) => number) => Promise<import("./storage").ReadBack | null>;
301
321
  };
302
322
  readonly rate: number;
303
323
  scopeRates: () => {
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Byte helpers for the growing uploader. Browser-safe on purpose (no Buffer, no node: imports): this module runs in a
3
+ * page, an extension's offscreen document, a Worker, Electron and Node alike.
4
+ */
5
+ export declare const toHex: (b: Uint8Array) => string;
6
+ export declare const fromHex: (h: string) => Uint8Array;
7
+ export declare const toB64: (b: Uint8Array) => string;
8
+ export declare const concat: (parts: Uint8Array[]) => Uint8Array;
9
+ export declare const isMd5Hex: (s: unknown) => s is string;
10
+ /** An ETag as storage sends it → bare lowercase (no quotes, no W/). */
11
+ export declare const bareEtag: (e: unknown) => string;
12
+ /**
13
+ * A fragment as a Blob. Copies a Uint8Array's bytes, so the recorder is free to reuse its buffer (ClipMe's `toBlob`).
14
+ * Blobs let a slow network's backlog live outside the JS heap where the runtime can (browsers spill big Blobs to disk).
15
+ */
16
+ export declare const toBlob: (fragment: Uint8Array | Blob) => Blob;
17
+ export declare const blobBytes: (b: Blob) => Promise<Uint8Array>;
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Growing uploads (upload while recording), ported from ClipMe's uploader. Browser-safe: no node: imports, no Buffer.
3
+ * See docs/platform/GROWING-UPLOADS.md.
4
+ */
5
+ export { createGrowingUpload, growingRetryDelayMs, COMPLETE_FAILURES_BEFORE_RESTART } from "./upload";
6
+ export type { GrowingUpload, GrowingUploadOptions, GrowingResult, GrowingState, GrowingProgress } from "./upload";
7
+ export { createGrowingRetryPump } from "./retry";
8
+ export type { GrowingLocalCopy, GrowingRetryOptions } from "./retry";
9
+ export { GrowingPartUploader, GROWING_PART_BYTES, MAX_PARALLEL_PARTS, MAX_ATTEMPTS, MAX_PENDING_BYTES, AHEAD_MAX_AGE_MS, SignRefused, PartIntegrityError, isGone, md5Of } from "./parts";
10
+ export type { GrowingPart, PartSender, PartHasher, PartUploaderOptions, SignedPart } from "./parts";
11
+ export { memoryGrowingStore, listRecords as listGrowingUploads, loadRecord as loadGrowingRecord, RECORD_PREFIX as GROWING_RECORD_PREFIX } from "./store";
12
+ export type { GrowingStore, GrowingRecord, GrowingRecordState } from "./store";
@@ -0,0 +1,157 @@
1
+ /**
2
+ * Turns a stream of fragments into multipart parts and gets them to storage while the recording runs.
3
+ *
4
+ * Ported from ClipMe's extension/src/upload/parts.ts (PartUploader, ClipMe origin/main 4684516), itself a port of its
5
+ * Electron uploader's part sizing. The behaviour is ClipMe's: every non-trailing part is EXACTLY `partSize` (R2
6
+ * refuses a multipart whose non-trailing parts differ in length, and only at complete, once a recording needs three
7
+ * parts), the trailing part is the only short one, at most 3 parts in flight, 6 attempts per part with backoff 0.5 s
8
+ * doubling to 20 s, the next part's URL signed while the current one uploads, a cap on unsent bytes that ends the
9
+ * upload rather than the recording, and parts an earlier attempt stored are kept by number.
10
+ *
11
+ * What the SDK adds (docs/platform/GROWING-UPLOADS.md lists every difference):
12
+ * - every part is proven: its MD5 is computed before it is sent, retries are signed with Content-MD5 (storage refuses
13
+ * a body that changed in transit), and storage's part ETag must equal that MD5 or the part is sent again;
14
+ * - a kept part (resume) is kept only when the bytes cut at its position now hash to what storage holds: ClipMe trusts
15
+ * the local copy to be the same bytes; here a different part is sent again;
16
+ * - a refusal the server marks as final (a quota, a key outside the layouts) ends the upload at once, without the
17
+ * remaining attempts.
18
+ *
19
+ * Buffers are Blobs so a slow network's backlog can live outside the JS heap (ClipMe's reason); a part's bytes are
20
+ * read into memory only once it has a connection, so memory stays near `maxParallel × partSize`.
21
+ */
22
+ import { StorageError } from "../errors";
23
+ /** ClipMe's part size: R2's floor for a non-trailing part. */
24
+ export declare const GROWING_PART_BYTES: number;
25
+ export declare const MAX_PARALLEL_PARTS = 3;
26
+ export declare const MAX_ATTEMPTS = 6;
27
+ /** Unsent bytes past which the upload (not the recording) ends: memory is what fails first otherwise. */
28
+ export declare const MAX_PENDING_BYTES: number;
29
+ /**
30
+ * A URL signed ahead is used while younger than this (ClipMe: the server signs for 10 minutes; this leaves room for a
31
+ * slow PUT). The SDK also wants a minute left on its expiresAt, for servers that sign shorter.
32
+ */
33
+ export declare const AHEAD_MAX_AGE_MS: number;
34
+ export interface GrowingPart {
35
+ n: number;
36
+ /** Storage's ETag for the part: its MD5, hex. */
37
+ etag: string;
38
+ size: number;
39
+ }
40
+ export interface SignedPart {
41
+ url: string;
42
+ headers: Record<string, string>;
43
+ expiresAt?: string;
44
+ }
45
+ /** How parts reach storage: the growing upload wires these to the SDK Signer and Transport. */
46
+ export interface PartSender {
47
+ /**
48
+ * A URL for part `n`. With `contentMd5` (base64) and `size` when the bytes are known; with neither when it is signed
49
+ * ahead, before the part exists.
50
+ */
51
+ sign(n: number, o: {
52
+ contentMd5?: string;
53
+ size?: number;
54
+ }, signal: AbortSignal): Promise<SignedPart>;
55
+ /** PUT the part; resolves with storage's ETag (quotes stripped, may be ""). */
56
+ put(signed: SignedPart, body: Uint8Array, n: number, signal: AbortSignal, onProgress: (sent: number) => void): Promise<string>;
57
+ }
58
+ export interface PartHasher {
59
+ /**
60
+ * MD5 of `b`: 16 bytes or 32 hex characters, sync (nodeHasher) or async (hash-wasm's `md5`, which answers hex, fits
61
+ * as it is: `hasher: { md5 }`).
62
+ */
63
+ md5(b: Uint8Array): Uint8Array | string | Promise<Uint8Array | string>;
64
+ }
65
+ /** The hasher's answer as 16 bytes (a malformed one is an error, never a digest). */
66
+ export declare function md5Of(h: PartHasher, b: Uint8Array): Promise<Uint8Array>;
67
+ export interface PartUploaderOptions {
68
+ partSize?: number;
69
+ maxParallel?: number;
70
+ maxAttempts?: number;
71
+ maxPendingBytes?: number;
72
+ /** Parts storage already holds for this multipart (a resume, proven by ListParts): kept when the same bytes come again. */
73
+ seed?: GrowingPart[];
74
+ /** Sign the next part while the current one uploads (ClipMe's tail-latency win). Default true. */
75
+ signAhead?: boolean;
76
+ sleep?: (ms: number) => Promise<void>;
77
+ now?: () => number;
78
+ /** A part stored and proven (`kept`: an earlier run's, proven again by its bytes). */
79
+ onPart?: (part: GrowingPart, acknowledged: number, kept: boolean) => void;
80
+ onFailure?: (error: Error) => void;
81
+ /** Bytes of in-flight PUTs as the transport reports them. */
82
+ onProgress?: () => void;
83
+ }
84
+ /** The multipart upload no longer exists in storage, so its parts are gone too (ClipMe's UploadGoneError). */
85
+ export declare const isGone: (e: unknown) => boolean;
86
+ /**
87
+ * The product's server refused to sign (a quota, a key outside the layouts, a deleted scope…). `retryable` false: the
88
+ * answer is final, so the part isn't tried again (ClipMe tried every failure 6 times).
89
+ */
90
+ export declare class SignRefused extends StorageError {
91
+ }
92
+ /** A part whose stored ETag isn't the MD5 of what was sent (or storage hid the ETag). Sent again. */
93
+ export declare class PartIntegrityError extends Error {
94
+ }
95
+ export declare class GrowingPartUploader {
96
+ private readonly sender;
97
+ private readonly hasher;
98
+ private readonly options;
99
+ readonly partSize: number;
100
+ private buffered;
101
+ private bufferedBytes;
102
+ private nextPartNumber;
103
+ private readonly parts;
104
+ private readonly seed;
105
+ private readonly inFlight;
106
+ private readonly inflightProgress;
107
+ private pendingBytes;
108
+ private slots;
109
+ private readonly waiters;
110
+ private failure;
111
+ private acked;
112
+ private appended;
113
+ private readonly controller;
114
+ private readonly aborted;
115
+ private readonly sleep;
116
+ private readonly now;
117
+ private readonly maxParallel;
118
+ private readonly maxAttempts;
119
+ private readonly maxPendingBytes;
120
+ private ahead;
121
+ constructor(sender: PartSender, hasher: PartHasher, options?: PartUploaderOptions);
122
+ /** Sign part `n` now, for when it is cut. Never throws: a failed signature is asked for again. */
123
+ private signAhead;
124
+ private takeAhead;
125
+ get failed(): Error | null;
126
+ /** Bytes storage holds and the uploader proved (sent now, or kept from an earlier run). */
127
+ get acknowledged(): number;
128
+ /** Bytes handed over so far. */
129
+ get total(): number;
130
+ /** Handed over and not yet acknowledged by storage. */
131
+ get pending(): number;
132
+ /** Bytes of in-flight PUTs the transport has sent (live progress, below `pending`). */
133
+ get sending(): number;
134
+ /** Never throws and never blocks: the recording must not notice that its upload died. */
135
+ add(chunk: Blob): void;
136
+ private send;
137
+ private fail;
138
+ private acquire;
139
+ private release;
140
+ private isFinal;
141
+ private sendWithRetry;
142
+ private record;
143
+ /** For a reader of a finished local copy: wait until the network has caught up a little (ClipMe's `capacity`). */
144
+ capacity(): Promise<void>;
145
+ /** Every PUT started has settled (stored, failed or aborted): nothing in flight can report a part any more. */
146
+ settle(): Promise<void>;
147
+ private drain;
148
+ /**
149
+ * Send the tail and wait for everything. Returns every part, in order.
150
+ *
151
+ * The tail goes out at once, beside the parts still in flight, not after them: it is what Stop is waiting on, and its
152
+ * part number was fixed when it was cut, so the order the parts land in does not matter (ClipMe).
153
+ */
154
+ finish(): Promise<GrowingPart[]>;
155
+ /** Stop now: in-flight requests are aborted and nothing further is sent. */
156
+ abort(): Promise<void>;
157
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Uploading what could not be uploaded live, from the product's local copy.
3
+ *
4
+ * Ported from ClipMe's extension/src/upload/retry.ts (createRetryPump, ClipMe origin/main 4684516). Everything it
5
+ * needs is in the record, because the context that started the upload may be long gone (a killed tab, a reload, a
6
+ * crash): the parts storage holds are proven and skipped, the rest is read again from the local copy. ClipMe's rules
7
+ * are kept: one pump at a time per context; nothing is tried while offline (an attempt that can't work doesn't count);
8
+ * uploads live in this context are left alone; a recording still being written waits; a deleted one is dropped (its
9
+ * multipart aborted); a failure backs off from one minute to fifteen and never gives up unless retrying can't help.
10
+ * ClipMe's product steps (opening its clip, finalize, cursor.json) are the product's: `onResult` is where they go.
11
+ */
12
+ import { type GrowingRecord, type GrowingStore } from "./store";
13
+ import { type GrowingResult, type GrowingUploadOptions } from "./upload";
14
+ export interface GrowingLocalCopy {
15
+ /**
16
+ * Where the local copy stands: "complete" (closed, every byte there: resume it), "recording" (still being written,
17
+ * or waiting for crash recovery to close it: not now), "gone" (deleted from the device, or never had anything:
18
+ * drop the record and abort its multipart). Default "complete".
19
+ */
20
+ state?(rec: GrowingRecord): Promise<"complete" | "recording" | "gone"> | "complete" | "recording" | "gone";
21
+ /**
22
+ * The whole recording from byte 0, in order. It MUST yield the same bytes as were appended live: parts are kept by
23
+ * position (a part whose bytes differ is sent again, so a differently muxed file is never stitched from both).
24
+ */
25
+ read(rec: GrowingRecord): AsyncIterable<Uint8Array | Blob>;
26
+ }
27
+ export interface GrowingRetryOptions extends Omit<GrowingUploadOptions, "key" | "scope" | "id" | "contentType" | "meta" | "partSize" | "store"> {
28
+ store: GrowingStore;
29
+ local: GrowingLocalCopy;
30
+ /** navigator.onLine. An attempt that cannot work should not count as one. */
31
+ isOnline?: () => boolean;
32
+ /** Each attempt's outcome (verified, or failed with the record's new state). */
33
+ onResult?(rec: GrowingRecord, result: GrowingResult): void;
34
+ }
35
+ /** One pump at a time per context; a wake-up that lands mid-run has nothing to add. */
36
+ export declare function createGrowingRetryPump(o: GrowingRetryOptions): {
37
+ run(): Promise<void>;
38
+ };
@@ -0,0 +1,66 @@
1
+ /**
2
+ * What a growing upload keeps between runs (a crash, a reload, a killed tab): one record per upload, under
3
+ * `growing:<id>`. The record is enough to resume: the multipart's upload id, its part size, and every part storage
4
+ * acknowledged (number → MD5 and size). The bytes themselves are the product's local copy (ClipMe: OPFS), which the
5
+ * resumed upload reads again from byte 0; parts proven against ListParts and the re-read bytes are not sent again.
6
+ *
7
+ * Stores are async (IndexedDB, chrome.storage, a server); the engine's synchronous ResumeStore (memoryStore,
8
+ * fileStore) fits too. ClipMe's rule is kept: one key per record, so two contexts never lose each other's writes to a
9
+ * read-modify-write.
10
+ */
11
+ export interface GrowingStore {
12
+ get(key: string): unknown | Promise<unknown>;
13
+ set(key: string, value: unknown): void | Promise<void>;
14
+ del(key: string): void | Promise<void>;
15
+ keys(): string[] | Promise<string[]>;
16
+ }
17
+ /** Where a record stands. */
18
+ export type GrowingRecordState =
19
+ /** Uploading while recording, or the context that did so died. */
20
+ "live"
21
+ /** To be uploaded again from the local copy (a retryable failure). */
22
+ | "queued"
23
+ /** Retrying cannot help (refused by the server, nothing recorded). The local copy is still the product's. */
24
+ | "failed";
25
+ export interface GrowingRecord {
26
+ v: 1;
27
+ id: string;
28
+ scope: string;
29
+ key: string;
30
+ contentType: string;
31
+ partSize: number;
32
+ state: GrowingRecordState;
33
+ createdAt: number;
34
+ uploadId?: string;
35
+ /**
36
+ * When a CreateMultipartUpload was asked for and its upload id isn't recorded yet (RL-10): a run that finds it set
37
+ * aborts what storage opened on the key before creating another.
38
+ */
39
+ creating?: number;
40
+ /** Parts storage acknowledged: number → { etag: MD5 hex, size }. */
41
+ parts: Record<number, {
42
+ etag: string;
43
+ size: number;
44
+ }>;
45
+ /** end() was called: the recording's total length. */
46
+ ended?: number;
47
+ /** Completes storage refused as unstitchable, or that kept failing: at COMPLETE_FAILURES_BEFORE_RESTART a fresh multipart. */
48
+ completeFailures?: number;
49
+ attempts: number;
50
+ nextAttemptAt: number;
51
+ lastError?: string;
52
+ lastCode?: string;
53
+ /** The product's own data (ClipMe: its clip, title, finish metadata). Kept as given. */
54
+ meta?: unknown;
55
+ }
56
+ export declare const RECORD_PREFIX = "growing:";
57
+ export declare const recordKey: (id: string) => string;
58
+ export declare function loadRecord(store: GrowingStore, id: string): Promise<GrowingRecord | null>;
59
+ export declare const saveRecord: (store: GrowingStore, rec: GrowingRecord) => Promise<undefined>;
60
+ export declare const removeRecord: (store: GrowingStore, id: string) => Promise<undefined>;
61
+ /** Every unfinished growing upload in the store (what a retry pump walks). */
62
+ export declare function listRecords(store: GrowingStore): Promise<GrowingRecord[]>;
63
+ /** An in-memory store (tests, or a product that keeps nothing across reloads). Values are copied as JSON. */
64
+ export declare function memoryGrowingStore(): GrowingStore & {
65
+ readonly size: number;
66
+ };
@@ -0,0 +1,126 @@
1
+ /**
2
+ * Upload while recording: one growing object's trip from appended chunks to a verified object in storage.
3
+ *
4
+ * Ported from ClipMe's extension/src/upload/session.ts (LiveUpload, ClipMe origin/main 4684516), keeping its
5
+ * behaviour: `createGrowingUpload` returns at once and opens the multipart in the background, so the recorder is never
6
+ * held up; chunks that arrive before the multipart exists are buffered (and count toward the backlog cap); whatever
7
+ * goes wrong, `append` never throws and the recording carries on: the product's local copy has every byte, so a
8
+ * failure just leaves a record for a resume (`createGrowingRetryPump`, or `createGrowingUpload` again with the same id).
9
+ * ClipMe's product steps (its clip, link, clipboard, poster frames, cursor track, finalize) stay ClipMe's.
10
+ *
11
+ * What the SDK adds is the proof (docs/platform/GROWING-UPLOADS.md):
12
+ * - a resumed multipart is checked against ListParts before anything is kept: a saved part is kept only if storage
13
+ * lists it with the same ETag and size, and the bytes read again at its position hash to that ETag;
14
+ * - `end()` resolves "verified" only after end() was called, every part 1..N is stored with ETag = its MD5, the
15
+ * non-trailing parts are exactly `partSize`, their sizes add up to every byte appended, and storage's answer to the
16
+ * complete (a HEAD) has that size and the composite ETag md5(part MD5s)-N. Anything else is "failed", never done;
17
+ * - the server signs every op (the SDK's Signer protocol), so its `signParts` hook can refuse a recording over budget.
18
+ */
19
+ import { type ErrorAdapter } from "../errors";
20
+ import type { Signer, Transport } from "../protocol";
21
+ import { type PartHasher } from "./parts";
22
+ import { type GrowingStore } from "./store";
23
+ /** Complete refusals after which the multipart is given up and a fresh one is started from the local copy (ClipMe). */
24
+ export declare const COMPLETE_FAILURES_BEFORE_RESTART = 3;
25
+ /** Backoff between resume attempts, from one minute up to fifteen. Never gives up (ClipMe's `nextDelayMs`). */
26
+ export declare function growingRetryDelayMs(attempts: number): number;
27
+ export type GrowingState = "opening" | "sending" | "finishing" | "verified" | "failed" | "cancelled";
28
+ export type GrowingResult = {
29
+ state: "verified";
30
+ id: string;
31
+ key: string;
32
+ size: number;
33
+ etag: string;
34
+ parts: number;
35
+ uploadId: string;
36
+ }
37
+ /** `retryable`: a resume from the local copy can still get it there (the record is kept, state "queued"). */
38
+ | {
39
+ state: "failed";
40
+ id: string;
41
+ key: string;
42
+ error: string;
43
+ code?: string;
44
+ retryable: boolean;
45
+ } | {
46
+ state: "cancelled";
47
+ id: string;
48
+ key: string;
49
+ };
50
+ export interface GrowingProgress {
51
+ /** Bytes appended so far. */
52
+ total: number;
53
+ /** Bytes storage holds and the uploader proved. */
54
+ acknowledged: number;
55
+ /** Appended and not yet acknowledged. */
56
+ pending: number;
57
+ /** Of `pending`, bytes in-flight PUTs have sent. */
58
+ sending: number;
59
+ }
60
+ export interface GrowingUploadOptions {
61
+ signer: Signer;
62
+ transport: Transport;
63
+ /** MD5 of each part: nodeHasher in Node/Electron; in browsers a WebAssembly MD5 (hash-wasm: `{ md5 }`). */
64
+ hasher: PartHasher;
65
+ /** Storage key. Must be a kind the server lets grow: an immutable kind needs its size at create, which a recording doesn't know. */
66
+ key: string;
67
+ /** Signer scope (default "default"). */
68
+ scope?: string;
69
+ /** Stable id (default `${scope}:${key}`). A record saved under it is resumed. */
70
+ id?: string;
71
+ contentType?: string;
72
+ /** Where the record lives between runs (default: none, nothing survives a crash). */
73
+ store?: GrowingStore;
74
+ /** The product's own data, kept on the record (ClipMe: its clip and finish metadata). */
75
+ meta?: unknown;
76
+ /** Part size for a new multipart (default 5 MiB, ClipMe's; at least 5 MiB). A resumed multipart keeps its own. */
77
+ partSize?: number;
78
+ maxParallel?: number;
79
+ maxAttempts?: number;
80
+ maxPendingBytes?: number;
81
+ /** Sign the next part ahead (default true). */
82
+ signAhead?: boolean;
83
+ /** PUT deadline basis passed to the transport (default 60 s). */
84
+ partTimeoutMs?: number;
85
+ /** Ids uploading in this context, which a retry pump must leave alone (ClipMe's `active`). */
86
+ active?: Set<string>;
87
+ /** The product's signer errors → StorageErrors (default duckErrors). */
88
+ errors?: ErrorAdapter;
89
+ sleep?: (ms: number) => Promise<void>;
90
+ now?: () => number;
91
+ }
92
+ export interface GrowingUpload {
93
+ readonly id: string;
94
+ readonly key: string;
95
+ /** Resolves true once the multipart is open (or resumed and proven), false if it could not be. */
96
+ readonly ready: Promise<boolean>;
97
+ readonly state: GrowingState;
98
+ /** A saved record was found and resumed. Known once `ready` resolved. */
99
+ readonly resumed: boolean;
100
+ /** Parts an earlier run stored that this one proved (ListParts + the same bytes) and didn't send again. */
101
+ readonly kept: number;
102
+ /** Why the upload ended early (null while it hasn't). */
103
+ readonly failure: {
104
+ error: string;
105
+ code?: string;
106
+ retryable: boolean;
107
+ } | null;
108
+ /** One chunk, in order. Never throws and never blocks; ignored once the upload has failed or ended. */
109
+ append(chunk: Uint8Array | Blob): void;
110
+ /** For a reader of a finished local copy: resolves when the network has caught up a little. */
111
+ capacity(): Promise<void>;
112
+ /** The recording stopped: send the tail, prove and complete. The same promise on every call. */
113
+ end(): Promise<GrowingResult>;
114
+ /** Stop, abort the multipart and forget the record. */
115
+ abort(): Promise<void>;
116
+ /**
117
+ * Stop sending and keep everything for a resume: the multipart, the parts stored, the record (state "queued", this
118
+ * reason). Resolves once nothing in flight can touch the record any more. For when the local copy can't be read to
119
+ * its end: never call end() then, it would complete a truncated object.
120
+ */
121
+ stop(reason?: string): Promise<void>;
122
+ progress(): GrowingProgress;
123
+ on(ev: "state", fn: (s: GrowingState) => void): () => void;
124
+ on(ev: "progress", fn: (p: GrowingProgress) => void): () => void;
125
+ }
126
+ export declare function createGrowingUpload(o: GrowingUploadOptions): GrowingUpload;
@@ -35,5 +35,6 @@ export { createHttpSigner, SignerError } from "./signer";
35
35
  export type { HttpSignerOptions } from "./signer";
36
36
  export { createTransferEngine, memoryStore, fileStore } from "./client";
37
37
  export type { ResumeStore, UploadRequest, UploadHandle, UploadResult, UploadState, TransferEngineOptions } from "./client";
38
+ export * from "./growing";
38
39
  export { sanitizeLimits, sanitizeCap, resolvePartSize, MAX_PART_BYTES } from "./limits";
39
40
  export type { ConnectionLimits, PartSizeHook } from "./limits";
@@ -21,6 +21,11 @@ export interface Signer {
21
21
  /** Lets the engine take back a PUT that has not sent a byte yet (a borrower lane yielding to the primary lane). */
22
22
  export interface Yieldable {
23
23
  cancelIfQueued?: () => boolean;
24
+ /**
25
+ * The caller gave up on this PUT (a growing upload aborted or failed): stop it, sending no more. Honoured by the
26
+ * fetch transport; the undici and Electron transports ignore it (the PUT runs to its end, its answer unused).
27
+ */
28
+ signal?: AbortSignal;
24
29
  }
25
30
  /**
26
31
  * What storage answered to a PUT that worked: its ETag (quotes stripped, may be "") and the checksum headers it sent
@@ -26,7 +26,7 @@ export type Deduped = {
26
26
  };
27
27
  /**
28
28
  * What the optional modes add to the transfer protocol (fields a server that doesn't know them ignores; the SDK's
29
- * createSignHandler ignores them today, see docs/platform/TRANSFER-GAPS.md):
29
+ * createSignHandler implements the SHA-256 ones, see its `checksums` option and docs/platform/TRANSFER-GAPS.md):
30
30
  * - create: `checksumAlgorithm: "SHA256"` (the multipart is created with SHA-256 COMPOSITE checksums), `partSize`
31
31
  * (the client's proposal), `resumeUploadId` (a multipart this client may continue: its own, or a dead holder's).
32
32
  * - part/put: `checksumSha256` (base64): the server signs `x-amz-checksum-sha256` with it.
@@ -59,8 +59,24 @@ export interface StorageOptions {
59
59
  timing: TransferTiming;
60
60
  /** Turns the signer's own errors into StorageErrors (default: duck-typed status/code/retryable). */
61
61
  errors?: ErrorAdapter;
62
+ /** The GETs of `readBack` (default: the runtime's fetch). */
63
+ fetch?: typeof fetch;
62
64
  }
65
+ /** What `readBack` read: the stored object's size, ETag (the version it was pinned to) and digests of its bytes. */
66
+ export type ReadBack = {
67
+ size: number;
68
+ etag: string | null;
69
+ sha256: string;
70
+ crc32: number;
71
+ };
63
72
  export type Storage = ReturnType<typeof createStorage>;
73
+ /**
74
+ * `proveByReadBack`: an "Exists" / 412 is never settled here by MD5 (ours412); it goes to the engine, which proves the
75
+ * copy by reading it back (EngineCoreOptions.readBack). Without it (other callers), checked by MD5 as before.
76
+ */
77
+ export type WriteMark = {
78
+ proveByReadBack: boolean;
79
+ };
64
80
  /** Unwrap a TransferResult or throw a StorageError the engine knows how to retry. */
65
81
  export declare function unwrap<R extends TransferResult>(r: TransferResult): Extract<R, {
66
82
  ok: true;
@@ -73,7 +89,12 @@ export declare function createStorage(o: StorageOptions): {
73
89
  completeMultipart: (scope: string, key: string, uploadId: string, parts: {
74
90
  n: number;
75
91
  etag: string;
76
- }[]) => Promise<Stat>;
92
+ }[]) => Promise<{
93
+ recovered?: true | undefined;
94
+ size: number | null;
95
+ etag: string | null;
96
+ weak?: boolean;
97
+ }>;
77
98
  abortMultipart: (scope: string, key: string, uploadId: string) => Promise<void | undefined>;
78
99
  head: (scope: string, key: string) => Promise<Stat>;
79
100
  listParts: (scope: string, key: string, uploadId: string) => Promise<Map<number, string>>;
@@ -82,7 +103,7 @@ export declare function createStorage(o: StorageOptions): {
82
103
  initiated?: string;
83
104
  }[]>;
84
105
  uploadPart: (scope: string, key: string, uploadId: string, partNumber: number, body: Uint8Array, contentMd5: string, onProgress: (sent: number) => void, timeoutMs: number, y?: Yieldable) => Promise<string>;
85
- putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: Yieldable, sha256?: string) => Promise<string>;
106
+ putObject: (scope: string, key: string, body: Uint8Array, contentType: string, onProgress?: (sent: number) => void, timeoutMs?: number, contentMd5?: string, counts?: boolean, y?: Yieldable, sha256?: string, w?: WriteMark) => Promise<string>;
86
107
  transport: Transport;
87
108
  createSession: (scope: string, key: string, contentType: string, size: number, counts: boolean, x: {
88
109
  sha256?: string;
@@ -105,6 +126,10 @@ export declare function createStorage(o: StorageOptions): {
105
126
  checksumSha256: string;
106
127
  }[]) => Promise<CheckedStat>;
107
128
  headWithChecksum: (scope: string, key: string) => Promise<CheckedStat>;
129
+ readBack: (scope: string, key: string, want: number, sha256Stream: () => {
130
+ update(b: Uint8Array): unknown;
131
+ digest(): Uint8Array;
132
+ }, crc32: (b: Uint8Array, prev?: number) => number) => Promise<ReadBack | null>;
108
133
  };
109
134
  /** Base64 → hex without Buffer (works in any runtime). */
110
135
  export declare function b64ToHex(b64: string): string;