@byline/core 6.11.0 → 6.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/@types/db-types.d.ts +68 -0
- package/dist/@types/field-data-types.d.ts +17 -0
- package/dist/@types/storage-types.d.ts +109 -1
- package/dist/@types/store-types.d.ts +3 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/lib/errors.d.ts +8 -0
- package/dist/lib/errors.js +8 -0
- package/dist/schemas/zod/builder.js +10 -1
- package/dist/services/field-upload.js +19 -1
- package/dist/services/field-upload.test.node.js +50 -0
- package/dist/storage/content-hash.d.ts +23 -0
- package/dist/storage/content-hash.js +26 -0
- package/dist/storage/content-hash.test.node.d.ts +8 -0
- package/dist/storage/content-hash.test.node.js +67 -0
- package/dist/storage/index.d.ts +1 -0
- package/dist/storage/index.js +1 -0
- package/dist/storage/storage-flatten-reconstruct.test.node.js +59 -0
- package/dist/storage/storage-flatten.js +2 -0
- package/dist/storage/storage-restore.js +13 -1
- package/dist/storage/storage-row-types.d.ts +4 -0
- package/dist/storage/store-manifest.js +10 -0
- package/dist/storage/store-manifest.test.node.js +5 -1
- package/dist/storage/stored-files-backfill.test.node.d.ts +8 -0
- package/dist/storage/stored-files-backfill.test.node.js +251 -0
- package/dist/storage/stored-files-server.d.ts +104 -0
- package/dist/storage/stored-files-server.js +292 -0
- package/dist/storage/stored-files-server.test.node.d.ts +8 -0
- package/dist/storage/stored-files-server.test.node.js +216 -0
- package/package.json +7 -2
|
@@ -3,6 +3,7 @@ import type { CollectionDefinition } from '@byline/core';
|
|
|
3
3
|
import type { DbErrorClassification } from '../lib/errors.js';
|
|
4
4
|
import type { ISchedulerStore } from '../scheduler/types.js';
|
|
5
5
|
import type { DocumentRevisionStore } from './document-revision.js';
|
|
6
|
+
import type { StorageValidator } from './storage-types.js';
|
|
6
7
|
/**
|
|
7
8
|
* Read mode for document queries.
|
|
8
9
|
*
|
|
@@ -245,6 +246,11 @@ export interface IDbAdapter {
|
|
|
245
246
|
audit: IAuditCommands;
|
|
246
247
|
/** Singleton slot → document mapping writes. */
|
|
247
248
|
singletons: ISingletonCommands;
|
|
249
|
+
/**
|
|
250
|
+
* Stored-file identity maintenance (backfill). Optional: an adapter
|
|
251
|
+
* without it cannot run `backfillStoredFileIdentity`.
|
|
252
|
+
*/
|
|
253
|
+
storedFiles?: IStoredFileCommands;
|
|
248
254
|
};
|
|
249
255
|
queries: {
|
|
250
256
|
collections: ICollectionQueries;
|
|
@@ -253,6 +259,8 @@ export interface IDbAdapter {
|
|
|
253
259
|
audit: IAuditQueries;
|
|
254
260
|
/** Singleton slot → document mapping reads. */
|
|
255
261
|
singletons: ISingletonQueries;
|
|
262
|
+
/** Stored-file identity maintenance reads (backfill). Optional. */
|
|
263
|
+
storedFiles?: IStoredFileQueries;
|
|
256
264
|
};
|
|
257
265
|
/**
|
|
258
266
|
* Run `fn` inside a single database transaction so the
|
|
@@ -549,6 +557,66 @@ export interface ISingletonQueries {
|
|
|
549
557
|
*/
|
|
550
558
|
getMappedDocumentId(collectionId: string): Promise<string | null>;
|
|
551
559
|
}
|
|
560
|
+
/**
|
|
561
|
+
* Selection for `IStoredFileQueries.listStoredFilePaths`. `limit` counts
|
|
562
|
+
* distinct storage paths, never rows or hash groups, so one path's groups are
|
|
563
|
+
* never split across pages.
|
|
564
|
+
*/
|
|
565
|
+
export interface ListStoredFilePathsParams {
|
|
566
|
+
/** Only rows written by this provider (`byline_store_file.storage_provider`). */
|
|
567
|
+
storageProvider: string;
|
|
568
|
+
/** Only rows of this collection, when set. */
|
|
569
|
+
collectionId?: string;
|
|
570
|
+
/** `'missing'`: rows with no content hash. `'present'`: rows with one. */
|
|
571
|
+
identity: 'missing' | 'present';
|
|
572
|
+
/** Keyset cursor: only paths ordered strictly after this one. */
|
|
573
|
+
afterPath?: string;
|
|
574
|
+
/** Maximum number of distinct paths to return. */
|
|
575
|
+
limit: number;
|
|
576
|
+
}
|
|
577
|
+
/** One storage path and, for `identity: 'present'`, every recorded hash on it. */
|
|
578
|
+
export interface StoredFilePathGroup {
|
|
579
|
+
storagePath: string;
|
|
580
|
+
/**
|
|
581
|
+
* Distinct recorded hashes on this path with the number of rows carrying
|
|
582
|
+
* each. Empty for `identity: 'missing'`. More than one entry means older
|
|
583
|
+
* versions recorded bytes that were later replaced at the same path.
|
|
584
|
+
*/
|
|
585
|
+
hashes: Array<{
|
|
586
|
+
fileHash: string;
|
|
587
|
+
rowCount: number;
|
|
588
|
+
}>;
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* Maintenance reads over `byline_store_file` for stored-file identity
|
|
592
|
+
* backfill and verification. Direct storage primitives: they bypass
|
|
593
|
+
* collection auth and hooks, like every `db.queries.*` call.
|
|
594
|
+
*/
|
|
595
|
+
export interface IStoredFileQueries {
|
|
596
|
+
/**
|
|
597
|
+
* Distinct storage paths in ascending byte order. Paths, provider names and
|
|
598
|
+
* digests compare exactly — bytes, not collation: paths that differ only in
|
|
599
|
+
* case, accents or trailing spaces are distinct, and `'LOCAL'` is not
|
|
600
|
+
* `'local'`. Adapters must not apply case folding or space padding.
|
|
601
|
+
*/
|
|
602
|
+
listStoredFilePaths(params: ListStoredFilePathsParams): Promise<StoredFilePathGroup[]>;
|
|
603
|
+
}
|
|
604
|
+
/** Maintenance writes over `byline_store_file` for stored-file identity backfill. */
|
|
605
|
+
export interface IStoredFileCommands {
|
|
606
|
+
/**
|
|
607
|
+
* Record `fileHash` and `storageValidator` on every row for this provider
|
|
608
|
+
* and path (and collection, when set) whose `file_hash` is still NULL.
|
|
609
|
+
* Never overwrites a recorded hash, so repeating it is harmless. Returns the
|
|
610
|
+
* number of rows updated.
|
|
611
|
+
*/
|
|
612
|
+
setIdentityForPath(params: {
|
|
613
|
+
storageProvider: string;
|
|
614
|
+
storagePath: string;
|
|
615
|
+
collectionId?: string;
|
|
616
|
+
fileHash: string;
|
|
617
|
+
storageValidator: StorageValidator;
|
|
618
|
+
}): Promise<number>;
|
|
619
|
+
}
|
|
552
620
|
/** Locked structural state returned by tree mutation commands. */
|
|
553
621
|
export interface TreePlacementState {
|
|
554
622
|
placed: boolean;
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import type { ArrayField, BlocksField, Field, FieldSet, GroupField, LocalizedField, OptionalField, RelationField, SelectField } from './field-types.js';
|
|
9
9
|
import type { RelatedDocumentValue } from './relation-types.js';
|
|
10
|
+
import type { StorageValidator } from './storage-types.js';
|
|
10
11
|
import type { Prettify, ValueUnion } from './type-utils.js';
|
|
11
12
|
/**
|
|
12
13
|
* Structural type for any JSON-serializable value. Used as the type for
|
|
@@ -116,7 +117,23 @@ export interface StoredFileValue {
|
|
|
116
117
|
storageProvider: string;
|
|
117
118
|
storagePath: string;
|
|
118
119
|
storageUrl?: string;
|
|
120
|
+
/**
|
|
121
|
+
* Content hash of the original object: SHA-256 of the bytes Byline passed
|
|
122
|
+
* to `IStorageProvider.upload()`, which providers store verbatim, as 64
|
|
123
|
+
* lower-case hexadecimal characters. Image variants are not covered.
|
|
124
|
+
*
|
|
125
|
+
* Recorded once, at upload or backfill time, and describes the bytes
|
|
126
|
+
* observed then; `storageValidator` is how a consumer detects that the
|
|
127
|
+
* object has since changed. Absent (never an empty string) when unknown — a
|
|
128
|
+
* consumer that requires identity must treat absence as an error.
|
|
129
|
+
*/
|
|
119
130
|
fileHash?: string;
|
|
131
|
+
/**
|
|
132
|
+
* Validator of the stored version that `fileHash` was computed from, when
|
|
133
|
+
* the provider issued one. Never compared with `fileHash`; see
|
|
134
|
+
* `StorageValidator`.
|
|
135
|
+
*/
|
|
136
|
+
storageValidator?: StorageValidator;
|
|
120
137
|
imageWidth?: number;
|
|
121
138
|
imageHeight?: number;
|
|
122
139
|
imageFormat?: string;
|
|
@@ -21,7 +21,75 @@ export interface StoredFileLocation {
|
|
|
21
21
|
* signed S3 URLs generated at read-time).
|
|
22
22
|
*/
|
|
23
23
|
storageUrl: string | null;
|
|
24
|
+
/**
|
|
25
|
+
* Validator of the stored version of the object, when the provider issues
|
|
26
|
+
* one. Absent when the provider has no usable validator for this write.
|
|
27
|
+
* See {@link StorageValidator}.
|
|
28
|
+
*/
|
|
29
|
+
storageValidator?: StorageValidator;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* An opaque token a storage provider issues for one stored version of an
|
|
33
|
+
* object — an S3 version id or ETag, or a local file's device, inode,
|
|
34
|
+
* modification time and size.
|
|
35
|
+
*
|
|
36
|
+
* A validator identifies a *stored version*, not content. Its only use is
|
|
37
|
+
* equality: "is the object at this path still the version I recorded?"
|
|
38
|
+
* Two validators match only when both `kind` and `value` are equal; a
|
|
39
|
+
* validator of a different `kind` is unknown, never "changed".
|
|
40
|
+
*
|
|
41
|
+
* A validator is never compared with `StoredFileValue.fileHash`. An ETag is
|
|
42
|
+
* not a content digest: multipart uploads and server-side encryption give
|
|
43
|
+
* ETags that are not the MD5 of the object.
|
|
44
|
+
*
|
|
45
|
+
* `kind` is 1–32 characters; `value` is 1–1024 characters.
|
|
46
|
+
*/
|
|
47
|
+
export interface StorageValidator {
|
|
48
|
+
kind: string;
|
|
49
|
+
value: string;
|
|
50
|
+
}
|
|
51
|
+
/** Result of a bound read — see `IStorageProvider.download`. */
|
|
52
|
+
export interface StoredObjectRead {
|
|
53
|
+
/** The object's bytes, streamed. */
|
|
54
|
+
body: NodeJS.ReadableStream;
|
|
55
|
+
/**
|
|
56
|
+
* Validator of exactly the bytes in `body`, taken from the same request or
|
|
57
|
+
* the same open file handle. This is the provider's canonical validator and
|
|
58
|
+
* may be of a different kind from a `download` `expect`.
|
|
59
|
+
*/
|
|
60
|
+
validator: StorageValidator;
|
|
61
|
+
/** Object size in bytes, when the provider reports it. */
|
|
62
|
+
size?: number;
|
|
63
|
+
}
|
|
64
|
+
/** Options for `IStorageProvider.download`. */
|
|
65
|
+
export interface StorageDownloadOptions {
|
|
66
|
+
/**
|
|
67
|
+
* Fail-closed precondition. The provider exposes the body only when it has
|
|
68
|
+
* established that the current object carries a token of `expect.kind`
|
|
69
|
+
* equal to `expect.value`; otherwise it rejects with
|
|
70
|
+
* `ERR_STORAGE_PRECONDITION` and exposes nothing.
|
|
71
|
+
*/
|
|
72
|
+
expect?: StorageValidator;
|
|
73
|
+
/** Abort the read. The provider disposes of any open body or handle. */
|
|
74
|
+
signal?: AbortSignal;
|
|
75
|
+
}
|
|
76
|
+
/** Result of `IStorageProvider.head`. */
|
|
77
|
+
export interface StoredObjectHead {
|
|
78
|
+
validator: StorageValidator;
|
|
79
|
+
size?: number;
|
|
24
80
|
}
|
|
81
|
+
/**
|
|
82
|
+
* Reasons carried in `details.reason` of an `ERR_STORAGE_PRECONDITION` error.
|
|
83
|
+
*
|
|
84
|
+
* - `changed` — the provider established that the current object's token of
|
|
85
|
+
* the expected kind differs.
|
|
86
|
+
* - `unsupported-kind` — the provider does not issue validators of the
|
|
87
|
+
* expected kind. Checked before any request.
|
|
88
|
+
* - `unestablishable` — the kind is supported, but the response lacks a usable
|
|
89
|
+
* token of that kind.
|
|
90
|
+
* - `missing` — no object exists at the path.
|
|
91
|
+
*/
|
|
92
|
+
export type StoragePreconditionReason = 'changed' | 'unsupported-kind' | 'unestablishable' | 'missing';
|
|
25
93
|
/**
|
|
26
94
|
* Options passed to `IStorageProvider.upload()` describing the file being
|
|
27
95
|
* stored.
|
|
@@ -93,7 +161,13 @@ export interface IStorageProvider {
|
|
|
93
161
|
readonly providerName: string;
|
|
94
162
|
/**
|
|
95
163
|
* Store a file. The `stream` may be a Node.js `Readable` or a raw `Buffer`.
|
|
96
|
-
* Returns the location metadata to be recorded alongside the file entry
|
|
164
|
+
* Returns the location metadata to be recorded alongside the file entry,
|
|
165
|
+
* including the stored version's validator when the provider issues one.
|
|
166
|
+
*
|
|
167
|
+
* Providers must store the bytes verbatim: no compression, transcoding or
|
|
168
|
+
* other transform. Byline's content hash (`StoredFileValue.fileHash`) is
|
|
169
|
+
* computed over the bytes passed here and relies on them being the bytes
|
|
170
|
+
* stored. A stream must be consumed to its end before the promise resolves.
|
|
97
171
|
*/
|
|
98
172
|
upload(stream: NodeJS.ReadableStream | Buffer, options: UploadFileOptions): Promise<StoredFileLocation>;
|
|
99
173
|
/**
|
|
@@ -131,4 +205,38 @@ export interface IStorageProvider {
|
|
|
131
205
|
* **Optional capability** — see `move`.
|
|
132
206
|
*/
|
|
133
207
|
exists?(storagePath: string): Promise<boolean>;
|
|
208
|
+
/**
|
|
209
|
+
* Bound read: stream the object and return the validator of exactly the
|
|
210
|
+
* bytes streamed, taken from the same request or the same open file
|
|
211
|
+
* handle. The returned validator is the provider's canonical one and may be
|
|
212
|
+
* of a different kind from `expect`.
|
|
213
|
+
*
|
|
214
|
+
* Without `expect`: resolves `null` when no object exists at the path, and
|
|
215
|
+
* fails with `ERR_STORAGE` (`details.reason: 'validator-unavailable'`) when
|
|
216
|
+
* the object has no usable validator. A bound read never returns bytes
|
|
217
|
+
* without a validator.
|
|
218
|
+
*
|
|
219
|
+
* With `expect`: exposes the body only when a token of `expect.kind` equal
|
|
220
|
+
* to `expect.value` is established; otherwise fails with
|
|
221
|
+
* `ERR_STORAGE_PRECONDITION` (see {@link StoragePreconditionReason}) and
|
|
222
|
+
* exposes nothing.
|
|
223
|
+
*
|
|
224
|
+
* On every rejection after a response or handle exists, the provider
|
|
225
|
+
* disposes of it before throwing.
|
|
226
|
+
*
|
|
227
|
+
* **Optional capability** — see `move`.
|
|
228
|
+
*/
|
|
229
|
+
download?(storagePath: string, options?: StorageDownloadOptions): Promise<StoredObjectRead | null>;
|
|
230
|
+
/**
|
|
231
|
+
* Cheap change probe: the current object's validator and size. Resolves
|
|
232
|
+
* `null` when no object exists; fails with `ERR_STORAGE`
|
|
233
|
+
* (`validator-unavailable`) for a present object without a usable
|
|
234
|
+
* validator.
|
|
235
|
+
*
|
|
236
|
+
* Not a provenance source — the object can change after `head` returns.
|
|
237
|
+
* Use `download` when bytes must be paired with their validator.
|
|
238
|
+
*
|
|
239
|
+
* **Optional capability** — see `move`.
|
|
240
|
+
*/
|
|
241
|
+
head?(storagePath: string): Promise<StoredObjectHead | null>;
|
|
134
242
|
}
|
|
@@ -58,6 +58,9 @@ export interface FileStore extends BaseStore {
|
|
|
58
58
|
storage_path: string;
|
|
59
59
|
storage_url?: string;
|
|
60
60
|
file_hash?: string;
|
|
61
|
+
/** Storage validator, persisted as a pair; both null when unknown. */
|
|
62
|
+
storage_validator_kind?: string;
|
|
63
|
+
storage_validator_value?: string;
|
|
61
64
|
image_width?: number;
|
|
62
65
|
image_height?: number;
|
|
63
66
|
image_format?: string;
|
package/dist/index.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export { resolvePasskeyConfig } from './config/validate-passkey-config.js';
|
|
|
9
9
|
export { type BylineCore, getBylineCore, initBylineCore } from './core.js';
|
|
10
10
|
export * from './defaults/default-values.js';
|
|
11
11
|
export { getHostRequestBridge, type HostCookieSetOptions, type HostRequestBridge, registerHostRequestBridge, tryGetHostRequestBridge, } from './host/host-request-bridge.js';
|
|
12
|
-
export { BylineError, type DbErrorClassification, type DbErrorCode, DbErrorCodes, ERR_AUDIT_UNSUPPORTED, ERR_CONFLICT, ERR_DATABASE, ERR_DOCUMENT_HOOK_COMMITTED, ERR_DOCUMENT_STALE, ERR_LOCK_CONFLICT, ERR_NOT_FOUND, ERR_READ_BUDGET_EXCEEDED, ERR_READ_RECURSION, ERR_STORAGE, ERR_TREE_HOOK_COMMITTED, ERR_UNHANDLED, ERR_VALIDATION, ErrorCodes, type ErrorReport, getLockConflictDetails, TREE_HOOK_COMMITTED_MARKER, TREE_PLACEMENT_STALE_MARKER, } from './lib/errors.js';
|
|
12
|
+
export { BylineError, type DbErrorClassification, type DbErrorCode, DbErrorCodes, ERR_AUDIT_UNSUPPORTED, ERR_CONFLICT, ERR_DATABASE, ERR_DOCUMENT_HOOK_COMMITTED, ERR_DOCUMENT_STALE, ERR_LOCK_CONFLICT, ERR_NOT_FOUND, ERR_READ_BUDGET_EXCEEDED, ERR_READ_RECURSION, ERR_STORAGE, ERR_STORAGE_PRECONDITION, ERR_TREE_HOOK_COMMITTED, ERR_UNHANDLED, ERR_VALIDATION, ErrorCodes, type ErrorReport, getLockConflictDetails, TREE_HOOK_COMMITTED_MARKER, TREE_PLACEMENT_STALE_MARKER, } from './lib/errors.js';
|
|
13
13
|
export { generateKeyBetween, generateNKeysBetween, validateOrderKey, } from './lib/fractional-index.js';
|
|
14
14
|
export { type BylineLogger, getLogger } from './lib/logger.js';
|
|
15
15
|
export { AsyncRegistry, type RegisteredServices, Registry } from './lib/registry.js';
|
package/dist/index.js
CHANGED
|
@@ -21,7 +21,7 @@ export { resolvePasskeyConfig } from './config/validate-passkey-config.js';
|
|
|
21
21
|
export { getBylineCore, initBylineCore } from './core.js';
|
|
22
22
|
export * from './defaults/default-values.js';
|
|
23
23
|
export { getHostRequestBridge, registerHostRequestBridge, tryGetHostRequestBridge, } from './host/host-request-bridge.js';
|
|
24
|
-
export { BylineError, DbErrorCodes, ERR_AUDIT_UNSUPPORTED, ERR_CONFLICT, ERR_DATABASE, ERR_DOCUMENT_HOOK_COMMITTED, ERR_DOCUMENT_STALE, ERR_LOCK_CONFLICT, ERR_NOT_FOUND, ERR_READ_BUDGET_EXCEEDED, ERR_READ_RECURSION, ERR_STORAGE, ERR_TREE_HOOK_COMMITTED, ERR_UNHANDLED, ERR_VALIDATION, ErrorCodes, getLockConflictDetails, TREE_HOOK_COMMITTED_MARKER, TREE_PLACEMENT_STALE_MARKER, } from './lib/errors.js';
|
|
24
|
+
export { BylineError, DbErrorCodes, ERR_AUDIT_UNSUPPORTED, ERR_CONFLICT, ERR_DATABASE, ERR_DOCUMENT_HOOK_COMMITTED, ERR_DOCUMENT_STALE, ERR_LOCK_CONFLICT, ERR_NOT_FOUND, ERR_READ_BUDGET_EXCEEDED, ERR_READ_RECURSION, ERR_STORAGE, ERR_STORAGE_PRECONDITION, ERR_TREE_HOOK_COMMITTED, ERR_UNHANDLED, ERR_VALIDATION, ErrorCodes, getLockConflictDetails, TREE_HOOK_COMMITTED_MARKER, TREE_PLACEMENT_STALE_MARKER, } from './lib/errors.js';
|
|
25
25
|
export { generateKeyBetween, generateNKeysBetween, validateOrderKey, } from './lib/fractional-index.js';
|
|
26
26
|
export { getLogger } from './lib/logger.js';
|
|
27
27
|
export { AsyncRegistry, Registry } from './lib/registry.js';
|
package/dist/lib/errors.d.ts
CHANGED
|
@@ -86,6 +86,7 @@ export declare const ErrorCodes: {
|
|
|
86
86
|
readonly DATABASE: 'ERR_DATABASE';
|
|
87
87
|
readonly LOCK_CONFLICT: 'ERR_LOCK_CONFLICT';
|
|
88
88
|
readonly STORAGE: 'ERR_STORAGE';
|
|
89
|
+
readonly STORAGE_PRECONDITION: 'ERR_STORAGE_PRECONDITION';
|
|
89
90
|
readonly READ_BUDGET_EXCEEDED: 'ERR_READ_BUDGET_EXCEEDED';
|
|
90
91
|
readonly READ_RECURSION: 'ERR_READ_RECURSION';
|
|
91
92
|
readonly PATH_CONFLICT: 'ERR_PATH_CONFLICT';
|
|
@@ -119,6 +120,13 @@ export declare function getLockConflictDetails(error: unknown): {
|
|
|
119
120
|
retryable: true;
|
|
120
121
|
} | null;
|
|
121
122
|
export declare const ERR_STORAGE: (opts: BylineErrorOptions, errorConstructor?: any) => BylineError;
|
|
123
|
+
/**
|
|
124
|
+
* A storage read precondition (`IStorageProvider.download` `expect`) was not
|
|
125
|
+
* established. `details.reason` is a `StoragePreconditionReason`: a proven
|
|
126
|
+
* change (`changed`) is distinct from an expectation that could not be checked
|
|
127
|
+
* (`unsupported-kind`, `unestablishable`) and from a missing object.
|
|
128
|
+
*/
|
|
129
|
+
export declare const ERR_STORAGE_PRECONDITION: (opts: BylineErrorOptions, errorConstructor?: any) => BylineError;
|
|
122
130
|
export declare const ERR_READ_BUDGET_EXCEEDED: (opts: BylineErrorOptions, errorConstructor?: any) => BylineError;
|
|
123
131
|
export declare const ERR_READ_RECURSION: (opts: BylineErrorOptions, errorConstructor?: any) => BylineError;
|
|
124
132
|
/**
|
package/dist/lib/errors.js
CHANGED
|
@@ -119,6 +119,7 @@ export const ErrorCodes = {
|
|
|
119
119
|
DATABASE: 'ERR_DATABASE',
|
|
120
120
|
LOCK_CONFLICT: 'ERR_LOCK_CONFLICT',
|
|
121
121
|
STORAGE: 'ERR_STORAGE',
|
|
122
|
+
STORAGE_PRECONDITION: 'ERR_STORAGE_PRECONDITION',
|
|
122
123
|
READ_BUDGET_EXCEEDED: 'ERR_READ_BUDGET_EXCEEDED',
|
|
123
124
|
READ_RECURSION: 'ERR_READ_RECURSION',
|
|
124
125
|
PATH_CONFLICT: 'ERR_PATH_CONFLICT',
|
|
@@ -173,6 +174,13 @@ export function getLockConflictDetails(error) {
|
|
|
173
174
|
}
|
|
174
175
|
}
|
|
175
176
|
export const ERR_STORAGE = createErrorType(ErrorCodes.STORAGE);
|
|
177
|
+
/**
|
|
178
|
+
* A storage read precondition (`IStorageProvider.download` `expect`) was not
|
|
179
|
+
* established. `details.reason` is a `StoragePreconditionReason`: a proven
|
|
180
|
+
* change (`changed`) is distinct from an expectation that could not be checked
|
|
181
|
+
* (`unsupported-kind`, `unestablishable`) and from a missing object.
|
|
182
|
+
*/
|
|
183
|
+
export const ERR_STORAGE_PRECONDITION = createErrorType(ErrorCodes.STORAGE_PRECONDITION, 'warn');
|
|
176
184
|
export const ERR_READ_BUDGET_EXCEEDED = createErrorType(ErrorCodes.READ_BUDGET_EXCEEDED);
|
|
177
185
|
export const ERR_READ_RECURSION = createErrorType(ErrorCodes.READ_RECURSION);
|
|
178
186
|
/**
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as z from 'zod';
|
|
2
2
|
import { REQUIRED_WORKFLOW_STATUSES } from '../../@types/index.js';
|
|
3
3
|
import { getCollectionDefinition } from '../../config/config.js';
|
|
4
|
+
import { CONTENT_HASH_PATTERN, STORAGE_VALIDATOR_LIMITS } from '../../storage/content-hash.js';
|
|
4
5
|
// Helper function to apply custom validation rules
|
|
5
6
|
const applyValidationRules = (schema, rules) => {
|
|
6
7
|
return rules.reduce((acc, rule) => {
|
|
@@ -170,7 +171,15 @@ export const fieldToZodSchema = (field, strict = true) => {
|
|
|
170
171
|
storageProvider: z.string(),
|
|
171
172
|
storagePath: z.string(),
|
|
172
173
|
storageUrl: z.string().nullable().optional(),
|
|
173
|
-
|
|
174
|
+
// Content identity: absent when unknown, never an empty string.
|
|
175
|
+
fileHash: z.string().regex(CONTENT_HASH_PATTERN).nullable().optional(),
|
|
176
|
+
storageValidator: z
|
|
177
|
+
.object({
|
|
178
|
+
kind: z.string().min(1).max(STORAGE_VALIDATOR_LIMITS.kind),
|
|
179
|
+
value: z.string().min(1).max(STORAGE_VALIDATOR_LIMITS.value),
|
|
180
|
+
})
|
|
181
|
+
.nullable()
|
|
182
|
+
.optional(),
|
|
174
183
|
imageWidth: z.number().nullable().optional(),
|
|
175
184
|
imageHeight: z.number().nullable().optional(),
|
|
176
185
|
imageFormat: z.string().nullable().optional(),
|
|
@@ -9,6 +9,7 @@ import { isSingleton, resolveUploadHooks } from '../@types/index.js';
|
|
|
9
9
|
import { assertActorCanPerform } from '../auth/assert-actor-can-perform.js';
|
|
10
10
|
import { ERR_DATABASE, ERR_STORAGE, ERR_VALIDATION } from '../lib/errors.js';
|
|
11
11
|
import { withLogContext } from '../lib/logger.js';
|
|
12
|
+
import { sha256Hex } from '../storage/content-hash.js';
|
|
12
13
|
import { resolveUploadFilename } from '../utils/slugify-filename.js';
|
|
13
14
|
import { createDocument, getDocumentHookCommittedDetails, } from './document-lifecycle/index.js';
|
|
14
15
|
function isMimeTypeAllowed(mimeType, allowedTypes) {
|
|
@@ -220,6 +221,22 @@ export async function uploadField(ctx, params) {
|
|
|
220
221
|
// `targetStoragePath`, which providers write verbatim (no UUID
|
|
221
222
|
// prefix / key derivation) — variants still derive sibling paths
|
|
222
223
|
// from the resulting `storedFile.storagePath`.
|
|
224
|
+
// -- Content identity. The upload arrives as an in-memory buffer, so
|
|
225
|
+
// hashing it reads no bytes from storage. Providers store bytes
|
|
226
|
+
// verbatim, so this is also the hash of the stored original.
|
|
227
|
+
if (buffer.length !== fileSize) {
|
|
228
|
+
throw ERR_VALIDATION({
|
|
229
|
+
message: `Upload is ${buffer.length} bytes but declared ${fileSize} bytes.`,
|
|
230
|
+
details: {
|
|
231
|
+
collectionPath,
|
|
232
|
+
fieldName,
|
|
233
|
+
reason: 'size-mismatch',
|
|
234
|
+
actualSize: buffer.length,
|
|
235
|
+
declaredSize: fileSize,
|
|
236
|
+
},
|
|
237
|
+
}, uploadField).log(logger);
|
|
238
|
+
}
|
|
239
|
+
const fileHash = await sha256Hex(buffer);
|
|
223
240
|
let storedFile;
|
|
224
241
|
try {
|
|
225
242
|
storedFile = await storage.upload(buffer, {
|
|
@@ -284,7 +301,8 @@ export async function uploadField(ctx, params) {
|
|
|
284
301
|
storageProvider: storedFile.storageProvider,
|
|
285
302
|
storagePath: storedFile.storagePath,
|
|
286
303
|
storageUrl: storedFile.storageUrl ?? undefined,
|
|
287
|
-
fileHash
|
|
304
|
+
fileHash,
|
|
305
|
+
...(storedFile.storageValidator && { storageValidator: storedFile.storageValidator }),
|
|
288
306
|
imageWidth: imageMeta.width ?? undefined,
|
|
289
307
|
imageHeight: imageMeta.height ?? undefined,
|
|
290
308
|
imageFormat: imageMeta.format ?? undefined,
|
|
@@ -656,3 +656,53 @@ describe('uploadField service', () => {
|
|
|
656
656
|
expect(del).toHaveBeenNthCalledWith(3, 'media/card.webp');
|
|
657
657
|
});
|
|
658
658
|
});
|
|
659
|
+
describe('uploadField content identity', () => {
|
|
660
|
+
it('records the SHA-256 of the uploaded bytes and the provider validator', async () => {
|
|
661
|
+
const { ctx, upload } = buildCtx();
|
|
662
|
+
upload.mockResolvedValueOnce({
|
|
663
|
+
storageProvider: 'local',
|
|
664
|
+
storagePath: 'media/original.png',
|
|
665
|
+
storageUrl: '/uploads/media/original.png',
|
|
666
|
+
storageValidator: { kind: 'local-stat', value: '1:2:3:3' },
|
|
667
|
+
});
|
|
668
|
+
const { createHash } = await import('node:crypto');
|
|
669
|
+
const expected = createHash('sha256').update(Buffer.from('png')).digest('hex');
|
|
670
|
+
const result = await uploadField(ctx, {
|
|
671
|
+
buffer: Buffer.from('png'),
|
|
672
|
+
originalFilename: 'hero.png',
|
|
673
|
+
mimeType: 'image/png',
|
|
674
|
+
fileSize: 3,
|
|
675
|
+
shouldCreateDocument: false,
|
|
676
|
+
});
|
|
677
|
+
expect(result.storedFile.fileHash).toBe(expected);
|
|
678
|
+
expect(result.storedFile.fileHash).toMatch(/^[0-9a-f]{64}$/);
|
|
679
|
+
expect(result.storedFile.storageValidator).toEqual({ kind: 'local-stat', value: '1:2:3:3' });
|
|
680
|
+
});
|
|
681
|
+
it('omits storageValidator when the provider issues none, but still records the hash', async () => {
|
|
682
|
+
const { ctx } = buildCtx();
|
|
683
|
+
const result = await uploadField(ctx, {
|
|
684
|
+
buffer: Buffer.from('png'),
|
|
685
|
+
originalFilename: 'hero.png',
|
|
686
|
+
mimeType: 'image/png',
|
|
687
|
+
fileSize: 3,
|
|
688
|
+
shouldCreateDocument: false,
|
|
689
|
+
});
|
|
690
|
+
expect(result.storedFile.fileHash).toMatch(/^[0-9a-f]{64}$/);
|
|
691
|
+
expect('storageValidator' in result.storedFile).toBe(false);
|
|
692
|
+
});
|
|
693
|
+
it('rejects a buffer whose length differs from the declared size before writing bytes', async () => {
|
|
694
|
+
const { ctx, upload, storedPaths } = buildCtx();
|
|
695
|
+
await expect(uploadField(ctx, {
|
|
696
|
+
buffer: Buffer.from('png'),
|
|
697
|
+
originalFilename: 'hero.png',
|
|
698
|
+
mimeType: 'image/png',
|
|
699
|
+
fileSize: 4,
|
|
700
|
+
shouldCreateDocument: false,
|
|
701
|
+
})).rejects.toMatchObject({
|
|
702
|
+
code: ErrorCodes.VALIDATION,
|
|
703
|
+
details: expect.objectContaining({ reason: 'size-mismatch', actualSize: 3, declaredSize: 4 }),
|
|
704
|
+
});
|
|
705
|
+
expect(upload).not.toHaveBeenCalled();
|
|
706
|
+
expect(storedPaths.size).toBe(0);
|
|
707
|
+
});
|
|
708
|
+
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import type { StorageValidator } from '../@types/storage-types.js';
|
|
9
|
+
/** A `StoredFileValue.fileHash`: SHA-256 as 64 lower-case hexadecimal characters. */
|
|
10
|
+
export declare const CONTENT_HASH_PATTERN: RegExp;
|
|
11
|
+
/** Upper bounds for `StorageValidator` members. */
|
|
12
|
+
export declare const STORAGE_VALIDATOR_LIMITS: {
|
|
13
|
+
readonly kind: 32;
|
|
14
|
+
readonly value: 1024;
|
|
15
|
+
};
|
|
16
|
+
/** SHA-256 of an in-memory byte sequence, as lower-case hex. */
|
|
17
|
+
export declare function sha256Hex(bytes: Uint8Array): Promise<string>;
|
|
18
|
+
/**
|
|
19
|
+
* Exact validator equality: both `kind` and `value` must match. Validators of
|
|
20
|
+
* different kinds are not comparable, so this returns `false` for them — the
|
|
21
|
+
* caller must treat that as "unknown", not "changed".
|
|
22
|
+
*/
|
|
23
|
+
export declare function isSameStorageValidator(a: StorageValidator, b: StorageValidator): boolean;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
/** A `StoredFileValue.fileHash`: SHA-256 as 64 lower-case hexadecimal characters. */
|
|
9
|
+
export const CONTENT_HASH_PATTERN = /^[0-9a-f]{64}$/;
|
|
10
|
+
/** Upper bounds for `StorageValidator` members. */
|
|
11
|
+
export const STORAGE_VALIDATOR_LIMITS = { kind: 32, value: 1024 };
|
|
12
|
+
/** SHA-256 of an in-memory byte sequence, as lower-case hex. */
|
|
13
|
+
export async function sha256Hex(bytes) {
|
|
14
|
+
// Copy into a plain ArrayBuffer view: `digest` rejects SharedArrayBuffer-backed
|
|
15
|
+
// views, and a Node Buffer may be a slice of a larger pooled allocation.
|
|
16
|
+
const digest = await globalThis.crypto.subtle.digest('SHA-256', new Uint8Array(bytes));
|
|
17
|
+
return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, '0')).join('');
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Exact validator equality: both `kind` and `value` must match. Validators of
|
|
21
|
+
* different kinds are not comparable, so this returns `false` for them — the
|
|
22
|
+
* caller must treat that as "unknown", not "changed".
|
|
23
|
+
*/
|
|
24
|
+
export function isSameStorageValidator(a, b) {
|
|
25
|
+
return a.kind === b.kind && a.value === b.value;
|
|
26
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { describe, expect, it } from 'vitest';
|
|
9
|
+
import { fieldToZodSchema } from '../schemas/zod/builder.js';
|
|
10
|
+
import { CONTENT_HASH_PATTERN, isSameStorageValidator, sha256Hex } from './content-hash.js';
|
|
11
|
+
// FIPS 180-2 test vectors.
|
|
12
|
+
const ABC_SHA256 = 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad';
|
|
13
|
+
const EMPTY_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855';
|
|
14
|
+
describe('content hash helpers', () => {
|
|
15
|
+
it('computes SHA-256 as lower-case hex', async () => {
|
|
16
|
+
expect(await sha256Hex(new TextEncoder().encode('abc'))).toBe(ABC_SHA256);
|
|
17
|
+
expect(await sha256Hex(new Uint8Array())).toBe(EMPTY_SHA256);
|
|
18
|
+
});
|
|
19
|
+
it('hashes only the viewed bytes of a sliced Buffer', async () => {
|
|
20
|
+
const pooled = Buffer.from('xxabcxx');
|
|
21
|
+
expect(await sha256Hex(pooled.subarray(2, 5))).toBe(ABC_SHA256);
|
|
22
|
+
});
|
|
23
|
+
it('compares validators by exact kind and value', () => {
|
|
24
|
+
const a = { kind: 's3-etag', value: 'abc' };
|
|
25
|
+
expect(isSameStorageValidator(a, { kind: 's3-etag', value: 'abc' })).toBe(true);
|
|
26
|
+
expect(isSameStorageValidator(a, { kind: 's3-etag', value: 'abd' })).toBe(false);
|
|
27
|
+
expect(isSameStorageValidator(a, { kind: 's3-version', value: 'abc' })).toBe(false);
|
|
28
|
+
});
|
|
29
|
+
it('matches only 64 lower-case hex characters', () => {
|
|
30
|
+
expect(CONTENT_HASH_PATTERN.test(ABC_SHA256)).toBe(true);
|
|
31
|
+
expect(CONTENT_HASH_PATTERN.test(ABC_SHA256.toUpperCase())).toBe(false);
|
|
32
|
+
expect(CONTENT_HASH_PATTERN.test(ABC_SHA256.slice(1))).toBe(false);
|
|
33
|
+
expect(CONTENT_HASH_PATTERN.test('')).toBe(false);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
describe('stored-file Zod rules', () => {
|
|
37
|
+
const schema = fieldToZodSchema({ name: 'file', label: 'File', type: 'file' });
|
|
38
|
+
const base = {
|
|
39
|
+
fileId: 'f1',
|
|
40
|
+
filename: 'a.pdf',
|
|
41
|
+
originalFilename: 'a.pdf',
|
|
42
|
+
mimeType: 'application/pdf',
|
|
43
|
+
fileSize: 3,
|
|
44
|
+
storageProvider: 'local',
|
|
45
|
+
storagePath: 'media/a.pdf',
|
|
46
|
+
processingStatus: 'complete',
|
|
47
|
+
};
|
|
48
|
+
it('accepts an absent, null or well-formed hash and validator', () => {
|
|
49
|
+
expect(schema.safeParse(base).success).toBe(true);
|
|
50
|
+
expect(schema.safeParse({ ...base, fileHash: null, storageValidator: null }).success).toBe(true);
|
|
51
|
+
expect(schema.safeParse({
|
|
52
|
+
...base,
|
|
53
|
+
fileHash: ABC_SHA256,
|
|
54
|
+
storageValidator: { kind: 's3-etag', value: 'abc' },
|
|
55
|
+
}).success).toBe(true);
|
|
56
|
+
});
|
|
57
|
+
it('rejects an empty or malformed hash', () => {
|
|
58
|
+
expect(schema.safeParse({ ...base, fileHash: '' }).success).toBe(false);
|
|
59
|
+
expect(schema.safeParse({ ...base, fileHash: ABC_SHA256.toUpperCase() }).success).toBe(false);
|
|
60
|
+
expect(schema.safeParse({ ...base, fileHash: ABC_SHA256.slice(2) }).success).toBe(false);
|
|
61
|
+
});
|
|
62
|
+
it('rejects an empty or oversized validator', () => {
|
|
63
|
+
expect(schema.safeParse({ ...base, storageValidator: { kind: '', value: 'x' } }).success).toBe(false);
|
|
64
|
+
expect(schema.safeParse({ ...base, storageValidator: { kind: 'k', value: '' } }).success).toBe(false);
|
|
65
|
+
expect(schema.safeParse({ ...base, storageValidator: { kind: 'k'.repeat(33), value: 'x' } }).success).toBe(false);
|
|
66
|
+
});
|
|
67
|
+
});
|
package/dist/storage/index.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* Copyright (c) Infonomic Company Limited
|
|
7
7
|
*/
|
|
8
8
|
export { fingerprintCollection } from './collection-fingerprint.js';
|
|
9
|
+
export { CONTENT_HASH_PATTERN, isSameStorageValidator } from './content-hash.js';
|
|
9
10
|
export { assertDocumentVersionParent, type DocumentVersionParentAssertion, type DocumentVersionParentConflictReason, } from './document-version-parent.js';
|
|
10
11
|
export { ALL_STORE_TYPES, type FieldStoreKind, type FieldStoreMapping, fieldTypeToStore, fieldTypeToStoreType, type StoreType, } from './field-store-map.js';
|
|
11
12
|
export { deriveVersionLocaleLedger, resolveLocaleVisibility, resolveReadLocale, type VersionLocaleLedger, } from './locale-resolution.js';
|
package/dist/storage/index.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* Copyright (c) Infonomic Company Limited
|
|
7
7
|
*/
|
|
8
8
|
export { fingerprintCollection } from './collection-fingerprint.js';
|
|
9
|
+
export { CONTENT_HASH_PATTERN, isSameStorageValidator } from './content-hash.js';
|
|
9
10
|
export { assertDocumentVersionParent, } from './document-version-parent.js';
|
|
10
11
|
export { ALL_STORE_TYPES, fieldTypeToStore, fieldTypeToStoreType, } from './field-store-map.js';
|
|
11
12
|
export { deriveVersionLocaleLedger, resolveLocaleVisibility, resolveReadLocale, } from './locale-resolution.js';
|